diff --git a/.gitignore b/.gitignore index d4e16ab..97976e2 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.swp start_server.sh +*.log diff --git a/api/browse.py b/api/browse.py index df1b691..20ddff7 100755 --- a/api/browse.py +++ b/api/browse.py @@ -5,6 +5,7 @@ from web import app from db import Folder, Artist, Album, Track from api import get_entity import uuid, time, string +import os.path @app.route('/rest/getMusicFolders.view', methods = [ 'GET', 'POST' ]) def list_folders(): @@ -89,6 +90,8 @@ def show_directory(): if not status: return res + res.tracks = [t for t in res.tracks if os.path.isfile(t.path)] + directory = { 'id': str(res.id), 'name': res.name, diff --git a/api/media.py b/api/media.py index e345737..d8a729f 100755 --- a/api/media.py +++ b/api/media.py @@ -6,12 +6,28 @@ from PIL import Image import subprocess import shlex import mutagen +import fnmatch +import mimetypes import config, scanner from web import app from db import Track, Folder, User, now, session from api import get_entity +from flask import g + +def after_this_request(func): + if not hasattr(g, 'call_after_request'): + g.call_after_request = [] + g.call_after_request.append(func) + return func + +@app.after_request +def per_request_callbacks(response): + for func in getattr(g, 'call_after_request', ()): + response = func(response) + return response + def prepare_transcoding_cmdline(base_cmdline, input_file, input_format, output_format, output_bitrate): if not base_cmdline: return None @@ -27,7 +43,7 @@ def stream_media(): status, res = get_entity(request, Track) if not status: - return res + return res maxBitRate, format, timeOffset, size, estimateContentLength = map(request.args.get, [ 'maxBitRate', 'format', 'timeOffset', 'size', 'estimateContentLength' ]) if format: @@ -35,9 +51,9 @@ def stream_media(): do_transcoding = False src_suffix = res.suffix() - dst_suffix = res.suffix() + dst_suffix = src_suffix dst_bitrate = res.bitrate - dst_mimetype = res.content_type + dst_mimetype = mimetypes.guess_type('a.' + src_suffix) if format != 'raw': # That's from API 1.9.0 but whatever if maxBitRate: @@ -53,17 +69,16 @@ def stream_media(): if format and format != src_suffix: do_transcoding = True dst_suffix = format - dst_mimetype = scanner.get_mime(dst_suffix) + dst_mimetype = mimetypes.guess_type(dst_suffix) if not format and src_suffix == 'flac': dst_suffix = 'ogg' dst_bitrate = 320 - dst_mimetype = scanner.get_mime(dst_suffix) + dst_mimetype = 'audio/ogg' do_transcoding = True - app.logger.debug('Serving file: ' + res.path) duration = mutagen.File(res.path).info.length - app.logger.debug('\tDuration of file: ' + str(duration)) + app.logger.debug('Serving file: ' + res.path + '\n\tDuration of file: ' + str(duration)) if do_transcoding: transcoder = config.get('transcoding', 'transcoder_{}_{}'.format(src_suffix, dst_suffix)) @@ -82,9 +97,7 @@ def stream_media(): encoder = map(lambda s: s.decode('UTF8'), shlex.split(encoder.encode('utf8'))) transcoder = map(lambda s: s.decode('UTF8'), shlex.split(transcoder.encode('utf8'))) - app.logger.debug(decoder) - app.logger.debug(encoder) - app.logger.debug(transcoder) + app.logger.debug(str( decoder ) + '\n' + str( encoder ) + '\n' + str(transcoder)) if '|' in transcoder: pipe_index = transcoder.index('|') @@ -92,8 +105,7 @@ def stream_media(): encoder = transcoder[pipe_index+1:] transcoder = None - app.logger.warn('decoder' + str(decoder)) - app.logger.warn('encoder' + str(encoder)) + app.logger.warn('decoder' + str(decoder) + '\nencoder' + str(encoder)) try: if transcoder: @@ -109,8 +121,14 @@ def stream_media(): else: app.logger.warn('no transcode') - response = send_file(res.path, mimetype = dst_mimetype) + response = send_file(res.path) + response.headers['Content-Type'] = dst_mimetype + response.headers['Accept-Ranges'] = 'bytes' response.headers['X-Content-Duration'] = str(duration) + redirect = config.get('base', 'accel-redirect') + if(redirect): + response.headers['X-Accel-Redirect'] = redirect + res.path + app.logger.debug('X-Accel-Redirect: ' + response.headers['X-Accel-Redirect']) res.play_count = res.play_count + 1 res.last_play = now() @@ -130,13 +148,35 @@ def download_media(): @app.route('/rest/getCoverArt.view', methods = [ 'GET', 'POST' ]) def cover_art(): + @after_this_request + def add_header(response): + if 'X-Sendfile' in response.headers: + redirect = response.headers['X-Sendfile'] or '' + xsendfile = config.get('base', 'accel-redirect') + if redirect and xsendfile: + response.headers['X-Accel-Redirect'] = xsendfile + redirect + app.logger.debug('X-Accel-Redirect: ' + xsendfile + redirect) + return response + status, res = get_entity(request, Folder) + if not status: return res - if not res.has_cover_art or not os.path.isfile(os.path.join(res.path, 'cover.jpg')): + app.logger.debug('Cover Art Check: ' + res.path + '/*.jp*g') + + coverfile = os.listdir(res.path) + coverfile = fnmatch.filter(coverfile, '*.jp*g') + app.logger.debug('Found Images: ' + str(coverfile)) + + if not coverfile: + app.logger.debug('No Art Found!') + res.has_cover_art = False + session.commit() return request.error_formatter(70, 'Cover art not found') + coverfile = coverfile[0] + size = request.args.get('size') if size: try: @@ -144,20 +184,25 @@ def cover_art(): except: return request.error_formatter(0, 'Invalid size value') else: - return send_file(os.path.join(res.path, 'cover.jpg')) + app.logger.debug('Serving cover art: ' + res.path + coverfile) + return send_file(os.path.join(res.path, coverfile)) - im = Image.open(os.path.join(res.path, 'cover.jpg')) + im = Image.open(os.path.join(res.path, coverfile)) if size > im.size[0] and size > im.size[1]: - return send_file(os.path.join(res.path, 'cover.jpg')) + app.logger.debug('Serving cover art: ' + res.path + coverfile) + return send_file(os.path.join(res.path, coverfile)) size_path = os.path.join(config.get('base', 'cache_dir'), str(size)) path = os.path.join(size_path, str(res.id)) if os.path.exists(path): + app.logger.debug('Serving cover art: ' + path) return send_file(path) if not os.path.exists(size_path): os.makedirs(size_path) im.thumbnail([size, size], Image.ANTIALIAS) im.save(path, 'JPEG') + + app.logger.debug('Serving cover art: ' + path) return send_file(path) diff --git a/db.py b/db.py index d0d3700..389148d 100755 --- a/db.py +++ b/db.py @@ -13,44 +13,13 @@ from sqlalchemy.dialects.postgresql import UUID as pgUUID import uuid, datetime, time import os.path -def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): - cache = getattr(session, '_unique_cache', None) - if cache is None: - session._unique_cache = cache = {} +Base = declarative_base() - key = (cls, hashfunc(*arg, **kw)) - if key in cache: - return cache[key] - else: - with session.no_autoflush: - q = session.query(cls) - q = queryfunc(q, *arg, **kw) - obj = q.first() - if not obj: - obj = constructor(*arg, **kw) - session.add(obj) - cache[key] = obj - return obj +engine = create_engine(config.get('base', 'database_uri'), convert_unicode = True, echo = True) -class UniqueMixin(object): - @classmethod - def unique_hash(cls, *arg, **kw): - raise NotImplementedError() +session = scoped_session(sessionmaker(autoflush = False, bind = engine)) - @classmethod - def unique_filter(cls, query, *arg, **kw): - raise NotImplementedError() - - @classmethod - def as_unique(cls, session, *arg, **kw): - return _unique( - session, - cls, - cls.unique_hash, - cls.unique_filter, - cls, - arg, kw - ) +Base.query = session.query_property() class UUID(TypeDecorator): """Platform-somewhat-independent UUID type @@ -96,11 +65,6 @@ class UUID(TypeDecorator): def now(): return datetime.datetime.now().replace(microsecond = 0) -engine = create_engine(config.get('base', 'database_uri'), convert_unicode = True) -session = scoped_session(sessionmaker(autoflush = False, bind = engine)) - -Base = declarative_base() -Base.query = session.query_property() class User(Base): __tablename__ = 'user' @@ -177,21 +141,13 @@ class Folder(Base): return info -class Artist(UniqueMixin, Base): +class Artist(Base): __tablename__ = 'artist' id = UUID.gen_id_column() - name = Column(String(256), unique = True, nullable=False) + name = Column(String(255), nullable=False) albums = relationship('Album', backref = 'artist') - @classmethod - def unique_hash(cls, name): - return name - - @classmethod - def unique_filter(cls, query, name): - return query.filter(Artist.name == name) - def as_subsonic_artist(self, user): info = { 'id': str(self.id), @@ -210,7 +166,7 @@ class Album(Base): __tablename__ = 'album' id = UUID.gen_id_column() - name = Column(String(256)) + name = Column(String(255)) artist_id = Column(UUID, ForeignKey('artist.id')) tracks = relationship('Track', backref = 'album') diff --git a/managers/folder.py b/managers/folder.py index e95a33f..30bad65 100755 --- a/managers/folder.py +++ b/managers/folder.py @@ -93,7 +93,6 @@ class FolderManager: scanner.scan(folder) scanner.prune(folder) - scanner.check_cover_art(folder) return FolderManager.SUCCESS @staticmethod diff --git a/scanner.py b/scanner.py index 05cc0d2..51ad11a 100755 --- a/scanner.py +++ b/scanner.py @@ -1,24 +1,29 @@ # coding: utf-8 -import sys import os, os.path -import time, mimetypes +import time import mutagen import config, db import math +import sys, traceback from web import app -def get_mime(ext): - return mimetypes.guess_type('dummy.' + ext, False)[0] or config.get('mimetypes', ext) or 'application/octet-stream' +from profilehooks import profile class Scanner: def __init__(self, session): self.__session = session + self.__tracks = db.Track.query.all() self.__tracks = {x.path: x for x in self.__tracks} self.__artists = db.Artist.query.all() + self.__artists = {x.name.lower(): x for x in self.__artists} + self.__folders = db.Folder.query.all() + self.__folders = {x.path: x for x in self.__folders} + + self.__playlists = db.Playlist.query.all() self.__added_artists = 0 self.__added_albums = 0 @@ -27,74 +32,59 @@ class Scanner: self.__deleted_albums = 0 self.__deleted_tracks = 0 - extensions = config.get('base', 'scanner_extensions') - self.__extensions = map(str.lower, extensions.split()) if extensions else None - def scan(self, folder): print "scanning", folder.path valid = [x.lower() for x in config.get('base','filetypes').split(',')] + valid = tuple(valid) print "valid filetypes: ",valid - n = 0 for root, subfolders, files in os.walk(folder.path, topdown=False): for f in files: - suffix = os.path.splitext(f)[1][1:].lower() - n += 1 - if n == 1000: - app.logger.debug('commit db') - self.__session.commit() - n = 0 - - if suffix in valid: + if f.lower().endswith(valid): try: - app.logger.debug('Scanning File: ' + os.path.join(root, f)) self.__scan_file(os.path.join(root, f), folder) - self.__session.flush() except: app.logger.error('Problem adding file: ' + os.path.join(root,f)) - app.logger.error(sys.exc_info()) + app.logger.error(traceback.print_exc()) + sys.exit(0) self.__session.rollback() - - + print "\a" + self.__session.add_all(self.__tracks.values()) self.__session.commit() folder.last_scan = int(time.time()) def prune(self, folder): - for k, track in self.__tracks.iteritems(): - if track.root_folder.id == folder.id and not self.__is_valid_path(k): - app.debug('Removed invalid path: ' + k) - self.__remove_track(track) + for path, root_folder_id, track_id in self.__session.query(db.Track.path, db.Track.root_folder_id, db.Track.id): + if root_folder_id == folder.id and not os.path.exists(path): + app.logger.debug('Removed invalid path: ' + path) + self.__remove_track(self.__session.merge(db.Track(id = track_id))) - for album in [ album for artist in self.__artists for album in artist.albums if len(album.tracks) == 0 ]: + self.__session.commit() + + for album in [ album for artist in self.__artists.values() for album in artist.albums if len(album.tracks) == 0 ]: album.artist.albums.remove(album) self.__session.delete(album) self.__deleted_albums += 1 - for artist in [ a for a in self.__artists if len(a.albums) == 0 ]: + self.__session.commit() + + for artist in [ a for a in self.__artists.values() if len(a.albums) == 0 ]: self.__session.delete(artist) self.__deleted_artists += 1 + self.__session.commit() + self.__cleanup_folder(folder) - def check_cover_art(self, folder): - folder.has_cover_art = os.path.isfile(os.path.join(folder.path, 'cover.jpg')) - for f in folder.children: - self.check_cover_art(f) - - def __is_valid_path(self, path): - if not os.path.exists(path): - return False - if not self.__extensions: - return True - return os.path.splitext(path)[1][1:].lower() in self.__extensions - + @profile def __scan_file(self, path, folder): curmtime = int(math.floor(os.path.getmtime(path))) if path in self.__tracks: tr = self.__tracks[path] + app.logger.debug('Existing File: ' + path) if not tr.last_modification: tr.last_modification = curmtime @@ -111,16 +101,16 @@ class Scanner: self.__remove_track(tr) return False else: - app.logger.debug('\tReading tag') + app.logger.debug('Scanning File: ' + path + '\n\tReading tag') tag = self.__try_load_tag(path) if not tag: app.logger.debug('\tProblem reading tag') return False tr = db.Track(path = path, root_folder = folder, folder = self.__find_folder(path, folder)) + self.__tracks[path] = tr self.__added_tracks += 1 - print "Added ", path tr.last_modification = curmtime tr.disc = self.__try_read_tag(tag, 'discnumber', 1, lambda x: int(x[0].split('/')[0])) @@ -129,49 +119,55 @@ class Scanner: tr.year = self.__try_read_tag(tag, 'date', None, lambda x: int(x[0].split('-')[0])) tr.genre = self.__try_read_tag(tag, 'genre') tr.duration = int(tag.info.length) + + # TODO: use album artist if available, then artist, then unknown tr.album = self.__find_album(self.__try_read_tag(tag, 'artist', 'Unknown'), self.__try_read_tag(tag, 'album', 'Unknown')) + tr.bitrate = (tag.info.bitrate if hasattr(tag.info, 'bitrate') else int(os.path.getsize(path) * 8 / tag.info.length)) / 1000 - tr.content_type = get_mime(os.path.splitext(path)[1][1:]) return True def __find_album(self, artist, album): - ar = self.__find_artist(artist) + # TODO : DB specific issues with single column name primary key + # for instance, case sensitivity and trailing spaces + artist = artist.rstrip() - al = filter(lambda a: a.name == album, ar.albums) - if al: - return al[0] + if artist in self.__artists: + ar = self.__artists[artist] + else: + #Flair! + sys.stdout.write('\033[K') + sys.stdout.write('%s\r' % artist) + sys.stdout.flush() + ar = db.Artist(name = artist) + self.__artists[artist] = ar + self.__added_artists += 1 - al = db.Album(name = album, artist = ar) - self.__added_albums += 1 - - return al - - def __find_artist(self, artist): - ar = db.Artist.as_unique(self.__session, name = artist) - - self.__artists.append(ar) - self.__added_artists += 1 - - return ar + al = {a.name: a for a in ar.albums} + if album in al: + return al[album] + else: + self.__added_albums += 1 + return db.Album(name = album, artist = ar) def __find_folder(self, path, folder): - path = os.path.dirname(path) - fold = filter(lambda f: f.path == path, self.__folders) - if fold: - return fold[0] + path = os.path.dirname(path) + if path in self.__folders: + return self.__folders[path] + + # must find parent directory to create new one full_path = folder.path path = path[len(folder.path) + 1:] for name in path.split(os.sep): full_path = os.path.join(full_path, name) - fold = filter(lambda f: f.path == full_path, self.__folders) - if fold: - folder = fold[0] + + if full_path in self.__folders: + folder = self.__folders[full_path] else: folder = db.Folder(root = False, name = name, path = full_path, parent = folder) - self.__folders.append(folder) + self.__folders[full_path] = folder return folder @@ -193,12 +189,12 @@ class Scanner: return default def __remove_track(self, track): - track.album.tracks.remove(track) - track.folder.tracks.remove(track) # As we don't have a track -> playlists relationship, SQLAlchemy doesn't know it has to remove tracks # from playlists as well, so let's help it - for playlist in db.Playlist.query.filter(db.Playlist.tracks.contains(track)): - playlist.tracks.remove(track) + for playlist in self.__playlists: + if track in playlist.tracks: + playlist.tracks.remove(track) + self.__session.delete(track) self.__deleted_tracks += 1 diff --git a/supysonic.ini b/supysonic.ini new file mode 100755 index 0000000..60fb3a3 --- /dev/null +++ b/supysonic.ini @@ -0,0 +1,10 @@ +[uwsgi] + + socket = /tmp/supysonic.sock + wsgi-file = /home/emory/supysonic/main.wsgi + master = true + processes = 4 + threads = 2 + uid = 1000 + vacuum = true + chmod-socket = 666 diff --git a/web.py b/web.py index 6509837..64a2387 100755 --- a/web.py +++ b/web.py @@ -6,13 +6,16 @@ import config app = Flask(__name__) app.secret_key = '?9huDM\\H' +if(config.get('base', 'accel-redirect')): + app.use_x_sendfile = True + if config.get('base', 'debug'): app.debug = True if config.get('base', 'log_file'): import logging from logging.handlers import TimedRotatingFileHandler - handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight') + handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight', encoding = 'UTF-8') handler.setLevel(logging.DEBUG) app.logger.addHandler(handler)