From a4c989c36a14008c1c6fc31d304ed038dfe21289 Mon Sep 17 00:00:00 2001 From: Emory P Date: Sun, 3 Nov 2013 15:46:19 -0500 Subject: [PATCH] fixed up transcoding, transcode flac to ogg by default, sped up folder scan a bit and made artists unique field work properly okay thats good --- api/media.py | 55 ++++++++++++++++++++++++++++++---------------- db.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++------ scanner.py | 41 +++++++++++++++++++---------------- 3 files changed, 114 insertions(+), 43 deletions(-) diff --git a/api/media.py b/api/media.py index 7a0f216..af0eb9f 100755 --- a/api/media.py +++ b/api/media.py @@ -4,6 +4,7 @@ from flask import request, send_file, Response import os.path from PIL import Image import subprocess +import shlex import config, scanner from web import app @@ -13,10 +14,12 @@ from api import get_entity def prepare_transcoding_cmdline(base_cmdline, input_file, input_format, output_format, output_bitrate): if not base_cmdline: return None - ret = base_cmdline.split() - for i in xrange(len(ret)): - ret[i] = ret[i].replace('%srcpath', input_file).replace('%srcfmt', input_format).replace('%outfmt', output_format).replace('%outrate', str(output_bitrate)) - return ret + + return base_cmdline.replace('%srcpath', '"'+input_file+'"').replace('%srcfmt', input_format).replace('%outfmt', output_format).replace('%outrate', str(output_bitrate)) + +def transcode(process): + for chunk in iter(process, ''): + yield chunk @app.route('/rest/stream.view', methods = [ 'GET', 'POST' ]) def stream_media(): @@ -51,37 +54,53 @@ def stream_media(): dst_suffix = format dst_mimetype = scanner.get_mime(dst_suffix) + if not format and src_suffix == 'flac': + dst_suffix = 'ogg' + dst_bitrate = 320 + dst_mimetype = scanner.get_mime(dst_suffix) + do_transcoding = True + if do_transcoding: transcoder = config.get('transcoding', 'transcoder_{}_{}'.format(src_suffix, dst_suffix)) + decoder = config.get('transcoding', 'decoder_' + src_suffix) or config.get('transcoding', 'decoder') encoder = config.get('transcoding', 'encoder_' + dst_suffix) or config.get('transcoding', 'encoder') + if not transcoder and (not decoder or not encoder): transcoder = config.get('transcoding', 'transcoder') if not transcoder: return request.error_formatter(0, 'No way to transcode from {} to {}'.format(src_suffix, dst_suffix)) transcoder, decoder, encoder = map(lambda x: prepare_transcoding_cmdline(x, res.path, src_suffix, dst_suffix, dst_bitrate), [ transcoder, decoder, encoder ]) + + decoder = shlex.split(decoder) + encoder = shlex.split(encoder) + + if '|' in shlex.split(transcoder): + transcoder = shlex.split(transcoder) + pipe_index = transcoder.index('|') + decoder = transcoder[:pipe_index] + encoder = transcoder[pipe_index+1:] + transcoder = None + + try: if transcoder: - proc = subprocess.Popen(transcoder, stdout = subprocess.PIPE) + app.logger.warn('single line transcode: '+transcoder) + proc = subprocess.Popen(shlex.split(transcoder), stdout = subprocess.PIPE, shell=False) else: - dec_proc = subprocess.Popen(decoder, stdout = subprocess.PIPE) - proc = subprocess.Popen(encoder, stdin = dec_proc.stdout, stdout = subprocess.PIPE) + app.logger.warn('multi process transcode: ') + app.logger.warn('decoder' + str(decoder)) + app.logger.warn('encoder' + str(encoder)) + dec_proc = subprocess.Popen(decoder, stdout = subprocess.PIPE, shell=False) + proc = subprocess.Popen(encoder, stdin = dec_proc.stdout, stdout = subprocess.PIPE, shell=False) + + response = Response(transcode(proc.stdout.readline), 200, {'Content-Type': dst_mimetype}) except: return request.error_formatter(0, 'Error while running the transcoding process') - def transcode(): - while True: - data = proc.stdout.read(8192) - if not data: - break - yield data - proc.terminate() - proc.wait() - - - response = Response(transcode(), mimetype = dst_mimetype) else: + app.logger.warn('no transcode') response = send_file(res.path, mimetype = dst_mimetype) res.play_count = res.play_count + 1 diff --git a/db.py b/db.py index 247fa6a..d20779e 100755 --- a/db.py +++ b/db.py @@ -13,6 +13,45 @@ from sqlalchemy.dialects.postgresql import UUID as pgUUID import uuid, datetime, time import os.path +def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): + cache = getattr(session, '_unique_cache', None) + if cache is None: + session._unique_cache = cache = {} + + key = (cls, hashfunc(*arg, **kw)) + if key in cache: + return cache[key] + else: + with session.no_autoflush: + q = session.query(cls) + q = queryfunc(q, *arg, **kw) + obj = q.first() + if not obj: + obj = constructor(*arg, **kw) + session.add(obj) + cache[key] = obj + return obj + +class UniqueMixin(object): + @classmethod + def unique_hash(cls, *arg, **kw): + raise NotImplementedError() + + @classmethod + def unique_filter(cls, query, *arg, **kw): + raise NotImplementedError() + + @classmethod + def as_unique(cls, session, *arg, **kw): + return _unique( + session, + cls, + cls.unique_hash, + cls.unique_filter, + cls, + arg, kw + ) + class UUID(TypeDecorator): """Platform-somewhat-independent UUID type @@ -58,7 +97,7 @@ def now(): return datetime.datetime.now().replace(microsecond = 0) engine = create_engine(config.get('base', 'database_uri'), convert_unicode = True) -session = scoped_session(sessionmaker(autoflush = True, bind = engine)) +session = scoped_session(sessionmaker(autoflush = False, bind = engine)) Base = declarative_base() Base.query = session.query_property() @@ -138,13 +177,21 @@ class Folder(Base): return info -class Artist(Base): +class Artist(UniqueMixin, Base): __tablename__ = 'artist' id = UUID.gen_id_column() - name = Column(String(256), unique = True) + name = Column(String(256), unique = True, nullable=False) albums = relationship('Album', backref = 'artist') + @classmethod + def unique_hash(cls, name): + return name + + @classmethod + def unique_filter(cls, query, name): + return query.filter(Artist.name == name) + def as_subsonic_artist(self, user): info = { 'id': str(self.id), @@ -257,8 +304,9 @@ class Track(Base): if avgRating: info['averageRating'] = avgRating - # transcodedContentType - # transcodedSuffix + if self.suffix() == 'flac': + info['transcodedContentType'] = 'audio/ogg' + info['transcodedSuffix'] = 'ogg' return info @@ -272,7 +320,7 @@ class Track(Base): return os.path.splitext(self.path)[1][1:].lower() def sort_key(self): - return (self.album.artist.name + self.album.name + ("%02i" % self.disc) + ("%02i" % self.number) + self.title).lower() + return (self.album.artist.name + self.album.name + ("%02i" % self.disc) + ("%02i" % self.number) + str(self.title)).lower() class StarredFolder(Base): __tablename__ = 'starred_folder' @@ -389,4 +437,3 @@ def init_db(): def recreate_db(): Base.metadata.drop_all(bind = engine) Base.metadata.create_all(bind = engine) - diff --git a/scanner.py b/scanner.py index 7131417..70aedfb 100755 --- a/scanner.py +++ b/scanner.py @@ -12,6 +12,9 @@ class Scanner: def __init__(self, session): self.__session = session self.__tracks = db.Track.query.all() + paths = {x.path for x in self.__tracks} + self.__tracks = dict(zip(paths,self.__tracks)) + self.__artists = db.Artist.query.all() self.__folders = db.Folder.query.all() @@ -24,11 +27,15 @@ class Scanner: def scan(self, folder): print "scanning", folder.path + valid = [x.lower() for x in config.get('base','filetypes').split(',')] + print "valid filetypes: ",valid + for root, subfolders, files in os.walk(folder.path, topdown=False): - for p in subfolders: - db.session.flush() for f in files: - self.__scan_file(os.path.join(root, f), folder) + suffix = os.path.splitext(f)[1][1:].lower() + if suffix in valid: + self.__scan_file(os.path.join(root, f), folder) + folder.last_scan = int(time.time()) def prune(self, folder): @@ -52,25 +59,24 @@ class Scanner: self.check_cover_art(f) def __scan_file(self, path, folder): - tr = filter(lambda t: t.path == path, self.__tracks) - if tr: - tr = tr[0] + if path in self.__tracks: + tr = self.__tracks[path] if not os.path.getmtime(path) > tr.last_modification: - return + return False tag = self.__try_load_tag(path) if not tag: self.__remove_track(tr) - return + return False else: - print "Added ", path tag = self.__try_load_tag(path) if not tag: - return + return False tr = db.Track(path = path, root_folder = folder, folder = self.__find_folder(path, folder)) - self.__tracks.append(tr) + self.__tracks[path] = tr self.__added_tracks += 1 + print "Added ", path tr.disc = self.__try_read_tag(tag, 'discnumber', 1, lambda x: int(x[0].split('/')[0])) tr.number = self.__try_read_tag(tag, 'tracknumber', 1, lambda x: int(x[0].split('/')[0])) @@ -78,13 +84,16 @@ class Scanner: tr.year = self.__try_read_tag(tag, 'date', None, lambda x: int(x[0].split('-')[0])) tr.genre = self.__try_read_tag(tag, 'genre') tr.duration = int(tag.info.length) - tr.album = self.__find_album(self.__try_read_tag(tag, 'artist'), self.__try_read_tag(tag, 'album')) + tr.album = self.__find_album(self.__try_read_tag(tag, 'artist', 'Unknown'), self.__try_read_tag(tag, 'album', 'Unknown')) tr.bitrate = (tag.info.bitrate if hasattr(tag.info, 'bitrate') else int(os.path.getsize(path) * 8 / tag.info.length)) / 1000 tr.content_type = get_mime(os.path.splitext(path)[1][1:]) tr.last_modification = os.path.getmtime(path) + return True + def __find_album(self, artist, album): ar = self.__find_artist(artist) + al = filter(lambda a: a.name == album, ar.albums) if al: return al[0] @@ -95,13 +104,9 @@ class Scanner: return al def __find_artist(self, artist): - ar = filter(lambda a: a.name.lower() == artist.lower(), self.__artists) - if ar: - return ar[0] + ar = db.Artist.as_unique(self.__session, name = artist) - ar = db.Artist(name = artist) self.__artists.append(ar) - self.__session.add(ar) self.__added_artists += 1 return ar @@ -128,7 +133,7 @@ class Scanner: def __try_load_tag(self, path): try: - return mutagen.File(path, easy = True) + return mutagen.File(path, easy = False) except: return None