diff --git a/README.md b/README.md index 0717d76..b57ea2f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ or as a WSGI application (on Apache for instance). But first: * simplejson (`apt-get install python-simplejson`) * [requests](http://docs.python-requests.org/) >= 1.0.0 (`pip install requests`) * [mutagen](https://code.google.com/p/mutagen/) (`apt-get install python-mutagen`) +* [watchdog](https://github.com/gorakhargosh/watchdog) (`pip install watchdog`) ### Configuration @@ -41,16 +42,21 @@ Available settings are: * **database_uri**: required, a Storm [database URI](https://storm.canonical.com/Manual#Databases). I personally use SQLite (`sqlite:////var/supysonic/supysonic.db`), but it might not be the brightest idea for large libraries. Note that to use PostgreSQL you'll need *psycopg2* version 2.4 (not 2.5!) or [patch storm](https://bugs.launchpad.net/storm/+bug/1170063). - * **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `/supysonic`. - * **log_file**: path and base name of a rolling log file. * **scanner_extensions**: space-separated list of file extensions the scanner is restricted to. If omitted, files will be scanned regardless of their extension +* Section **webapp** + * **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `/supysonic`. + * **log_file**: path and base name of a rolling log file. + * **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*. * Section **lastfm**: * **api_key**: Last.FM [API key](http://www.last.fm/api/accounts) to enable scrobbling * **secret**: Last.FM API secret matching the key. * Section **transcoding**: see [Transcoding](https://github.com/spl0k/supysonic/wiki/Transcoding) * Section **mimetypes**: extension to content-type mappings. Designed to help the system guess types, to help clients relying on the content-type. See [the list of common types](https://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types). +* Section **daemon** + * **log_file**: path and base name of a rolling log file. + * **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*. ### Database initialization diff --git a/bin/supysonic-cli b/bin/supysonic-cli index 829156f..5c2a704 100755 --- a/bin/supysonic-cli +++ b/bin/supysonic-cli @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # coding: utf-8 # This file is part of Supysonic. @@ -141,18 +141,19 @@ class CLI(cmd.Cmd): print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total) self.__last_display = time.time() - s = Scanner(self.__store) + scanner = Scanner(self.__store) if folders: folders = map(lambda n: self.__store.find(Folder, Folder.name == n, Folder.root == True).one() or n, folders) if any(map(lambda f: isinstance(f, basestring), folders)): print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring)) for folder in filter(lambda f: isinstance(f, Folder), folders): - FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name)) + scanner.scan(folder, TimedProgressDisplay(folder.name)) else: for folder in self.__store.find(Folder, Folder.root == True): - FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name)) + scanner.scan(folder, TimedProgressDisplay(folder.name)) - added, deleted = s.stats() + scanner.finish() + added, deleted = scanner.stats() self.__store.commit() print "Scanning done" diff --git a/bin/supysonic-watcher b/bin/supysonic-watcher new file mode 100755 index 0000000..33d5b7e --- /dev/null +++ b/bin/supysonic-watcher @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# coding: utf-8 + +# This file is part of Supysonic. +# +# Supysonic is a Python implementation of the Subsonic server API. +# Copyright (C) 2014 Alban 'spl0k' Féron +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from daemon.runner import DaemonRunner +from supysonic.watcher import SupysonicWatcher + +watcher = SupysonicWatcher() +watcher.stdin_path = '/dev/null' +watcher.stdout_path = '/dev/tty' +watcher.stderr_path = '/dev/tty' +watcher.pidfile_path = '/tmp/supysonic-watcher.pid' +watcher.pidfile_timeout = 5 + +daemon_runner = DaemonRunner(watcher) +daemon_runner.do_action() + diff --git a/setup.py b/setup.py index e8d85ea..565211d 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setup(name='supysonic', """, packages=['supysonic', 'supysonic.api', 'supysonic.frontend', 'supysonic.managers'], - scripts=['bin/supysonic-cli'], + scripts=['bin/supysonic-cli', 'bin/supysonic-watcher'], package_data={'supysonic': ['templates/*.html']} ) diff --git a/supysonic/api/media.py b/supysonic/api/media.py index 4554824..1d87b11 100644 --- a/supysonic/api/media.py +++ b/supysonic/api/media.py @@ -151,7 +151,7 @@ def cover_art(): if size > im.size[0] and size > im.size[1]: return send_file(os.path.join(res.path, 'cover.jpg')) - size_path = os.path.join(config.get('base', 'cache_dir'), str(size)) + size_path = os.path.join(config.get('webapp', 'cache_dir'), str(size)) path = os.path.join(size_path, str(res.id)) if os.path.exists(path): return send_file(path) diff --git a/supysonic/frontend/folder.py b/supysonic/frontend/folder.py index 898ddf8..5a40ed1 100644 --- a/supysonic/frontend/folder.py +++ b/supysonic/frontend/folder.py @@ -84,17 +84,19 @@ def del_folder(id): @app.route('/folder/scan') @app.route('/folder/scan/') def scan_folder(id = None): - s = Scanner(store) + scanner = Scanner(store) if id is None: for folder in store.find(Folder, Folder.root == True): - FolderManager.scan(store, folder.id, s) + scanner.scan(folder) else: - status = FolderManager.scan(store, id, s) + status, folder = FolderManager.get(store, id) if status != FolderManager.SUCCESS: flash(FolderManager.error_str(status)) return redirect(url_for('folder_index')) + scanner.scan(folder) - added, deleted = s.stats() + scanner.finish() + added, deleted = scanner.stats() store.commit() flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2])) diff --git a/supysonic/managers/folder.py b/supysonic/managers/folder.py index 9ac31b3..d92c288 100644 --- a/supysonic/managers/folder.py +++ b/supysonic/managers/folder.py @@ -20,6 +20,7 @@ import os.path, uuid from supysonic.db import Folder, Artist, Album, Track +from supysonic.scanner import Scanner class FolderManager: SUCCESS = 0 @@ -77,24 +78,11 @@ class FolderManager: if not folder.root: return FolderManager.NO_SUCH_FOLDER - # delete associated tracks and prune empty albums/artists - potentially_removed_albums = set() + scanner = Scanner(store) for track in store.find(Track, Track.root_folder_id == folder.id): - potentially_removed_albums.add(track.album) - store.remove(track) - potentially_removed_artists = set() - for album in filter(lambda album: album.tracks.count() == 0, potentially_removed_albums): - potentially_removed_artists.add(album.artist) - store.remove(album) - for artist in filter(lambda artist: artist.albums.count() == 0, potentially_removed_artists): - store.remove(artist) - - def cleanup_folder(folder): - for f in folder.children: - cleanup_folder(f) - store.remove(folder) - - cleanup_folder(folder) + scanner.remove_file(track.path) + scanner.finish() + store.remove(folder) store.commit() return FolderManager.SUCCESS @@ -106,17 +94,6 @@ class FolderManager: return FolderManager.NO_SUCH_FOLDER return FolderManager.delete(store, folder.id) - @staticmethod - def scan(store, uid, scanner, progress_callback = None): - status, folder = FolderManager.get(store, uid) - if status != FolderManager.SUCCESS: - return status - - scanner.scan(folder, progress_callback) - scanner.prune(folder) - scanner.check_cover_art(folder) - return FolderManager.SUCCESS - @staticmethod def error_str(err): if err == FolderManager.SUCCESS: diff --git a/supysonic/scanner.py b/supysonic/scanner.py index c1e2bd2..72e3a28 100644 --- a/supysonic/scanner.py +++ b/supysonic/scanner.py @@ -21,6 +21,7 @@ import os, os.path import time, mimetypes import mutagen +from storm.expr import Like, SQL from supysonic import config from supysonic.db import Folder, Artist, Album, Track @@ -41,43 +42,59 @@ class Scanner: extensions = config.get('base', 'scanner_extensions') self.__extensions = map(str.lower, extensions.split()) if extensions else None + self.__folders_to_check = set() + self.__artists_to_check = set() + self.__albums_to_check = set() + + def __del__(self): + if self.__folders_to_check or self.__artists_to_check or self.__albums_to_check: + raise Exception("There's still something to check. Did you run Scanner.finish()?") + def scan(self, folder, progress_callback = None): + # Scan new/updated files files = [ os.path.join(root, f) for root, _, fs in os.walk(folder.path) for f in fs if self.__is_valid_path(os.path.join(root, f)) ] total = len(files) current = 0 for path in files: - self.__scan_file(path, folder) + self.scan_file(path) current += 1 if progress_callback: progress_callback(current, total) + # Remove files that have been deleted + for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]: + self.remove_file(track.path) + + # Update cover art info + folders = [ folder ] + while folders: + f = folders.pop() + f.has_cover_art = os.path.isfile(os.path.join(f.path, 'cover.jpg')) + folders += f.children + folder.last_scan = int(time.time()) - self.__store.flush() - - def prune(self, folder): - for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]: - self.__store.remove(track) - self.__deleted_tracks += 1 - - # TODO execute the conditional part on SQL - for album in [ a for a in self.__store.find(Album) if a.tracks.count() == 0 ]: + def finish(self): + for album in [ a for a in self.__albums_to_check if not a.tracks.count() ]: + self.__artists_to_check.add(album.artist) self.__store.remove(album) self.__deleted_albums += 1 + self.__albums_to_check.clear() - # TODO execute the conditional part on SQL - for artist in [ a for a in self.__store.find(Artist) if a.albums.count() == 0 ]: + for artist in [ a for a in self.__artists_to_check if not a.albums.count() ]: self.__store.remove(artist) self.__deleted_artists += 1 + self.__artists_to_check.clear() - self.__cleanup_folder(folder) - self.__store.flush() + while self.__folders_to_check: + folder = self.__folders_to_check.pop() + if folder.root: + continue - def check_cover_art(self, folder): - folder.has_cover_art = os.path.isfile(os.path.join(folder.path, 'cover.jpg')) - for f in folder.children: - self.check_cover_art(f) + if not folder.tracks.count() and not folder.children.count(): + self.__folders_to_check.add(folder.parent) + self.__store.remove(folder) def __is_valid_path(self, path): if not os.path.exists(path): @@ -86,7 +103,7 @@ class Scanner: return True return os.path.splitext(path)[1][1:].lower() in self.__extensions - def __scan_file(self, path, folder): + def scan_file(self, path): tr = self.__store.find(Track, Track.path == path).one() add = False if tr: @@ -95,8 +112,7 @@ class Scanner: tag = self.__try_load_tag(path) if not tag: - self.__store.remove(tr) - self.__deleted_tracks += 1 + self.remove_file(path) return else: tag = self.__try_load_tag(path) @@ -121,17 +137,46 @@ class Scanner: if add: tralbum = self.__find_album(self.__try_read_tag(tag, 'artist', ''), self.__try_read_tag(tag, 'album', '')) - trfolder = self.__find_folder(path, folder) + trroot = self.__find_root_folder(path) + trfolder = self.__find_folder(path) # Set the references at the very last as searching for them will cause the added track to be flushed, even if # it is incomplete, causing not null constraints errors. tr.album = tralbum tr.folder = trfolder - tr.root_folder = folder + tr.root_folder = trroot self.__store.add(tr) self.__added_tracks += 1 + def remove_file(self, path): + tr = self.__store.find(Track, Track.path == path).one() + if not tr: + return + + self.__folders_to_check.add(tr.folder) + self.__albums_to_check.add(tr.album) + self.__store.remove(tr) + self.__deleted_tracks += 1 + + def move_file(self, src_path, dst_path): + tr = self.__store.find(Track, Track.path == src_path).one() + if not tr: + return + + self.__folders_to_check.add(tr.folder) + tr_dst = self.__store.find(Track, Track.path == dst_path).one() + if tr_dst: + tr.root_folder = tr_dst.root_folder + tr.folder = tr_dst.folder + self.remove_file(dst_path) + else: + root = self.__find_root_folder(dst_path) + folder = self.__find_folder(dst_path) + tr.root_folder = root + tr.folder = folder + tr.path = dst_path + def __find_album(self, artist, album): ar = self.__find_artist(artist) al = ar.albums.find(name = album).one() @@ -160,26 +205,40 @@ class Scanner: return ar - def __find_folder(self, path, folder): + def __find_root_folder(self, path): path = os.path.dirname(path) - fold = self.__store.find(Folder, Folder.path == path).one() - if fold: - return fold + folders = self.__store.find(Folder, Like(path, SQL("folder.path||'%'")), Folder.root == True) + count = folders.count() + if count > 1: + raise Exception("Found multiple root folders for '{}'.".format(path)) + elif count == 0: + raise Exception("Couldn't find the root folder for '{}'.\nDon't scan files that aren't located in a defined music folder".format(path)) + return folders.one() + + def __find_folder(self, path): + path = os.path.dirname(path) + folders = self.__store.find(Folder, Folder.path == path) + count = folders.count() + if count > 1: + raise Exception("Found multiple folders for '{}'.".format(path)) + elif count == 1: + return folders.one() + + folder = self.__store.find(Folder, Like(path, SQL("folder.path||'%'"))).order_by(Folder.path).last() full_path = folder.path path = path[len(folder.path) + 1:] for name in path.split(os.sep): full_path = os.path.join(full_path, name) - fold = self.__store.find(Folder, Folder.path == full_path).one() - if not fold: - fold = Folder() - fold.root = False - fold.name = name - fold.path = full_path - fold.parent = folder - self.__store.add(fold) + fold = Folder() + fold.root = False + fold.name = name + fold.path = full_path + fold.parent = folder + + self.__store.add(fold) folder = fold @@ -202,12 +261,6 @@ class Scanner: except: return default - def __cleanup_folder(self, folder): - for f in folder.children: - self.__cleanup_folder(f) - if folder.children.count() == 0 and folder.tracks.count() == 0 and not folder.root: - self.__store.remove(folder) - def stats(self): return (self.__added_artists, self.__added_albums, self.__added_tracks), (self.__deleted_artists, self.__deleted_albums, self.__deleted_tracks) diff --git a/supysonic/watcher.py b/supysonic/watcher.py new file mode 100644 index 0000000..e502ff7 --- /dev/null +++ b/supysonic/watcher.py @@ -0,0 +1,259 @@ +# coding: utf-8 + +# This file is part of Supysonic. +# +# Supysonic is a Python implementation of the Subsonic server API. +# Copyright (C) 2014 Alban 'spl0k' Féron +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import time +import logging +from signal import signal, SIGTERM +from threading import Thread, Condition, Timer +from logging.handlers import TimedRotatingFileHandler +from watchdog.observers import Observer +from watchdog.events import PatternMatchingEventHandler + +from supysonic import config, db +from supysonic.scanner import Scanner + +OP_SCAN = 1 +OP_REMOVE = 2 +OP_MOVE = 4 + +class SupysonicWatcherEventHandler(PatternMatchingEventHandler): + def __init__(self, queue, logger): + extensions = config.get('base', 'scanner_extensions') + patterns = map(lambda e: "*." + e.lower(), extensions.split()) if extensions else None + super(SupysonicWatcherEventHandler, self).__init__(patterns = patterns, ignore_directories = True) + + self.__queue = queue + self.__logger = logger + + def dispatch(self, event): + try: + super(SupysonicWatcherEventHandler, self).dispatch(event) + except Exception, e: + self.__logger.critical(e) + + def on_created(self, event): + self.__logger.debug("File created: '%s'", event.src_path) + self.__queue.put(event.src_path, OP_SCAN) + + def on_deleted(self, event): + self.__logger.debug("File deleted: '%s'", event.src_path) + self.__queue.put(event.src_path, OP_REMOVE) + + def on_modified(self, event): + self.__logger.debug("File modified: '%s'", event.src_path) + self.__queue.put(event.src_path, OP_SCAN) + + def on_moved(self, event): + self.__logger.debug("File moved: '%s' -> '%s'", event.src_path, event.dest_path) + self.__queue.put(event.dest_path, OP_MOVE, src_path = event.src_path) + +class Event(object): + def __init__(self, path, operation, **kwargs): + if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE): + raise Exception("Flags SCAN and REMOVE both set") + + self.__path = path + self.__time = time.time() + self.__op = operation + self.__src = kwargs.get("src_path") + + def set(self, operation, **kwargs): + if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE): + raise Exception("Flags SCAN and REMOVE both set") + + self.__time = time.time() + if operation & OP_SCAN: + self.__op &= ~OP_REMOVE + if operation & OP_REMOVE: + self.__op &= ~OP_SCAN + self.__op |= operation + + src_path = kwargs.get("src_path") + if src_path: + self.__src = src_path + + @property + def path(self): + return self.__path + + @property + def time(self): + return self.__time + + @property + def operation(self): + return self.__op + + @property + def src_path(self): + return self.__src + +class ScannerProcessingQueue(Thread): + def __init__(self, logger): + super(ScannerProcessingQueue, self).__init__() + + self.__logger = logger + self.__cond = Condition() + self.__timer = None + self.__queue = {} + self.__running = True + + def run(self): + try: + self.__run() + except Exception, e: + self.__logger.critical(e) + + def __run(self): + while self.__running: + time.sleep(0.1) + + with self.__cond: + self.__cond.wait() + + if not self.__queue: + continue + + self.__logger.debug("Instantiating scanner") + store = db.get_store(config.get('base', 'database_uri')) + scanner = Scanner(store) + + item = self.__next_item() + while item: + if item.operation & OP_MOVE: + self.__logger.info("Moving: '%s' -> '%s'", item.src_path, item.path) + scanner.move_file(item.src_path, item.path) + if item.operation & OP_SCAN: + self.__logger.info("Scanning: '%s'", item.path) + scanner.scan_file(item.path) + if item.operation & OP_REMOVE: + self.__logger.info("Removing: '%s'", item.path) + scanner.remove_file(item.path) + item = self.__next_item() + + scanner.finish() + store.commit() + store.close() + self.__logger.debug("Freeing scanner") + del scanner + + def stop(self): + self.__running = False + with self.__cond: + self.__cond.notify() + + def put(self, path, operation, **kwargs): + if not self.__running: + raise RuntimeError("Trying to put an item in a stopped queue") + + with self.__cond: + if path in self.__queue: + event = self.__queue[path] + event.set(operation, **kwargs) + else: + event = Event(path, operation, **kwargs) + self.__queue[path] = event + + if operation & OP_MOVE and kwargs["src_path"] in self.__queue: + previous = self.__queue[kwargs["src_path"]] + event.set(previous.operation, src_path = previous.src_path) + del self.__queue[kwargs["src_path"]] + + if self.__timer: + self.__timer.cancel() + self.__timer = Timer(5, self.__wakeup) + self.__timer.start() + + def __wakeup(self): + with self.__cond: + self.__cond.notify() + self.__timer = None + + def __next_item(self): + with self.__cond: + if not self.__queue: + return None + + next = min(self.__queue.iteritems(), key = lambda i: i[1].time) + if not self.__running or next[1].time + 5 <= time.time(): + del self.__queue[next[0]] + return next[1] + + return None + +class SupysonicWatcher(object): + def run(self): + if not config.check(): + return + + logger = logging.getLogger(__name__) + if config.get('daemon', 'log_file'): + log_handler = TimedRotatingFileHandler(config.get('daemon', 'log_file'), when = 'midnight') + else: + log_handler = logging.NullHandler() + log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(log_handler) + if config.get('daemon', 'log_level'): + mapping = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRTICAL': logging.CRITICAL + } + logger.setLevel(mapping.get(config.get('daemon', 'log_level').upper(), logging.NOTSET)) + + store = db.get_store(config.get('base', 'database_uri')) + folders = store.find(db.Folder, db.Folder.root == True) + + if not folders.count(): + logger.info("No folder set. Exiting.") + store.close() + return + + queue = ScannerProcessingQueue(logger) + handler = SupysonicWatcherEventHandler(queue, logger) + observer = Observer() + + for folder in folders: + logger.info("Starting watcher for %s", folder.path) + observer.schedule(handler, folder.path, recursive = True) + + store.close() + signal(SIGTERM, self.__terminate) + + self.__running = True + queue.start() + observer.start() + while self.__running: + time.sleep(2) + + logger.info("Stopping watcher") + observer.stop() + observer.join() + queue.stop() + queue.join() + + def stop(self): + self.__running = False + + def __terminate(self, signum, frame): + self.stop() + diff --git a/supysonic/web.py b/supysonic/web.py index 167cca8..8c35aac 100644 --- a/supysonic/web.py +++ b/supysonic/web.py @@ -45,19 +45,27 @@ def create_application(): if not config.check(): return None - if not os.path.exists(config.get('base', 'cache_dir')): - os.makedirs(config.get('base', 'cache_dir')) + if not os.path.exists(config.get('webapp', 'cache_dir')): + os.makedirs(config.get('webapp', 'cache_dir')) app = Flask(__name__) app.secret_key = '?9huDM\\H' app.teardown_appcontext(teardown_db) - if config.get('base', 'log_file'): + if config.get('webapp', 'log_file'): import logging from logging.handlers import TimedRotatingFileHandler - handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight') - handler.setLevel(logging.WARNING) + handler = TimedRotatingFileHandler(config.get('webapp', 'log_file'), when = 'midnight') + if config.get('webapp', 'log_level'): + mapping = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRTICAL': logging.CRITICAL + } + handler.setLevel(mapping.get(config.get('webapp', 'log_level').upper(), logging.NOTSET)) app.logger.addHandler(handler) from supysonic import frontend