From 7bbbdac41c2aecf441a102e7ff53b0178830a4ae Mon Sep 17 00:00:00 2001 From: spl0k Date: Mon, 22 Apr 2019 16:42:38 +0200 Subject: [PATCH] Queue scans rather than rejecting them --- supysonic/cli.py | 7 +--- supysonic/daemon/__init__.py | 68 ++++++++++++++++++++++++---------- supysonic/daemon/client.py | 17 +-------- supysonic/daemon/exceptions.py | 3 -- 4 files changed, 53 insertions(+), 42 deletions(-) diff --git a/supysonic/cli.py b/supysonic/cli.py index 21f3777..e122bf9 100644 --- a/supysonic/cli.py +++ b/supysonic/cli.py @@ -17,7 +17,7 @@ from pony.orm import db_session from pony.orm import ObjectNotFound from .daemon.client import DaemonClient -from .daemon.exceptions import DaemonUnavailableError, ScannerAlreadyRunningError +from .daemon.exceptions import DaemonUnavailableError from .db import Folder, User from .managers.folder import FolderManager from .managers.user import UserManager @@ -180,10 +180,7 @@ class SupysonicCLI(cmd.Cmd): self.__folder_scan_foreground(folders, force) def __folder_scan_background(self, folders, force): - try: - self.__daemon.scan(folders, force) - except ScannerAlreadyRunningError: - self.write_error_line('The daemon is already scanning, please try again later') + self.__daemon.scan(folders, force) def __folder_scan_foreground(self, folders, force): try: diff --git a/supysonic/daemon/__init__.py b/supysonic/daemon/__init__.py index 0fc5c92..2400a0d 100644 --- a/supysonic/daemon/__init__.py +++ b/supysonic/daemon/__init__.py @@ -8,13 +8,16 @@ # Distributed under terms of the GNU AGPLv3 license. import logging +try: + from queue import Queue, Empty +except ImportError: + from Queue import Queue, Empty from multiprocessing.connection import Listener -from pony.orm import db_session +from pony.orm import db_session, select from threading import Thread from .client import DaemonCommand -from .exceptions import ScannerAlreadyRunningError from ..db import Folder from ..scanner import Scanner from ..utils import get_secret_key @@ -55,14 +58,23 @@ class Daemon(object): self.__handle_connection(conn) def start_scan(self, folders = [], force = False): + if not folders: + with db_session: + folders = select(f.name for f in Folder if f.root)[:] + if self.__scanner is not None and self.__scanner.is_alive(): - raise ScannerAlreadyRunningError() + for f in folders: + self.__scanner.queue_folder(f) + return extensions = self.__config.BASE['scanner_extensions'] if extensions: extensions = extensions.split(' ') - self.__scanner = ScannerThread(self.__watcher, folders, kwargs = { 'force': force, 'extensions': extensions, 'notify_watcher': False }) + self.__scanner = ScannerThread(self.__watcher, kwargs = { 'force': force, 'extensions': extensions, 'notify_watcher': False }) + for f in folders: + self.__scanner.queue_folder(f) + self.__scanner.start() def terminate(self): @@ -70,32 +82,50 @@ class Daemon(object): if self.__watcher is not None: self.__watcher.stop() +class ScanQueue(Queue): + def _init(self, maxsize): + self.queue = set() + self.__last_got = None + + def _put(self, item): + if self.__last_got != item: + self.queue.add(item) + + def _get(self): + self.__last_got = self.queue.pop() + return self.__last_got + class ScannerThread(Thread): - def __init__(self, watcher, folders, *args, **kwargs): + def __init__(self, watcher, *args, **kwargs): super(ScannerThread, self).__init__(*args, **kwargs) self.__watcher = watcher - self.__folders = folders self.__scanned = {} + self.__queue = ScanQueue() + + def queue_folder(self, folder): + self.__queue.put(folder) def run(self): s = Scanner(*self._args, **self._kwargs) with db_session: - if self.__folders: - folders = Folder.select(lambda f: f.root and f.name in self.__folders) - else: - folders = Folder.select(lambda f: f.root) + try: + while True: + name = self.__queue.get(False) + folder = Folder.get(root = True, name = name) + if folder is None: + continue - for f in folders: - name = f.name - if self.__watcher is not None: - self.__watcher.remove_folder(f) - try: - logger.info('Scanning %s', name) - s.scan(f, lambda x: self.__scanned.update({ name: x })) - finally: if self.__watcher is not None: - self.__watcher.add_folder(f) + self.__watcher.remove_folder(folder) + try: + logger.info('Scanning %s', name) + s.scan(folder, lambda x: self.__scanned.update({ name: x })) + finally: + if self.__watcher is not None: + self.__watcher.add_folder(folder) + except Empty: + pass s.finish() diff --git a/supysonic/daemon/client.py b/supysonic/daemon/client.py index ed98f8c..7436a7e 100644 --- a/supysonic/daemon/client.py +++ b/supysonic/daemon/client.py @@ -9,7 +9,7 @@ from multiprocessing.connection import Client -from .exceptions import DaemonUnavailableError, ScannerAlreadyRunningError +from .exceptions import DaemonUnavailableError from ..config import get_current_config from ..py23 import strtype from ..utils import get_secret_key @@ -49,11 +49,7 @@ class ScannerStartCommand(ScannerCommand): self.__force = force def apply(self, connection, daemon): - try: - daemon.start_scan(self.__folders, self.__force) - connection.send(ScannerStartResult(None)) - except ScannerAlreadyRunningError as e: - connection.send(ScannerStartResult(e)) + daemon.start_scan(self.__folders, self.__force) class DaemonCommandResult(object): pass @@ -64,12 +60,6 @@ class ScannerProgressResult(DaemonCommandResult): scanned = property(lambda self: self.__scanned) -class ScannerStartResult(DaemonCommandResult): - def __init__(self, exception): - self.__exception = exception - - exception = property(lambda self: self.__exception) - class DaemonClient(object): def __init__(self, address = None): self.__address = address or get_current_config().DAEMON['socket'] @@ -105,6 +95,3 @@ class DaemonClient(object): raise TypeError('Expecting list, got ' + str(type(folders))) with self.__get_connection() as c: c.send(ScannerStartCommand(folders, force)) - rv = c.recv() - if rv.exception is not None: - raise rv.exception diff --git a/supysonic/daemon/exceptions.py b/supysonic/daemon/exceptions.py index c4287ab..3c6f511 100644 --- a/supysonic/daemon/exceptions.py +++ b/supysonic/daemon/exceptions.py @@ -9,6 +9,3 @@ class DaemonUnavailableError(Exception): pass - -class ScannerAlreadyRunningError(Exception): - pass