From 0957fef1487bbac3f9597c5648f04c6758ba7d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alban=20F=C3=A9ron?= Date: Tue, 17 Jan 2023 22:59:43 +0100 Subject: [PATCH] Fix failing deletions from the scanner --- supysonic/db.py | 46 ++++++++++++++++++++---- supysonic/managers/folder.py | 38 ++------------------ supysonic/scanner.py | 26 ++++++++------ tests/base/test_scanner.py | 68 +++++++++++++++++++++++++++++++++++- tests/issue101.py | 49 -------------------------- 5 files changed, 126 insertions(+), 101 deletions(-) delete mode 100644 tests/issue101.py diff --git a/supysonic/db.py b/supysonic/db.py index bc5650d..7b3da49 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -167,18 +167,52 @@ class Folder(PathMixin, _Model): @classmethod def prune(cls): - query = cls.delete().where( + alias = cls.alias() + query = cls.select(cls.id).where( ~cls.root, - cls.id.not_in(Track.select(Track.folder)), - cls.id.not_in(cls.select(cls.parent)), + Track.select(fn.count("*")).where(Track.folder == cls.id) == 0, + alias.select(fn.count("*")).where(alias.parent == cls.id) == 0, ) total = 0 while True: - count = query.execute() - total += count - if not count: + clone = query.clone() # peewee caches the results, clone to force a refetch + for f in clone: + f.delete_instance(recursive=True) + total += 1 + if not len(clone): return total + def delete_hierarchy(self): + if self.root: + cond = Track.root_folder == self + else: + cond = Track.path.startswith(self.path) + + return self.__delete_hierarchy(cond) + + def __delete_hierarchy(self, cond): + users = User.select(User.id).join(Track).where(cond) + User.update(last_play=None).where(User.id.in_(users)).execute() + + tracks = Track.select(Track.id).where(cond) + RatingTrack.delete().where(RatingTrack.rated.in_(tracks)).execute() + StarredTrack.delete().where(StarredTrack.starred.in_(tracks)).execute() + + path_cond = Folder.path.startswith(self.path) + folders = Folder.select(Folder.id).where(path_cond) + RatingFolder.delete().where(RatingFolder.rated.in_(folders)).execute() + StarredFolder.delete().where(StarredFolder.starred.in_(folders)).execute() + + deleted_tracks = Track.delete().where(cond).execute() + + query = Folder.delete().where(path_cond) + if isinstance(db.obj, MySQLDatabase): + # MySQL can't propery resolve deletion order when it has several to handle + query = query.order_by(Folder.path.desc()) + query.execute() + + return deleted_tracks + class Artist(_Model): id = PrimaryKeyField() diff --git a/supysonic/managers/folder.py b/supysonic/managers/folder.py index 27cd9e7..29f9476 100644 --- a/supysonic/managers/folder.py +++ b/supysonic/managers/folder.py @@ -1,27 +1,15 @@ # This file is part of Supysonic. # Supysonic is a Python implementation of the Subsonic server API. # -# Copyright (C) 2013-2022 Alban 'spl0k' Féron +# Copyright (C) 2013-2023 Alban 'spl0k' Féron # # Distributed under terms of the GNU AGPLv3 license. import os.path -from peewee import IntegrityError - from ..daemon.client import DaemonClient from ..daemon.exceptions import DaemonUnavailableError -from ..db import ( - Folder, - Track, - Artist, - Album, - User, - RatingFolder, - RatingTrack, - StarredFolder, - StarredTrack, -) +from ..db import Folder, Artist, Album class FolderManager: @@ -79,29 +67,9 @@ class FolderManager: except DaemonUnavailableError: pass - root_cond = Track.root_folder == folder - users = User.select(User.id).join(Track).where(root_cond) - User.update(last_play=None).where(User.id.in_(users)).execute() - - tracks = Track.select(Track.id).where(root_cond) - RatingTrack.delete().where(RatingTrack.rated.in_(tracks)).execute() - StarredTrack.delete().where(StarredTrack.starred.in_(tracks)).execute() - - path_cond = Folder.path.startswith(folder.path) - folders = Folder.select(Folder.id).where(path_cond) - RatingFolder.delete().where(RatingFolder.rated.in_(folders)).execute() - StarredFolder.delete().where(StarredFolder.starred.in_(folders)).execute() - - Track.delete().where(root_cond).execute() + folder.delete_hierarchy() Album.prune() Artist.prune() - query = Folder.delete().where(path_cond) - try: - query.execute() - except IntegrityError: - # Integrity error most likely due to MySQL poor handling of delete order - query = query.order_by(Folder.path.desc()) - query.execute() @staticmethod def delete_by_name(name): diff --git a/supysonic/scanner.py b/supysonic/scanner.py index a6bbd48..70b582b 100644 --- a/supysonic/scanner.py +++ b/supysonic/scanner.py @@ -141,8 +141,19 @@ class Scanner(Thread): self.__report_progress(folder.name, scanned) + # Remove deleted/moved folders + folders = [folder] + while not self.__stopped.is_set() and folders: + f = folders.pop() + + if not f.root and not os.path.isdir(f.path): + self.__stats.deleted.tracks += f.delete_hierarchy() + continue + + folders += f.children[:] + # Remove files that have been deleted - # Could be more efficient if done above + # Could be more efficient if done when walking on the files if not self.__stopped.is_set(): for track in Track.select().where(Track.root_folder == folder): if not os.path.exists(track.path) or not self.__check_extension( @@ -150,15 +161,10 @@ class Scanner(Thread): ): self.remove_file(track.path) - # Remove deleted/moved folders and update cover art info + # Update cover art info folders = [folder] while not self.__stopped.is_set() and folders: f = folders.pop() - - if not f.root and not os.path.isdir(f.path): - f.delete_instance(recursive=True) - continue - self.find_cover(f.path) folders += f.children[:] @@ -173,8 +179,8 @@ class Scanner(Thread): if self.__stopped.is_set(): return - self.__stats.deleted.albums = Album.prune() - self.__stats.deleted.artists = Artist.prune() + self.__stats.deleted.albums += Album.prune() + self.__stats.deleted.artists += Artist.prune() Folder.prune() def __check_extension(self, path): @@ -272,7 +278,7 @@ class Scanner(Thread): raise TypeError("Expecting string, got " + str(type(path))) try: - Track.get(path=path).delete_instance() + Track.get(path=path).delete_instance(recursive=True) self.__stats.deleted.tracks += 1 except Track.DoesNotExist: pass diff --git a/tests/base/test_scanner.py b/tests/base/test_scanner.py index b5fb212..4f837f3 100644 --- a/tests/base/test_scanner.py +++ b/tests/base/test_scanner.py @@ -1,13 +1,14 @@ # This file is part of Supysonic. # Supysonic is a Python implementation of the Subsonic server API. # -# Copyright (C) 2017-2022 Alban 'spl0k' Féron +# Copyright (C) 2017-2023 Alban 'spl0k' Féron # # Distributed under terms of the GNU AGPLv3 license. import mutagen import os import os.path +import shutil import tempfile import unittest @@ -164,5 +165,70 @@ class ScannerTestCase(unittest.TestCase): self.assertEqual(stats.deleted.tracks, 0) +class ScannerDeletionsTestCase(unittest.TestCase): + def setUp(self): + self.__dir = tempfile.mkdtemp() + db.init_database("sqlite:") + FolderManager.add("folder", self.__dir) + + # Create folder hierarchy + self._firstsubdir = tempfile.mkdtemp(dir=self.__dir) + subdir = self._firstsubdir + for _ in range(4): + subdir = tempfile.mkdtemp(dir=subdir) + + # Put a file in the deepest folder + self._trackpath = os.path.join(subdir, "silence.mp3") + shutil.copyfile("tests/assets/folder/silence.mp3", self._trackpath) + + self._scan() + + # Create annotation data + track = db.Track.get() + firstdir = db.Folder.get(path=self._firstsubdir) + user = db.User.create( + name="user", password="password", salt="salt", last_play=track + ) + db.StarredFolder.create(user=user, starred=track.folder_id) + db.StarredFolder.create(user=user, starred=firstdir) + db.StarredArtist.create(user=user, starred=track.artist_id) + db.StarredAlbum.create(user=user, starred=track.album_id) + db.StarredTrack.create(user=user, starred=track) + db.RatingFolder.create(user=user, rated=track.folder_id, rating=2) + db.RatingFolder.create(user=user, rated=firstdir, rating=2) + db.RatingTrack.create(user=user, rated=track, rating=2) + + def tearDown(self): + db.release_database() + shutil.rmtree(self.__dir) + + def _scan(self): + scanner = Scanner() + scanner.queue_folder("folder") + scanner.run() + + return scanner.stats() + + def _check_assertions(self, stats): + self.assertEqual(stats.deleted.artists, 1) + self.assertEqual(stats.deleted.albums, 1) + self.assertEqual(stats.deleted.tracks, 1) + self.assertEqual(db.Track.select().count(), 0) + self.assertEqual(db.Album.select().count(), 0) + self.assertEqual(db.Artist.select().count(), 0) + self.assertEqual(db.User.select().count(), 1) + self.assertEqual(db.Folder.select().count(), 1) + + def test_parent_folder(self): + shutil.rmtree(self._firstsubdir) + stats = self._scan() + self._check_assertions(stats) + + def test_track(self): + os.remove(self._trackpath) + stats = self._scan() + self._check_assertions(stats) + + if __name__ == "__main__": unittest.main() diff --git a/tests/issue101.py b/tests/issue101.py deleted file mode 100644 index 6c41525..0000000 --- a/tests/issue101.py +++ /dev/null @@ -1,49 +0,0 @@ -# This file is part of Supysonic. -# Supysonic is a Python implementation of the Subsonic server API. -# -# Copyright (C) 2018-2022 Alban 'spl0k' Féron -# -# Distributed under terms of the GNU AGPLv3 license. - -import os.path -import shutil -import tempfile -import unittest - -from supysonic.db import init_database, release_database -from supysonic.managers.folder import FolderManager -from supysonic.scanner import Scanner - - -class Issue101TestCase(unittest.TestCase): - def setUp(self): - self.__dir = tempfile.mkdtemp() - init_database("sqlite:") - FolderManager.add("folder", self.__dir) - - def tearDown(self): - release_database() - shutil.rmtree(self.__dir) - - def test_issue(self): - firstsubdir = tempfile.mkdtemp(dir=self.__dir) - subdir = firstsubdir - for _ in range(4): - subdir = tempfile.mkdtemp(dir=subdir) - shutil.copyfile( - "tests/assets/folder/silence.mp3", os.path.join(subdir, "silence.mp3") - ) - - scanner = Scanner() - scanner.queue_folder("folder") - scanner.run() - - shutil.rmtree(firstsubdir) - - scanner = Scanner() - scanner.queue_folder("folder") - scanner.run() - - -if __name__ == "__main__": - unittest.main()