Fix failing deletions from the scanner

supysonic/db.py

@@ -167,18 +167,52 @@ class Folder(PathMixin, _Model):
 
     @classmethod
     def prune(cls):
-        query = cls.delete().where(
+        alias = cls.alias()
+        query = cls.select(cls.id).where(
             ~cls.root,
-            cls.id.not_in(Track.select(Track.folder)),
+            Track.select(fn.count("*")).where(Track.folder == cls.id) == 0,
-            cls.id.not_in(cls.select(cls.parent)),
+            alias.select(fn.count("*")).where(alias.parent == cls.id) == 0,
         )
         total = 0
         while True:
-            count = query.execute()
+            clone = query.clone()  # peewee caches the results, clone to force a refetch
-            total += count
+            for f in clone:
-            if not count:
+                f.delete_instance(recursive=True)
+                total += 1
+            if not len(clone):
                 return total
 
+    def delete_hierarchy(self):
+        if self.root:
+            cond = Track.root_folder == self
+        else:
+            cond = Track.path.startswith(self.path)
+
+        return self.__delete_hierarchy(cond)
+
+    def __delete_hierarchy(self, cond):
+        users = User.select(User.id).join(Track).where(cond)
+        User.update(last_play=None).where(User.id.in_(users)).execute()
+
+        tracks = Track.select(Track.id).where(cond)
+        RatingTrack.delete().where(RatingTrack.rated.in_(tracks)).execute()
+        StarredTrack.delete().where(StarredTrack.starred.in_(tracks)).execute()
+
+        path_cond = Folder.path.startswith(self.path)
+        folders = Folder.select(Folder.id).where(path_cond)
+        RatingFolder.delete().where(RatingFolder.rated.in_(folders)).execute()
+        StarredFolder.delete().where(StarredFolder.starred.in_(folders)).execute()
+
+        deleted_tracks = Track.delete().where(cond).execute()
+
+        query = Folder.delete().where(path_cond)
+        if isinstance(db.obj, MySQLDatabase):
+            # MySQL can't propery resolve deletion order when it has several to handle
+            query = query.order_by(Folder.path.desc())
+        query.execute()
+
+        return deleted_tracks
+
 
 class Artist(_Model):
     id = PrimaryKeyField()

supysonic/managers/folder.py

@@ -1,27 +1,15 @@
 # This file is part of Supysonic.
 # Supysonic is a Python implementation of the Subsonic server API.
 #
-# Copyright (C) 2013-2022 Alban 'spl0k' Féron
+# Copyright (C) 2013-2023 Alban 'spl0k' Féron
 #
 # Distributed under terms of the GNU AGPLv3 license.
 
 import os.path
 
-from peewee import IntegrityError
-
 from ..daemon.client import DaemonClient
 from ..daemon.exceptions import DaemonUnavailableError
-from ..db import (
+from ..db import Folder, Artist, Album
-    Folder,
-    Track,
-    Artist,
-    Album,
-    User,
-    RatingFolder,
-    RatingTrack,
-    StarredFolder,
-    StarredTrack,
-)
 
 
 class FolderManager:
@@ -79,29 +67,9 @@ class FolderManager:
         except DaemonUnavailableError:
             pass
 
-        root_cond = Track.root_folder == folder
+        folder.delete_hierarchy()
-        users = User.select(User.id).join(Track).where(root_cond)
-        User.update(last_play=None).where(User.id.in_(users)).execute()
-
-        tracks = Track.select(Track.id).where(root_cond)
-        RatingTrack.delete().where(RatingTrack.rated.in_(tracks)).execute()
-        StarredTrack.delete().where(StarredTrack.starred.in_(tracks)).execute()
-
-        path_cond = Folder.path.startswith(folder.path)
-        folders = Folder.select(Folder.id).where(path_cond)
-        RatingFolder.delete().where(RatingFolder.rated.in_(folders)).execute()
-        StarredFolder.delete().where(StarredFolder.starred.in_(folders)).execute()
-
-        Track.delete().where(root_cond).execute()
         Album.prune()
         Artist.prune()
-        query = Folder.delete().where(path_cond)
-        try:
-            query.execute()
-        except IntegrityError:
-            # Integrity error most likely due to MySQL poor handling of delete order
-            query = query.order_by(Folder.path.desc())
-            query.execute()
 
     @staticmethod
     def delete_by_name(name):

supysonic/scanner.py

@@ -141,8 +141,19 @@ class Scanner(Thread):
 
                     self.__report_progress(folder.name, scanned)
 
+        # Remove deleted/moved folders
+        folders = [folder]
+        while not self.__stopped.is_set() and folders:
+            f = folders.pop()
+
+            if not f.root and not os.path.isdir(f.path):
+                self.__stats.deleted.tracks += f.delete_hierarchy()
+                continue
+
+            folders += f.children[:]
+
         # Remove files that have been deleted
-        # Could be more efficient if done above
+        # Could be more efficient if done when walking on the files
         if not self.__stopped.is_set():
             for track in Track.select().where(Track.root_folder == folder):
                 if not os.path.exists(track.path) or not self.__check_extension(
@@ -150,15 +161,10 @@ class Scanner(Thread):
                 ):
                     self.remove_file(track.path)
 
-        # Remove deleted/moved folders and update cover art info
+        # Update cover art info
         folders = [folder]
         while not self.__stopped.is_set() and folders:
             f = folders.pop()
-
-            if not f.root and not os.path.isdir(f.path):
-                f.delete_instance(recursive=True)
-                continue
-
             self.find_cover(f.path)
             folders += f.children[:]
 
@@ -173,8 +179,8 @@ class Scanner(Thread):
         if self.__stopped.is_set():
             return
 
-        self.__stats.deleted.albums = Album.prune()
+        self.__stats.deleted.albums += Album.prune()
-        self.__stats.deleted.artists = Artist.prune()
+        self.__stats.deleted.artists += Artist.prune()
         Folder.prune()
 
     def __check_extension(self, path):
@@ -272,7 +278,7 @@ class Scanner(Thread):
             raise TypeError("Expecting string, got " + str(type(path)))
 
         try:
-            Track.get(path=path).delete_instance()
+            Track.get(path=path).delete_instance(recursive=True)
             self.__stats.deleted.tracks += 1
         except Track.DoesNotExist:
             pass

tests/base/test_scanner.py

@@ -1,13 +1,14 @@
 # This file is part of Supysonic.
 # Supysonic is a Python implementation of the Subsonic server API.
 #
-# Copyright (C) 2017-2022 Alban 'spl0k' Féron
+# Copyright (C) 2017-2023 Alban 'spl0k' Féron
 #
 # Distributed under terms of the GNU AGPLv3 license.
 
 import mutagen
 import os
 import os.path
+import shutil
 import tempfile
 import unittest
 
@@ -164,5 +165,70 @@ class ScannerTestCase(unittest.TestCase):
         self.assertEqual(stats.deleted.tracks, 0)
 
 
+class ScannerDeletionsTestCase(unittest.TestCase):
+    def setUp(self):
+        self.__dir = tempfile.mkdtemp()
+        db.init_database("sqlite:")
+        FolderManager.add("folder", self.__dir)
+
+        # Create folder hierarchy
+        self._firstsubdir = tempfile.mkdtemp(dir=self.__dir)
+        subdir = self._firstsubdir
+        for _ in range(4):
+            subdir = tempfile.mkdtemp(dir=subdir)
+
+        # Put a file in the deepest folder
+        self._trackpath = os.path.join(subdir, "silence.mp3")
+        shutil.copyfile("tests/assets/folder/silence.mp3", self._trackpath)
+
+        self._scan()
+
+        # Create annotation data
+        track = db.Track.get()
+        firstdir = db.Folder.get(path=self._firstsubdir)
+        user = db.User.create(
+            name="user", password="password", salt="salt", last_play=track
+        )
+        db.StarredFolder.create(user=user, starred=track.folder_id)
+        db.StarredFolder.create(user=user, starred=firstdir)
+        db.StarredArtist.create(user=user, starred=track.artist_id)
+        db.StarredAlbum.create(user=user, starred=track.album_id)
+        db.StarredTrack.create(user=user, starred=track)
+        db.RatingFolder.create(user=user, rated=track.folder_id, rating=2)
+        db.RatingFolder.create(user=user, rated=firstdir, rating=2)
+        db.RatingTrack.create(user=user, rated=track, rating=2)
+
+    def tearDown(self):
+        db.release_database()
+        shutil.rmtree(self.__dir)
+
+    def _scan(self):
+        scanner = Scanner()
+        scanner.queue_folder("folder")
+        scanner.run()
+
+        return scanner.stats()
+
+    def _check_assertions(self, stats):
+        self.assertEqual(stats.deleted.artists, 1)
+        self.assertEqual(stats.deleted.albums, 1)
+        self.assertEqual(stats.deleted.tracks, 1)
+        self.assertEqual(db.Track.select().count(), 0)
+        self.assertEqual(db.Album.select().count(), 0)
+        self.assertEqual(db.Artist.select().count(), 0)
+        self.assertEqual(db.User.select().count(), 1)
+        self.assertEqual(db.Folder.select().count(), 1)
+
+    def test_parent_folder(self):
+        shutil.rmtree(self._firstsubdir)
+        stats = self._scan()
+        self._check_assertions(stats)
+
+    def test_track(self):
+        os.remove(self._trackpath)
+        stats = self._scan()
+        self._check_assertions(stats)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/issue101.py

@@ -1,49 +0,0 @@
-# This file is part of Supysonic.
-# Supysonic is a Python implementation of the Subsonic server API.
-#
-# Copyright (C) 2018-2022 Alban 'spl0k' Féron
-#
-# Distributed under terms of the GNU AGPLv3 license.
-
-import os.path
-import shutil
-import tempfile
-import unittest
-
-from supysonic.db import init_database, release_database
-from supysonic.managers.folder import FolderManager
-from supysonic.scanner import Scanner
-
-
-class Issue101TestCase(unittest.TestCase):
-    def setUp(self):
-        self.__dir = tempfile.mkdtemp()
-        init_database("sqlite:")
-        FolderManager.add("folder", self.__dir)
-
-    def tearDown(self):
-        release_database()
-        shutil.rmtree(self.__dir)
-
-    def test_issue(self):
-        firstsubdir = tempfile.mkdtemp(dir=self.__dir)
-        subdir = firstsubdir
-        for _ in range(4):
-            subdir = tempfile.mkdtemp(dir=subdir)
-        shutil.copyfile(
-            "tests/assets/folder/silence.mp3", os.path.join(subdir, "silence.mp3")
-        )
-
-        scanner = Scanner()
-        scanner.queue_folder("folder")
-        scanner.run()
-
-        shutil.rmtree(firstsubdir)
-
-        scanner = Scanner()
-        scanner.queue_folder("folder")
-        scanner.run()
-
-
-if __name__ == "__main__":
-    unittest.main()