feat(api): configure scan directory exclusions at library level

note that the existing values from configuration will not be migrated

komga/src/flyway/resources/db/migration/sqlite/V20230921112923__library_directory_exclusions.sql

@@ -0,0 +1,17 @@
+CREATE TABLE LIBRARY_EXCLUSIONS
+(
+    LIBRARY_ID varchar NOT NULL,
+    EXCLUSION  varchar NOT NULL,
+    PRIMARY KEY (LIBRARY_ID, EXCLUSION),
+    FOREIGN KEY (LIBRARY_ID) REFERENCES LIBRARY (ID)
+);
+
+CREATE INDEX idx__library_exclusions__library_id on LIBRARY_EXCLUSIONS (LIBRARY_ID);
+
+INSERT INTO LIBRARY_EXCLUSIONS
+WITH cte_exclusions(exclude) AS (VALUES ('#recycle'),
+                                        ('@eaDir'),
+                                        ('@Recycle'))
+SELECT LIBRARY.ID, cte_exclusions.exclude
+FROM LIBRARY
+         cross join cte_exclusions;

komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt

@@ -26,6 +26,7 @@ data class Library(
   val scanCbx: Boolean = true,
   val scanPdf: Boolean = true,
   val scanEpub: Boolean = true,
+  val scanDirectoryExclusions: Set<String> = emptySet(),
   val repairExtensions: Boolean = false,
   val convertToCbz: Boolean = false,
   val emptyTrashAfterScan: Boolean = false,

komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt

@@ -6,7 +6,6 @@ import org.gotson.komga.domain.model.DirectoryNotFoundException
 import org.gotson.komga.domain.model.ScanResult
 import org.gotson.komga.domain.model.Series
 import org.gotson.komga.domain.model.Sidecar
-import org.gotson.komga.infrastructure.configuration.KomgaProperties
 import org.gotson.komga.infrastructure.sidecar.SidecarBookConsumer
 import org.gotson.komga.infrastructure.sidecar.SidecarSeriesConsumer
 import org.springframework.stereotype.Service
@@ -34,7 +33,6 @@ private val logger = KotlinLogging.logger {}
 
 @Service
 class FileSystemScanner(
-  private val komgaProperties: KomgaProperties,
   private val sidecarBookConsumers: List<SidecarBookConsumer>,
   private val sidecarSeriesConsumers: List<SidecarSeriesConsumer>,
 ) {
@@ -55,6 +53,7 @@ class FileSystemScanner(
     scanCbx: Boolean = true,
     scanPdf: Boolean = true,
     scanEpub: Boolean = true,
+    directoryExclusions: Set<String> = emptySet(),
   ): ScanResult {
     val scanForExtensions = buildList {
       if (scanCbx) addAll(listOf("cbz", "zip", "cbr", "rar"))
@@ -63,7 +62,7 @@ class FileSystemScanner(
     }
     logger.info { "Scanning folder: $root" }
     logger.info { "Scan for extensions: $scanForExtensions" }
-    logger.info { "Excluded patterns: ${komgaProperties.librariesScanDirectoryExclusions}" }
+    logger.info { "Excluded directory patterns: $directoryExclusions" }
     logger.info { "Force directory modified time: $forceDirectoryModifiedTime" }
 
     if (!(Files.isDirectory(root) && Files.isReadable(root)))
@@ -88,7 +87,7 @@ class FileSystemScanner(
           override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
             logger.trace { "preVisit: $dir (regularFile:${attrs.isRegularFile}, directory:${attrs.isDirectory}, symbolicLink:${attrs.isSymbolicLink}, other:${attrs.isOther})" }
             if (dir.name.startsWith(".") ||
-              komgaProperties.librariesScanDirectoryExclusions.any { exclude ->
+              directoryExclusions.any { exclude ->
                 dir.pathString.contains(exclude, true)
               }
             ) return FileVisitResult.SKIP_SUBTREE

komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt

@@ -73,6 +73,7 @@ class LibraryContentLifecycle(
           library.scanCbx,
           library.scanPdf,
           library.scanEpub,
+          library.scanDirectoryExclusions,
         )
       } catch (e: DirectoryNotFoundException) {
         library.copy(unavailableDate = LocalDateTime.now()).let {

komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryLifecycle.kt

@@ -77,6 +77,7 @@ class LibraryLifecycle(
     if (existing.scanPdf != updated.scanPdf) return true
     if (existing.scanEpub != updated.scanEpub) return true
     if (existing.scanForceModifiedTime != updated.scanForceModifiedTime) return true
+    if (existing.scanDirectoryExclusions != updated.scanDirectoryExclusions) return true
     return false
   }
 

komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt

@@ -31,6 +31,7 @@ class KomgaProperties {
   @Deprecated("Moved to library options since 1.5.0")
   var librariesScanStartup: Boolean = false
 
+  @Deprecated("Moved to library options since 1.5.0")
   var librariesScanDirectoryExclusions: List<String> = emptyList()
 
   var deleteEmptyReadLists: Boolean = true

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt

@@ -5,6 +5,8 @@ import org.gotson.komga.domain.persistence.LibraryRepository
 import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.LibraryRecord
 import org.jooq.DSLContext
+import org.jooq.Record
+import org.jooq.ResultQuery
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.net.URL
@@ -18,39 +20,51 @@ class LibraryDao(
 
   private val l = Tables.LIBRARY
   private val ul = Tables.USER_LIBRARY_SHARING
+  private val le = Tables.LIBRARY_EXCLUSIONS
 
   override fun findByIdOrNull(libraryId: String): Library? =
     findOne(libraryId)
-      ?.toDomain()
+      .fetchAndMap()
+      .firstOrNull()
 
   override fun findById(libraryId: String): Library =
-    findOne(libraryId)!!
-      .toDomain()
+    findOne(libraryId)
+      .fetchAndMap()
+      .first()
 
   private fun findOne(libraryId: String) =
-    dsl.selectFrom(l)
+    selectBase()
       .where(l.ID.eq(libraryId))
-      .fetchOneInto(l)
-
   override fun findAll(): Collection<Library> =
-    dsl.selectFrom(l)
-      .fetchInto(l)
-      .map { it.toDomain() }
+    selectBase()
+      .fetchAndMap()
 
   override fun findAllByIds(libraryIds: Collection<String>): Collection<Library> =
-    dsl.selectFrom(l)
+    selectBase()
       .where(l.ID.`in`(libraryIds))
-      .fetchInto(l)
-      .map { it.toDomain() }
+      .fetchAndMap()
+
+  private fun selectBase() =
+    dsl.select()
+      .from(l)
+      .leftJoin(le).onKey()
+
+  private fun ResultQuery<Record>.fetchAndMap(): Collection<Library> =
+    this.fetchGroups({ it.into(l) }, { it.into(le) })
+      .map { (lr, ler) ->
+        lr.toDomain(ler.mapNotNull { it.exclusion }.toSet())
+      }
 
   @Transactional
   override fun delete(libraryId: String) {
+    dsl.deleteFrom(le).where(le.LIBRARY_ID.eq(libraryId)).execute()
     dsl.deleteFrom(ul).where(ul.LIBRARY_ID.eq(libraryId)).execute()
     dsl.deleteFrom(l).where(l.ID.eq(libraryId)).execute()
   }
 
   @Transactional
   override fun deleteAll() {
+    dsl.deleteFrom(le).execute()
     dsl.deleteFrom(ul).execute()
     dsl.deleteFrom(l).execute()
   }
@@ -87,6 +101,8 @@ class LibraryDao(
       .set(l.ONESHOTS_DIRECTORY, library.oneshotsDirectory)
       .set(l.UNAVAILABLE_DATE, library.unavailableDate)
       .execute()
+
+    insertDirectoryExclusions(library)
   }
 
   @Transactional
@@ -122,11 +138,32 @@ class LibraryDao(
       .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
       .where(l.ID.eq(library.id))
       .execute()
+
+    dsl.deleteFrom(le).where(le.LIBRARY_ID.eq(library.id)).execute()
+    insertDirectoryExclusions(library)
   }
 
   override fun count(): Long = dsl.fetchCount(l).toLong()
+  fun findDirectoryExclusions(libraryId: String): Set<String> =
+    dsl.select(le.EXCLUSION)
+      .from(le)
+      .where(le.LIBRARY_ID.eq(libraryId))
+      .fetchSet(le.EXCLUSION)
 
-  private fun LibraryRecord.toDomain() =
+  private fun insertDirectoryExclusions(library: Library) {
+    if (library.scanDirectoryExclusions.isNotEmpty()) {
+      dsl.batch(
+        dsl.insertInto(le, le.LIBRARY_ID, le.EXCLUSION)
+          .values(null as String?, null),
+      ).also { step ->
+        library.scanDirectoryExclusions.forEach {
+          step.bind(library.id, it)
+        }
+      }.execute()
+    }
+  }
+
+  private fun LibraryRecord.toDomain(directoryExclusions: Set<String>) =
     Library(
       name = name,
       root = URL(root),
@@ -146,6 +183,7 @@ class LibraryDao(
       scanEpub = scanEpub,
       scanOnStartup = scanStartup,
       scanInterval = Library.ScanInterval.valueOf(scanInterval),
+      scanDirectoryExclusions = directoryExclusions,
       repairExtensions = repairExtensions,
       convertToCbz = convertToCbz,
       emptyTrashAfterScan = emptyTrashAfterScan,

komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt

@@ -98,6 +98,7 @@ class LibraryController(
           scanCbx = library.scanCbx,
           scanPdf = library.scanPdf,
           scanEpub = library.scanEpub,
+          scanDirectoryExclusions = library.scanDirectoryExclusions,
           repairExtensions = library.repairExtensions,
           convertToCbz = library.convertToCbz,
           emptyTrashAfterScan = library.emptyTrashAfterScan,
@@ -165,6 +166,7 @@ class LibraryController(
           scanCbx = scanCbx ?: existing.scanCbx,
           scanPdf = scanPdf ?: existing.scanPdf,
           scanEpub = scanEpub ?: existing.scanEpub,
+          scanDirectoryExclusions = if (isSet("scanDirectoryExclusions")) scanDirectoryExclusions ?: emptySet() else existing.scanDirectoryExclusions,
           repairExtensions = repairExtensions ?: existing.repairExtensions,
           convertToCbz = convertToCbz ?: existing.convertToCbz,
           emptyTrashAfterScan = emptyTrashAfterScan ?: existing.emptyTrashAfterScan,

komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryCreationDto.kt

@@ -21,6 +21,7 @@ data class LibraryCreationDto(
   val scanCbx: Boolean = true,
   val scanPdf: Boolean = true,
   val scanEpub: Boolean = true,
+  val scanDirectoryExclusions: Set<String> = emptySet(),
   val repairExtensions: Boolean = false,
   val convertToCbz: Boolean = false,
   val emptyTrashAfterScan: Boolean = false,

komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryDto.kt

@@ -23,6 +23,7 @@ data class LibraryDto(
   val scanCbx: Boolean,
   val scanPdf: Boolean,
   val scanEpub: Boolean,
+  val scanDirectoryExclusions: Set<String>,
   val repairExtensions: Boolean,
   val convertToCbz: Boolean,
   val emptyTrashAfterScan: Boolean,
@@ -54,6 +55,7 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto(
   scanCbx = scanCbx,
   scanPdf = scanPdf,
   scanEpub = scanEpub,
+  scanDirectoryExclusions = scanDirectoryExclusions,
   repairExtensions = repairExtensions,
   convertToCbz = convertToCbz,
   emptyTrashAfterScan = emptyTrashAfterScan,

komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryUpdateDto.kt

@@ -30,6 +30,10 @@ class LibraryUpdateDto {
   val scanCbx: Boolean? = null
   val scanPdf: Boolean? = null
   val scanEpub: Boolean? = null
+  var scanDirectoryExclusions: Set<String>?
+    by Delegates.observable(null) { prop, _, _ ->
+      isSet[prop.name] = true
+    }
 
   val repairExtensions: Boolean? = null
   val convertToCbz: Boolean? = null

komga/src/main/resources/application.yml

@@ -14,10 +14,6 @@ logging:
 
 komga:
   libraries-scan-cron: "0 0 */8 * * ?"
-  libraries-scan-directory-exclusions:
-    - "#recycle"  # Synology
-    - "@eaDir"    # Synology
-    - "@Recycle"  # Qnap
   database:
     file: \${komga.config-dir}/database.sqlite
   lucene:

komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt

@@ -6,7 +6,6 @@ import org.apache.commons.io.FilenameUtils
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.catchThrowable
 import org.gotson.komga.domain.model.DirectoryNotFoundException
-import org.gotson.komga.infrastructure.configuration.KomgaProperties
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.Arguments
@@ -16,12 +15,7 @@ import java.nio.file.Path
 import java.util.stream.Stream
 
 class FileSystemScannerTest {
-
-  private val komgaProperties = KomgaProperties().apply {
-    librariesScanDirectoryExclusions = listOf("#recycle")
-  }
-
-  private val scanner = FileSystemScanner(komgaProperties, emptyList(), emptyList())
+  private val scanner = FileSystemScanner(emptyList(), emptyList())
 
   @Test
   fun `given unavailable root directory when scanning then throw exception`() {
@@ -294,7 +288,7 @@ class FileSystemScannerTest {
       makeSubDir(recycle, "subtrash", listOf("trash2.cbz"))
 
       // when
-      val scan = scanner.scanRootFolder(root).series
+      val scan = scanner.scanRootFolder(root, directoryExclusions = setOf("#recycle")).series
 
       // then
       assertThat(scan).hasSize(2)

komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt

@@ -74,6 +74,7 @@ class LibraryDaoTest(
         scanPdf = false,
         scanInterval = Library.ScanInterval.DAILY,
         scanOnStartup = true,
+        scanDirectoryExclusions = setOf("a", "b"),
       )
     }
 
@@ -111,6 +112,7 @@ class LibraryDaoTest(
     assertThat(modified.scanPdf).isEqualTo(updated.scanPdf)
     assertThat(modified.scanInterval).isEqualTo(updated.scanInterval)
     assertThat(modified.scanOnStartup).isEqualTo(updated.scanOnStartup)
+    assertThat(modified.scanDirectoryExclusions).containsExactlyInAnyOrderElementsOf(updated.scanDirectoryExclusions)
   }
 
   @Test

komga/src/test/kotlin/org/gotson/komga/interfaces/api/rest/LibraryControllerTest.kt

@@ -5,10 +5,12 @@ import org.gotson.komga.domain.model.ROLE_USER
 import org.gotson.komga.domain.model.makeLibrary
 import org.gotson.komga.domain.persistence.LibraryRepository
 import org.hamcrest.Matchers
-import org.junit.jupiter.api.AfterAll
-import org.junit.jupiter.api.BeforeAll
+import org.hamcrest.Matchers.hasItems
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc
 import org.springframework.boot.test.context.SpringBootTest
@@ -17,7 +19,9 @@ import org.springframework.security.test.context.support.WithAnonymousUser
 import org.springframework.security.test.context.support.WithMockUser
 import org.springframework.test.web.servlet.MockMvc
 import org.springframework.test.web.servlet.get
+import org.springframework.test.web.servlet.patch
 import org.springframework.test.web.servlet.post
+import java.nio.file.Path
 
 @SpringBootTest
 @AutoConfigureMockMvc(printOnlyOnFailure = false)
@@ -30,12 +34,12 @@ class LibraryControllerTest(
 
   private val library = makeLibrary(path = "file:/library1", id = "1")
 
-  @BeforeAll
+  @BeforeEach
   fun `setup library`() {
     libraryRepository.insert(library)
   }
 
-  @AfterAll
+  @AfterEach
   fun `teardown library`() {
     libraryRepository.deleteAll()
   }
@@ -132,4 +136,55 @@ class LibraryControllerTest(
         }
     }
   }
+
+  @Nested
+  inner class DirectoryExclusions {
+    @Test
+    @WithMockCustomUser
+    fun `given library with exclusions when getting libraries then exclusions are present`() {
+      libraryRepository.update(library.copy(scanDirectoryExclusions = setOf("test", "value")))
+
+      mockMvc.get(route)
+        .andExpect {
+          status { isOk() }
+          jsonPath("$[0].scanDirectoryExclusions.length()") { value(2) }
+          jsonPath("$[0].scanDirectoryExclusions") { hasItems("test", "value") }
+        }
+
+      mockMvc.get("$route/${library.id}")
+        .andExpect {
+          status { isOk() }
+          jsonPath("$.scanDirectoryExclusions.length()") { value(2) }
+          jsonPath("$.scanDirectoryExclusions") { hasItems("test", "value") }
+        }
+    }
+
+    @Test
+    @WithMockCustomUser(roles = [ROLE_ADMIN])
+    fun `given library with exclusions when updating library then exclusions are updated`(@TempDir tmp: Path) {
+      libraryRepository.update(library.copy(root = tmp.toUri().toURL(), scanDirectoryExclusions = setOf("test", "value")))
+
+      // language=JSON
+      val jsonString = """
+        {
+          "scanDirectoryExclusions": ["updated"]
+        }
+      """.trimIndent()
+
+      mockMvc.patch("$route/${library.id}") {
+        contentType = MediaType.APPLICATION_JSON
+        content = jsonString
+      }
+        .andExpect {
+          status { isNoContent() }
+        }
+
+      mockMvc.get("$route/${library.id}")
+        .andExpect {
+          status { isOk() }
+          jsonPath("$.scanDirectoryExclusions.length()") { value(1) }
+          jsonPath("$.scanDirectoryExclusions") { hasItems("updated") }
+        }
+    }
+  }
 }