diff --git a/graphql/schema/types/file.graphql b/graphql/schema/types/file.graphql index 835479fad..37fb5539f 100644 --- a/graphql/schema/types/file.graphql +++ b/graphql/schema/types/file.graphql @@ -6,11 +6,14 @@ type Fingerprint { type Folder { id: ID! path: String! + basename: String! parent_folder_id: ID @deprecated(reason: "Use parent_folder instead") zip_file_id: ID @deprecated(reason: "Use zip_file instead") parent_folder: Folder + "Returns all parent folders in order from immediate parent to top-level" + parent_folders: [Folder!]! zip_file: BasicFile mod_time: Time! diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index d9814ef34..6eda473b4 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -822,6 +822,7 @@ input FolderFilterType { NOT: FolderFilterType path: StringCriterionInput + basename: StringCriterionInput parent_folder: HierarchicalMultiCriterionInput zip_file: MultiCriterionInput diff --git a/internal/api/loaders/dataloaders.go b/internal/api/loaders/dataloaders.go index dac8ba6b8..2ba650962 100644 --- a/internal/api/loaders/dataloaders.go +++ b/internal/api/loaders/dataloaders.go @@ -11,6 +11,7 @@ //go:generate go run github.com/vektah/dataloaden GroupLoader int *github.com/stashapp/stash/pkg/models.Group //go:generate go run github.com/vektah/dataloaden FileLoader github.com/stashapp/stash/pkg/models.FileID github.com/stashapp/stash/pkg/models.File //go:generate go run github.com/vektah/dataloaden FolderLoader github.com/stashapp/stash/pkg/models.FolderID *github.com/stashapp/stash/pkg/models.Folder +//go:generate go run github.com/vektah/dataloaden FolderParentFolderIDsLoader github.com/stashapp/stash/pkg/models.FolderID []github.com/stashapp/stash/pkg/models.FolderID //go:generate go run github.com/vektah/dataloaden SceneFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID //go:generate go run github.com/vektah/dataloaden ImageFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID //go:generate go run github.com/vektah/dataloaden GalleryFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID @@ -65,12 +66,16 @@ type Loaders struct { StudioByID *StudioLoader StudioCustomFields *CustomFieldsLoader - TagByID *TagLoader - TagCustomFields *CustomFieldsLoader + TagByID *TagLoader + TagCustomFields *CustomFieldsLoader + GroupByID *GroupLoader GroupCustomFields *CustomFieldsLoader - FileByID *FileLoader - FolderByID *FolderLoader + + FileByID *FileLoader + + FolderByID *FolderLoader + FolderParentFolderIDs *FolderParentFolderIDsLoader } type Middleware struct { @@ -161,6 +166,11 @@ func (m Middleware) Middleware(next http.Handler) http.Handler { maxBatch: maxBatch, fetch: m.fetchFolders(ctx), }, + FolderParentFolderIDs: &FolderParentFolderIDsLoader{ + wait: wait, + maxBatch: maxBatch, + fetch: m.fetchFoldersParentFolderIDs(ctx), + }, SceneFiles: &SceneFileIDsLoader{ wait: wait, maxBatch: maxBatch, @@ -406,6 +416,17 @@ func (m Middleware) fetchFolders(ctx context.Context) func(keys []models.FolderI } } +func (m Middleware) fetchFoldersParentFolderIDs(ctx context.Context) func(keys []models.FolderID) ([][]models.FolderID, []error) { + return func(keys []models.FolderID) (ret [][]models.FolderID, errs []error) { + err := m.Repository.WithDB(ctx, func(ctx context.Context) error { + var err error + ret, err = m.Repository.Folder.GetManyParentFolderIDs(ctx, keys) + return err + }) + return ret, toErrorSlice(err) + } +} + func (m Middleware) fetchScenesFileIDs(ctx context.Context) func(keys []int) ([][]models.FileID, []error) { return func(keys []int) (ret [][]models.FileID, errs []error) { err := m.Repository.WithDB(ctx, func(ctx context.Context) error { diff --git a/internal/api/loaders/folderparentfolderidsloader_gen.go b/internal/api/loaders/folderparentfolderidsloader_gen.go new file mode 100644 index 000000000..c9eca3a3d --- /dev/null +++ b/internal/api/loaders/folderparentfolderidsloader_gen.go @@ -0,0 +1,225 @@ +// Code generated by github.com/vektah/dataloaden, DO NOT EDIT. + +package loaders + +import ( + "sync" + "time" + + "github.com/stashapp/stash/pkg/models" +) + +// FolderParentFolderIDsLoaderConfig captures the config to create a new FolderParentFolderIDsLoader +type FolderParentFolderIDsLoaderConfig struct { + // Fetch is a method that provides the data for the loader + Fetch func(keys []models.FolderID) ([][]models.FolderID, []error) + + // Wait is how long wait before sending a batch + Wait time.Duration + + // MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit + MaxBatch int +} + +// NewFolderParentFolderIDsLoader creates a new FolderParentFolderIDsLoader given a fetch, wait, and maxBatch +func NewFolderParentFolderIDsLoader(config FolderParentFolderIDsLoaderConfig) *FolderParentFolderIDsLoader { + return &FolderParentFolderIDsLoader{ + fetch: config.Fetch, + wait: config.Wait, + maxBatch: config.MaxBatch, + } +} + +// FolderParentFolderIDsLoader batches and caches requests +type FolderParentFolderIDsLoader struct { + // this method provides the data for the loader + fetch func(keys []models.FolderID) ([][]models.FolderID, []error) + + // how long to done before sending a batch + wait time.Duration + + // this will limit the maximum number of keys to send in one batch, 0 = no limit + maxBatch int + + // INTERNAL + + // lazily created cache + cache map[models.FolderID][]models.FolderID + + // the current batch. keys will continue to be collected until timeout is hit, + // then everything will be sent to the fetch method and out to the listeners + batch *folderParentFolderIDsLoaderBatch + + // mutex to prevent races + mu sync.Mutex +} + +type folderParentFolderIDsLoaderBatch struct { + keys []models.FolderID + data [][]models.FolderID + error []error + closing bool + done chan struct{} +} + +// Load a FolderID by key, batching and caching will be applied automatically +func (l *FolderParentFolderIDsLoader) Load(key models.FolderID) ([]models.FolderID, error) { + return l.LoadThunk(key)() +} + +// LoadThunk returns a function that when called will block waiting for a FolderID. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *FolderParentFolderIDsLoader) LoadThunk(key models.FolderID) func() ([]models.FolderID, error) { + l.mu.Lock() + if it, ok := l.cache[key]; ok { + l.mu.Unlock() + return func() ([]models.FolderID, error) { + return it, nil + } + } + if l.batch == nil { + l.batch = &folderParentFolderIDsLoaderBatch{done: make(chan struct{})} + } + batch := l.batch + pos := batch.keyIndex(l, key) + l.mu.Unlock() + + return func() ([]models.FolderID, error) { + <-batch.done + + var data []models.FolderID + if pos < len(batch.data) { + data = batch.data[pos] + } + + var err error + // its convenient to be able to return a single error for everything + if len(batch.error) == 1 { + err = batch.error[0] + } else if batch.error != nil { + err = batch.error[pos] + } + + if err == nil { + l.mu.Lock() + l.unsafeSet(key, data) + l.mu.Unlock() + } + + return data, err + } +} + +// LoadAll fetches many keys at once. It will be broken into appropriate sized +// sub batches depending on how the loader is configured +func (l *FolderParentFolderIDsLoader) LoadAll(keys []models.FolderID) ([][]models.FolderID, []error) { + results := make([]func() ([]models.FolderID, error), len(keys)) + + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + + folderIDs := make([][]models.FolderID, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + folderIDs[i], errors[i] = thunk() + } + return folderIDs, errors +} + +// LoadAllThunk returns a function that when called will block waiting for a FolderIDs. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *FolderParentFolderIDsLoader) LoadAllThunk(keys []models.FolderID) func() ([][]models.FolderID, []error) { + results := make([]func() ([]models.FolderID, error), len(keys)) + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + return func() ([][]models.FolderID, []error) { + folderIDs := make([][]models.FolderID, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + folderIDs[i], errors[i] = thunk() + } + return folderIDs, errors + } +} + +// Prime the cache with the provided key and value. If the key already exists, no change is made +// and false is returned. +// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).) +func (l *FolderParentFolderIDsLoader) Prime(key models.FolderID, value []models.FolderID) bool { + l.mu.Lock() + var found bool + if _, found = l.cache[key]; !found { + // make a copy when writing to the cache, its easy to pass a pointer in from a loop var + // and end up with the whole cache pointing to the same value. + cpy := make([]models.FolderID, len(value)) + copy(cpy, value) + l.unsafeSet(key, cpy) + } + l.mu.Unlock() + return !found +} + +// Clear the value at key from the cache, if it exists +func (l *FolderParentFolderIDsLoader) Clear(key models.FolderID) { + l.mu.Lock() + delete(l.cache, key) + l.mu.Unlock() +} + +func (l *FolderParentFolderIDsLoader) unsafeSet(key models.FolderID, value []models.FolderID) { + if l.cache == nil { + l.cache = map[models.FolderID][]models.FolderID{} + } + l.cache[key] = value +} + +// keyIndex will return the location of the key in the batch, if its not found +// it will add the key to the batch +func (b *folderParentFolderIDsLoaderBatch) keyIndex(l *FolderParentFolderIDsLoader, key models.FolderID) int { + for i, existingKey := range b.keys { + if key == existingKey { + return i + } + } + + pos := len(b.keys) + b.keys = append(b.keys, key) + if pos == 0 { + go b.startTimer(l) + } + + if l.maxBatch != 0 && pos >= l.maxBatch-1 { + if !b.closing { + b.closing = true + l.batch = nil + go b.end(l) + } + } + + return pos +} + +func (b *folderParentFolderIDsLoaderBatch) startTimer(l *FolderParentFolderIDsLoader) { + time.Sleep(l.wait) + l.mu.Lock() + + // we must have hit a batch limit and are already finalizing this batch + if b.closing { + l.mu.Unlock() + return + } + + l.batch = nil + l.mu.Unlock() + + b.end(l) +} + +func (b *folderParentFolderIDsLoaderBatch) end(l *FolderParentFolderIDsLoader) { + b.data, b.error = l.fetch(b.keys) + close(b.done) +} diff --git a/internal/api/resolver_model_folder.go b/internal/api/resolver_model_folder.go index ee6bbfd05..c203a3f82 100644 --- a/internal/api/resolver_model_folder.go +++ b/internal/api/resolver_model_folder.go @@ -2,11 +2,16 @@ package api import ( "context" + "path/filepath" "github.com/stashapp/stash/internal/api/loaders" "github.com/stashapp/stash/pkg/models" ) +func (r *folderResolver) Basename(ctx context.Context, obj *models.Folder) (string, error) { + return filepath.Base(obj.Path), nil +} + func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) (*models.Folder, error) { if obj.ParentFolderID == nil { return nil, nil @@ -15,6 +20,17 @@ func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) ( return loaders.From(ctx).FolderByID.Load(*obj.ParentFolderID) } +func (r *folderResolver) ParentFolders(ctx context.Context, obj *models.Folder) ([]*models.Folder, error) { + ids, err := loaders.From(ctx).FolderParentFolderIDs.Load(obj.ID) + if err != nil { + return nil, err + } + + var errs []error + ret, errs := loaders.From(ctx).FolderByID.LoadAll(ids) + return ret, firstError(errs) +} + func (r *folderResolver) ZipFile(ctx context.Context, obj *models.Folder) (*BasicFile, error) { return zipFileResolver(ctx, obj.ZipFileID) } diff --git a/pkg/file/zip.go b/pkg/file/zip.go index 5afcd5329..6d00c7e35 100644 --- a/pkg/file/zip.go +++ b/pkg/file/zip.go @@ -99,7 +99,9 @@ func (f *zipFS) rel(name string) (string, error) { relName, err := filepath.Rel(f.zipPath, name) if err != nil { - return "", fmt.Errorf("internal error getting relative path: %w", err) + // if the path is not relative to the zip path, then it's not found in the zip file, + // so treat this as a file not found + return "", fs.ErrNotExist } // convert relName to use slash, since zip files do so regardless diff --git a/pkg/models/folder.go b/pkg/models/folder.go index ada9e17b7..e9e9a3971 100644 --- a/pkg/models/folder.go +++ b/pkg/models/folder.go @@ -18,10 +18,8 @@ type FolderQueryOptions struct { type FolderFilterType struct { OperatorFilter[FolderFilterType] - Path *StringCriterionInput `json:"path,omitempty"` - Basename *StringCriterionInput `json:"basename,omitempty"` - // Filter by parent directory path - Dir *StringCriterionInput `json:"dir,omitempty"` + Path *StringCriterionInput `json:"path,omitempty"` + Basename *StringCriterionInput `json:"basename,omitempty"` ParentFolder *HierarchicalMultiCriterionInput `json:"parent_folder,omitempty"` ZipFile *MultiCriterionInput `json:"zip_file,omitempty"` // Filter by modification time diff --git a/pkg/models/mocks/FolderReaderWriter.go b/pkg/models/mocks/FolderReaderWriter.go index 7bca013fe..5d4d95027 100644 --- a/pkg/models/mocks/FolderReaderWriter.go +++ b/pkg/models/mocks/FolderReaderWriter.go @@ -201,6 +201,29 @@ func (_m *FolderReaderWriter) FindMany(ctx context.Context, id []models.FolderID return r0, r1 } +// GetManyParentFolderIDs provides a mock function with given fields: ctx, folderIDs +func (_m *FolderReaderWriter) GetManyParentFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) { + ret := _m.Called(ctx, folderIDs) + + var r0 [][]models.FolderID + if rf, ok := ret.Get(0).(func(context.Context, []models.FolderID) [][]models.FolderID); ok { + r0 = rf(ctx, folderIDs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([][]models.FolderID) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []models.FolderID) error); ok { + r1 = rf(ctx, folderIDs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Query provides a mock function with given fields: ctx, options func (_m *FolderReaderWriter) Query(ctx context.Context, options models.FolderQueryOptions) (*models.FolderQueryResult, error) { ret := _m.Called(ctx, options) diff --git a/pkg/models/repository_folder.go b/pkg/models/repository_folder.go index 3d0fdb822..539d51cb9 100644 --- a/pkg/models/repository_folder.go +++ b/pkg/models/repository_folder.go @@ -15,6 +15,7 @@ type FolderFinder interface { FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error) FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error) + GetManyParentFolderIDs(ctx context.Context, folderIDs []FolderID) ([][]FolderID, error) } type FolderQueryer interface { diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 003c6eebc..f8c2cdef7 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -34,7 +34,7 @@ const ( cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE" ) -var appSchemaVersion uint = 83 +var appSchemaVersion uint = 84 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index f250f7861..73a065cff 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -20,6 +20,7 @@ const folderIDColumn = "folder_id" type folderRow struct { ID models.FolderID `db:"id" goqu:"skipinsert"` + Basename string `db:"basename"` Path string `db:"path"` ZipFileID null.Int `db:"zip_file_id"` ParentFolderID null.Int `db:"parent_folder_id"` @@ -30,6 +31,8 @@ type folderRow struct { func (r *folderRow) fromFolder(o models.Folder) { r.ID = o.ID + // derive basename from path + r.Basename = filepath.Base(o.Path) r.Path = o.Path r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID) r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID) @@ -322,6 +325,90 @@ func (qb *FolderStore) FindByParentFolderID(ctx context.Context, parentFolderID return ret, nil } +func (qb *FolderStore) GetManyParentFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) { + table := qb.table() + + // SQL recursive query to get all parent folder IDs for each folder ID + /* + WITH RECURSIVE parent_folders AS ( + SELECT id, parent_folder_id + FROM folders + WHERE id IN (folderIDs) + + UNION ALL + + SELECT f.id, f.parent_folder_id + FROM folders f + INNER JOIN parent_folders pf ON f.id = pf.parent_folder_id + ) + SELECT id, parent_folder_id FROM parent_folders; + */ + const parentFolders = "parent_folders" + const parentFolderID = "parent_folder_id" + const parentID = "parent_id" + const foldersAlias = "f" + + const parentFoldersAlias = "pf" + foldersAliasedI := table.As(foldersAlias) + parentFoldersI := goqu.T(parentFolders).As(parentFoldersAlias) + + q := dialect.From(parentFolders).Prepared(true). + WithRecursive(parentFolders, + dialect.From(table).Select(table.Col(idColumn), table.Col(parentFolderID).As(parentID)). + Where(table.Col(idColumn).In(folderIDs)). + Union( + dialect.From(foldersAliasedI).InnerJoin( + parentFoldersI, + goqu.On(foldersAliasedI.Col(idColumn).Eq(parentFoldersI.Col(parentID))), + ).Select(foldersAliasedI.Col(idColumn), foldersAliasedI.Col(parentFolderID).As(parentID)), + ), + ).Select(idColumn, parentID) + + type resultRow struct { + FolderID models.FolderID `db:"id"` + ParentFolderID null.Int `db:"parent_id"` + } + + folderMap := make(map[models.FolderID]models.FolderID) + + if err := queryFunc(ctx, q, false, func(r *sqlx.Rows) error { + var row resultRow + if err := r.StructScan(&row); err != nil { + return err + } + + if row.ParentFolderID.Valid { + folderMap[row.FolderID] = models.FolderID(row.ParentFolderID.Int64) + } else { + folderMap[row.FolderID] = 0 + } + + return nil + }); err != nil { + return nil, err + } + + ret := make([][]models.FolderID, len(folderIDs)) + + for i, folderID := range folderIDs { + var parents []models.FolderID + currentID := folderID + + for { + parentID, exists := folderMap[currentID] + if !exists || parentID == 0 { + break + } + parents = append(parents, parentID) + currentID = parentID + } + + ret[i] = parents + } + + return ret, nil +} + func (qb *FolderStore) allInPaths(q *goqu.SelectDataset, p []string) *goqu.SelectDataset { table := qb.table() diff --git a/pkg/sqlite/folder_filter.go b/pkg/sqlite/folder_filter.go index 6b2bd96e9..e0145bcca 100644 --- a/pkg/sqlite/folder_filter.go +++ b/pkg/sqlite/folder_filter.go @@ -65,6 +65,7 @@ func (qb *folderFilterHandler) criterionHandler() criterionHandler { folderFilter := qb.folderFilter return compoundHandler{ stringCriterionHandler(folderFilter.Path, qb.table.Col("path")), + stringCriterionHandler(folderFilter.Basename, qb.table.Col("basename")), ×tampCriterionHandler{folderFilter.ModTime, qb.table.Col("mod_time"), nil}, qb.parentFolderCriterionHandler(folderFilter.ParentFolder), diff --git a/pkg/sqlite/folder_filter_test.go b/pkg/sqlite/folder_filter_test.go index c1c7d7a37..c08208f30 100644 --- a/pkg/sqlite/folder_filter_test.go +++ b/pkg/sqlite/folder_filter_test.go @@ -33,6 +33,17 @@ func TestFolderQuery(t *testing.T) { includeIdxs: []int{folderIdxWithSubFolder, folderIdxWithParentFolder}, excludeIdxs: []int{folderIdxInZip}, }, + { + name: "basename", + filter: &models.FolderFilterType{ + Basename: &models.StringCriterionInput{ + Value: getFolderBasename(folderIdxWithParentFolder, nil), + Modifier: models.CriterionModifierIncludes, + }, + }, + includeIdxs: []int{folderIdxWithParentFolder}, + excludeIdxs: []int{folderIdxInZip}, + }, { name: "parent folder", filter: &models.FolderFilterType{ diff --git a/pkg/sqlite/folder_test.go b/pkg/sqlite/folder_test.go index 15b2b96b8..072a1167f 100644 --- a/pkg/sqlite/folder_test.go +++ b/pkg/sqlite/folder_test.go @@ -186,8 +186,6 @@ func Test_FolderStore_Update(t *testing.T) { } assert.Equal(copy, *s) - - return }) } } @@ -239,3 +237,75 @@ func Test_FolderStore_FindByPath(t *testing.T) { }) } } + +func Test_FolderStore_GetManyParentFolderIDs(t *testing.T) { + var empty []models.FolderID + emptyResult := [][]models.FolderID{empty} + tests := []struct { + name string + parentFolderIDs []models.FolderID + want [][]models.FolderID + wantErr bool + }{ + { + "valid with parent folders", + []models.FolderID{folderIDs[folderIdxWithParentFolder]}, + [][]models.FolderID{ + { + folderIDs[folderIdxWithSubFolder], + folderIDs[folderIdxRoot], + }, + }, + false, + }, + { + "valid multiple folders", + []models.FolderID{ + folderIDs[folderIdxWithParentFolder], + folderIDs[folderIdxWithSceneFiles], + }, + [][]models.FolderID{ + { + folderIDs[folderIdxWithSubFolder], + folderIDs[folderIdxRoot], + }, + { + folderIDs[folderIdxForObjectFiles], + folderIDs[folderIdxRoot], + }, + }, + false, + }, + { + "valid without parent folders", + []models.FolderID{folderIDs[folderIdxRoot]}, + emptyResult, + false, + }, + { + "invalid folder id", + []models.FolderID{invalidFolderID}, + emptyResult, + // does not error, just returns empty result + false, + }, + } + + qb := db.Folder + + for _, tt := range tests { + runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { + assert := assert.New(t) + got, err := qb.GetManyParentFolderIDs(ctx, tt.parentFolderIDs) + if (err != nil) != tt.wantErr { + assert.Errorf(err, "FolderStore.GetManyParentFolderIDs() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + + assert.Equal(got, tt.want) + }) + } +} diff --git a/pkg/sqlite/migrations/84_folder_basename.up.sql b/pkg/sqlite/migrations/84_folder_basename.up.sql new file mode 100644 index 000000000..5cfd5c2d9 --- /dev/null +++ b/pkg/sqlite/migrations/84_folder_basename.up.sql @@ -0,0 +1,50 @@ +-- we cannot add basename column directly because we require it to be NOT NULL +-- recreate folders table with basename column +PRAGMA foreign_keys=OFF; + +CREATE TABLE `folders_new` ( + `id` integer not null primary key autoincrement, + `basename` varchar(255) NOT NULL, + `path` varchar(255) NOT NULL, + `parent_folder_id` integer, + `zip_file_id` integer REFERENCES `files`(`id`), + `mod_time` datetime not null, + `created_at` datetime not null, + `updated_at` datetime not null, + foreign key(`parent_folder_id`) references `folders`(`id`) on delete SET NULL +); + +-- copy data from old table to new table, setting basename to path temporarily +INSERT INTO `folders_new` ( + `id`, + `basename`, + `path`, + `parent_folder_id`, + `zip_file_id`, + `mod_time`, + `created_at`, + `updated_at` +) SELECT + `id`, + `path`, + `path`, + `parent_folder_id`, + `zip_file_id`, + `mod_time`, + `created_at`, + `updated_at` +FROM `folders`; + +DROP INDEX IF EXISTS `index_folders_on_parent_folder_id`; +DROP INDEX IF EXISTS `index_folders_on_path_unique`; +DROP INDEX IF EXISTS `index_folders_on_zip_file_id`; +DROP TABLE `folders`; + +ALTER TABLE `folders_new` RENAME TO `folders`; + +CREATE UNIQUE INDEX `index_folders_on_path_unique` on `folders` (`path`); +CREATE UNIQUE INDEX `index_folders_on_parent_folder_id_basename_unique` on `folders` (`parent_folder_id`, `basename`); +CREATE INDEX `index_folders_on_zip_file_id` on `folders` (`zip_file_id`) WHERE `zip_file_id` IS NOT NULL; +CREATE INDEX `index_folders_on_basename` on `folders` (`basename`); + +PRAGMA foreign_keys=ON; \ No newline at end of file diff --git a/pkg/sqlite/migrations/84_postmigrate.go b/pkg/sqlite/migrations/84_postmigrate.go new file mode 100644 index 000000000..71b0feeb0 --- /dev/null +++ b/pkg/sqlite/migrations/84_postmigrate.go @@ -0,0 +1,285 @@ +package migrations + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "slices" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/sqlite" + "gopkg.in/guregu/null.v4" +) + +func post84(ctx context.Context, db *sqlx.DB) error { + logger.Info("Running post-migration for schema version 76") + + m := schema84Migrator{ + migrator: migrator{ + db: db, + }, + folderCache: make(map[string]folderInfo), + } + + rootPaths := config.GetInstance().GetStashPaths().Paths() + + if err := m.createMissingFolderHierarchies(ctx, rootPaths); err != nil { + return fmt.Errorf("creating missing folder hierarchies: %w", err) + } + + if err := m.migrateFolders(ctx); err != nil { + return fmt.Errorf("migrating folders: %w", err) + } + + return nil +} + +type schema84Migrator struct { + migrator + folderCache map[string]folderInfo +} + +func (m *schema84Migrator) createMissingFolderHierarchies(ctx context.Context, rootPaths []string) error { + // before we set the basenames, we need to address any folders that are missing their + // parent folders. + const ( + limit = 1000 + logEvery = 10000 + ) + + lastID := 0 + count := 0 + logged := false + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` WHERE `folders`.`parent_folder_id` IS NULL " + + if lastID != 0 { + query += fmt.Sprintf("AND `folders`.`id` > %d ", lastID) + } + + query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit) + + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + // log once if we find any folders with missing parent folders + if !logged { + logger.Info("Migrating folders with missing parents...") + logged = true + } + + var id int + var p string + + err := rows.Scan(&id, &p) + if err != nil { + return err + } + + lastID = id + gotSome = true + count++ + + // don't try to create parent folders for root paths + if slices.Contains(rootPaths, p) { + continue + } + + parentDir := filepath.Dir(p) + if parentDir == p { + // this can happen if the path is something like "C:\", where the parent directory is the same as the current directory + continue + } + + parentID, err := m.getOrCreateFolderHierarchy(tx, parentDir, rootPaths) + if err != nil { + return fmt.Errorf("error creating parent folder for folder %d %q: %w", id, p, err) + } + + if parentID == nil { + continue + } + + // now set the parent folder ID for the current folder + logger.Debugf("Migrating folder %d %q: setting parent folder ID to %d", id, p, *parentID) + + _, err = tx.Exec("UPDATE `folders` SET `parent_folder_id` = ? WHERE `id` = ?", *parentID, id) + if err != nil { + return fmt.Errorf("error setting parent folder for folder %d %q: %w", id, p, err) + } + } + + return rows.Err() + }); err != nil { + return err + } + + if !gotSome { + break + } + + if count%logEvery == 0 { + logger.Infof("Migrated %d folders", count) + } + } + + return nil +} + +func (m *schema84Migrator) findFolderByPath(tx *sqlx.Tx, path string) (*int, error) { + query := "SELECT `folders`.`id` FROM `folders` WHERE `folders`.`path` = ?" + + var id int + if err := tx.Get(&id, query, path); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + return nil, err + } + + return &id, nil +} + +// this is a copy of the GetOrCreateFolderHierarchy function from pkg/file/folder.go, +// but modified to use low-level SQL queries instead of the models.FolderFinderCreator interface, to avoid +func (m *schema84Migrator) getOrCreateFolderHierarchy(tx *sqlx.Tx, path string, rootPaths []string) (*int, error) { + // get or create folder hierarchy + folderID, err := m.findFolderByPath(tx, path) + if err != nil { + return nil, err + } + + if folderID == nil { + var parentID *int + + if !slices.Contains(rootPaths, path) { + parentPath := filepath.Dir(path) + + // it's possible that the parent path is the same as the current path, if there are folders outside + // of the root paths. In that case, we should just return nil for the parent ID. + if parentPath == path { + return nil, nil + } + + parentID, err = m.getOrCreateFolderHierarchy(tx, parentPath, rootPaths) + if err != nil { + return nil, err + } + } + + logger.Debugf("%s doesn't exist. Creating new folder entry...", path) + + // we need to set basename to path, which will be addressed in the next step + const insertSQL = "INSERT INTO `folders` (`path`,`basename`,`parent_folder_id`,`mod_time`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?)" + + var parentFolderID null.Int + if parentID != nil { + parentFolderID = null.IntFrom(int64(*parentID)) + } + + now := time.Now() + result, err := tx.Exec(insertSQL, path, path, parentFolderID, time.Time{}, now, now) + if err != nil { + return nil, fmt.Errorf("creating folder %s: %w", path, err) + } + + id, err := result.LastInsertId() + if err != nil { + return nil, fmt.Errorf("creating folder %s: %w", path, err) + } + + idInt := int(id) + folderID = &idInt + } + + return folderID, nil +} + +func (m *schema84Migrator) migrateFolders(ctx context.Context) error { + const ( + limit = 1000 + logEvery = 10000 + ) + + lastID := 0 + count := 0 + logged := false + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` " + + if lastID != 0 { + query += fmt.Sprintf("WHERE `folders`.`id` > %d ", lastID) + } + + query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit) + + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + if !logged { + logger.Infof("Migrating folders to set basenames...") + logged = true + } + + var id int + var p string + + err := rows.Scan(&id, &p) + if err != nil { + return err + } + + lastID = id + gotSome = true + count++ + + basename := filepath.Base(p) + logger.Debugf("Migrating folder %d %q: setting basename to %q", id, p, basename) + _, err = tx.Exec("UPDATE `folders` SET `basename` = ? WHERE `id` = ?", basename, id) + if err != nil { + return fmt.Errorf("error migrating folder %d %q: %w", id, p, err) + } + } + + return rows.Err() + }); err != nil { + return err + } + + if !gotSome { + break + } + + if count%logEvery == 0 { + logger.Infof("Migrated %d folders", count) + } + } + + return nil +} + +func init() { + sqlite.RegisterPostMigration(84, post84) +} diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 2848a0a14..d8baae3b8 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -31,7 +31,8 @@ const ( ) const ( - folderIdxWithSubFolder = iota + folderIdxRoot = iota + folderIdxWithSubFolder folderIdxWithParentFolder folderIdxWithFiles folderIdxInZip @@ -359,6 +360,8 @@ func (m linkMap) reverseLookup(idx int) []int { var ( folderParentFolders = map[int]int{ + folderIdxWithSubFolder: folderIdxRoot, + folderIdxForObjectFiles: folderIdxRoot, folderIdxWithParentFolder: folderIdxWithSubFolder, folderIdxWithSceneFiles: folderIdxForObjectFiles, folderIdxWithImageFiles: folderIdxForObjectFiles, @@ -785,6 +788,10 @@ func getFolderPath(index int, parentFolderIdx *int) string { return path } +func getFolderBasename(index int, parentFolderIdx *int) string { + return filepath.Base(getFolderPath(index, parentFolderIdx)) +} + func getFolderModTime(index int) time.Time { return time.Date(2000, 1, (index%10)+1, 0, 0, 0, 0, time.UTC) }