Add basename and parent_folders fields to Folder graphql interface (#6494)

* Add basename field to folder
* Add parent_folders field to folder
* Add basename column to folder table
* Add basename filter field
* Create missing folder hierarchies during migration
* Treat files/folders in zips where path can't be made relative as not found

Addresses an issue during clean where corrupt folder entries in zip files could not be removed due to an error during the call to Rel.

graphql/schema/types/file.graphql

@@ -6,11 +6,14 @@ type Fingerprint {
 type Folder {
   id: ID!
   path: String!
+  basename: String!
 
   parent_folder_id: ID @deprecated(reason: "Use parent_folder instead")
   zip_file_id: ID @deprecated(reason: "Use zip_file instead")
 
   parent_folder: Folder
+  "Returns all parent folders in order from immediate parent to top-level"
+  parent_folders: [Folder!]!
   zip_file: BasicFile
 
   mod_time: Time!

graphql/schema/types/filters.graphql

@@ -822,6 +822,7 @@ input FolderFilterType {
   NOT: FolderFilterType
 
   path: StringCriterionInput
+  basename: StringCriterionInput
 
   parent_folder: HierarchicalMultiCriterionInput
   zip_file: MultiCriterionInput

internal/api/loaders/dataloaders.go

@@ -11,6 +11,7 @@
 //go:generate go run github.com/vektah/dataloaden GroupLoader int *github.com/stashapp/stash/pkg/models.Group
 //go:generate go run github.com/vektah/dataloaden FileLoader github.com/stashapp/stash/pkg/models.FileID github.com/stashapp/stash/pkg/models.File
 //go:generate go run github.com/vektah/dataloaden FolderLoader github.com/stashapp/stash/pkg/models.FolderID *github.com/stashapp/stash/pkg/models.Folder
+//go:generate go run github.com/vektah/dataloaden FolderParentFolderIDsLoader github.com/stashapp/stash/pkg/models.FolderID []github.com/stashapp/stash/pkg/models.FolderID
 //go:generate go run github.com/vektah/dataloaden SceneFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
 //go:generate go run github.com/vektah/dataloaden ImageFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
 //go:generate go run github.com/vektah/dataloaden GalleryFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
@@ -65,12 +66,16 @@ type Loaders struct {
 	StudioByID         *StudioLoader
 	StudioCustomFields *CustomFieldsLoader
 
-	TagByID           *TagLoader
-	TagCustomFields   *CustomFieldsLoader
+	TagByID         *TagLoader
+	TagCustomFields *CustomFieldsLoader
+
 	GroupByID         *GroupLoader
 	GroupCustomFields *CustomFieldsLoader
-	FileByID          *FileLoader
-	FolderByID        *FolderLoader
+
+	FileByID *FileLoader
+
+	FolderByID            *FolderLoader
+	FolderParentFolderIDs *FolderParentFolderIDsLoader
 }
 
 type Middleware struct {
@@ -161,6 +166,11 @@ func (m Middleware) Middleware(next http.Handler) http.Handler {
 				maxBatch: maxBatch,
 				fetch:    m.fetchFolders(ctx),
 			},
+			FolderParentFolderIDs: &FolderParentFolderIDsLoader{
+				wait:     wait,
+				maxBatch: maxBatch,
+				fetch:    m.fetchFoldersParentFolderIDs(ctx),
+			},
 			SceneFiles: &SceneFileIDsLoader{
 				wait:     wait,
 				maxBatch: maxBatch,
@@ -406,6 +416,17 @@ func (m Middleware) fetchFolders(ctx context.Context) func(keys []models.FolderI
 	}
 }
 
+func (m Middleware) fetchFoldersParentFolderIDs(ctx context.Context) func(keys []models.FolderID) ([][]models.FolderID, []error) {
+	return func(keys []models.FolderID) (ret [][]models.FolderID, errs []error) {
+		err := m.Repository.WithDB(ctx, func(ctx context.Context) error {
+			var err error
+			ret, err = m.Repository.Folder.GetManyParentFolderIDs(ctx, keys)
+			return err
+		})
+		return ret, toErrorSlice(err)
+	}
+}
+
 func (m Middleware) fetchScenesFileIDs(ctx context.Context) func(keys []int) ([][]models.FileID, []error) {
 	return func(keys []int) (ret [][]models.FileID, errs []error) {
 		err := m.Repository.WithDB(ctx, func(ctx context.Context) error {

internal/api/loaders/folderparentfolderidsloader_gen.go

@@ -0,0 +1,225 @@
+// Code generated by github.com/vektah/dataloaden, DO NOT EDIT.
+
+package loaders
+
+import (
+	"sync"
+	"time"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+// FolderParentFolderIDsLoaderConfig captures the config to create a new FolderParentFolderIDsLoader
+type FolderParentFolderIDsLoaderConfig struct {
+	// Fetch is a method that provides the data for the loader
+	Fetch func(keys []models.FolderID) ([][]models.FolderID, []error)
+
+	// Wait is how long wait before sending a batch
+	Wait time.Duration
+
+	// MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit
+	MaxBatch int
+}
+
+// NewFolderParentFolderIDsLoader creates a new FolderParentFolderIDsLoader given a fetch, wait, and maxBatch
+func NewFolderParentFolderIDsLoader(config FolderParentFolderIDsLoaderConfig) *FolderParentFolderIDsLoader {
+	return &FolderParentFolderIDsLoader{
+		fetch:    config.Fetch,
+		wait:     config.Wait,
+		maxBatch: config.MaxBatch,
+	}
+}
+
+// FolderParentFolderIDsLoader batches and caches requests
+type FolderParentFolderIDsLoader struct {
+	// this method provides the data for the loader
+	fetch func(keys []models.FolderID) ([][]models.FolderID, []error)
+
+	// how long to done before sending a batch
+	wait time.Duration
+
+	// this will limit the maximum number of keys to send in one batch, 0 = no limit
+	maxBatch int
+
+	// INTERNAL
+
+	// lazily created cache
+	cache map[models.FolderID][]models.FolderID
+
+	// the current batch. keys will continue to be collected until timeout is hit,
+	// then everything will be sent to the fetch method and out to the listeners
+	batch *folderParentFolderIDsLoaderBatch
+
+	// mutex to prevent races
+	mu sync.Mutex
+}
+
+type folderParentFolderIDsLoaderBatch struct {
+	keys    []models.FolderID
+	data    [][]models.FolderID
+	error   []error
+	closing bool
+	done    chan struct{}
+}
+
+// Load a FolderID by key, batching and caching will be applied automatically
+func (l *FolderParentFolderIDsLoader) Load(key models.FolderID) ([]models.FolderID, error) {
+	return l.LoadThunk(key)()
+}
+
+// LoadThunk returns a function that when called will block waiting for a FolderID.
+// This method should be used if you want one goroutine to make requests to many
+// different data loaders without blocking until the thunk is called.
+func (l *FolderParentFolderIDsLoader) LoadThunk(key models.FolderID) func() ([]models.FolderID, error) {
+	l.mu.Lock()
+	if it, ok := l.cache[key]; ok {
+		l.mu.Unlock()
+		return func() ([]models.FolderID, error) {
+			return it, nil
+		}
+	}
+	if l.batch == nil {
+		l.batch = &folderParentFolderIDsLoaderBatch{done: make(chan struct{})}
+	}
+	batch := l.batch
+	pos := batch.keyIndex(l, key)
+	l.mu.Unlock()
+
+	return func() ([]models.FolderID, error) {
+		<-batch.done
+
+		var data []models.FolderID
+		if pos < len(batch.data) {
+			data = batch.data[pos]
+		}
+
+		var err error
+		// its convenient to be able to return a single error for everything
+		if len(batch.error) == 1 {
+			err = batch.error[0]
+		} else if batch.error != nil {
+			err = batch.error[pos]
+		}
+
+		if err == nil {
+			l.mu.Lock()
+			l.unsafeSet(key, data)
+			l.mu.Unlock()
+		}
+
+		return data, err
+	}
+}
+
+// LoadAll fetches many keys at once. It will be broken into appropriate sized
+// sub batches depending on how the loader is configured
+func (l *FolderParentFolderIDsLoader) LoadAll(keys []models.FolderID) ([][]models.FolderID, []error) {
+	results := make([]func() ([]models.FolderID, error), len(keys))
+
+	for i, key := range keys {
+		results[i] = l.LoadThunk(key)
+	}
+
+	folderIDs := make([][]models.FolderID, len(keys))
+	errors := make([]error, len(keys))
+	for i, thunk := range results {
+		folderIDs[i], errors[i] = thunk()
+	}
+	return folderIDs, errors
+}
+
+// LoadAllThunk returns a function that when called will block waiting for a FolderIDs.
+// This method should be used if you want one goroutine to make requests to many
+// different data loaders without blocking until the thunk is called.
+func (l *FolderParentFolderIDsLoader) LoadAllThunk(keys []models.FolderID) func() ([][]models.FolderID, []error) {
+	results := make([]func() ([]models.FolderID, error), len(keys))
+	for i, key := range keys {
+		results[i] = l.LoadThunk(key)
+	}
+	return func() ([][]models.FolderID, []error) {
+		folderIDs := make([][]models.FolderID, len(keys))
+		errors := make([]error, len(keys))
+		for i, thunk := range results {
+			folderIDs[i], errors[i] = thunk()
+		}
+		return folderIDs, errors
+	}
+}
+
+// Prime the cache with the provided key and value. If the key already exists, no change is made
+// and false is returned.
+// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).)
+func (l *FolderParentFolderIDsLoader) Prime(key models.FolderID, value []models.FolderID) bool {
+	l.mu.Lock()
+	var found bool
+	if _, found = l.cache[key]; !found {
+		// make a copy when writing to the cache, its easy to pass a pointer in from a loop var
+		// and end up with the whole cache pointing to the same value.
+		cpy := make([]models.FolderID, len(value))
+		copy(cpy, value)
+		l.unsafeSet(key, cpy)
+	}
+	l.mu.Unlock()
+	return !found
+}
+
+// Clear the value at key from the cache, if it exists
+func (l *FolderParentFolderIDsLoader) Clear(key models.FolderID) {
+	l.mu.Lock()
+	delete(l.cache, key)
+	l.mu.Unlock()
+}
+
+func (l *FolderParentFolderIDsLoader) unsafeSet(key models.FolderID, value []models.FolderID) {
+	if l.cache == nil {
+		l.cache = map[models.FolderID][]models.FolderID{}
+	}
+	l.cache[key] = value
+}
+
+// keyIndex will return the location of the key in the batch, if its not found
+// it will add the key to the batch
+func (b *folderParentFolderIDsLoaderBatch) keyIndex(l *FolderParentFolderIDsLoader, key models.FolderID) int {
+	for i, existingKey := range b.keys {
+		if key == existingKey {
+			return i
+		}
+	}
+
+	pos := len(b.keys)
+	b.keys = append(b.keys, key)
+	if pos == 0 {
+		go b.startTimer(l)
+	}
+
+	if l.maxBatch != 0 && pos >= l.maxBatch-1 {
+		if !b.closing {
+			b.closing = true
+			l.batch = nil
+			go b.end(l)
+		}
+	}
+
+	return pos
+}
+
+func (b *folderParentFolderIDsLoaderBatch) startTimer(l *FolderParentFolderIDsLoader) {
+	time.Sleep(l.wait)
+	l.mu.Lock()
+
+	// we must have hit a batch limit and are already finalizing this batch
+	if b.closing {
+		l.mu.Unlock()
+		return
+	}
+
+	l.batch = nil
+	l.mu.Unlock()
+
+	b.end(l)
+}
+
+func (b *folderParentFolderIDsLoaderBatch) end(l *FolderParentFolderIDsLoader) {
+	b.data, b.error = l.fetch(b.keys)
+	close(b.done)
+}

internal/api/resolver_model_folder.go

@@ -2,11 +2,16 @@ package api
 
 import (
 	"context"
+	"path/filepath"
 
 	"github.com/stashapp/stash/internal/api/loaders"
 	"github.com/stashapp/stash/pkg/models"
 )
 
+func (r *folderResolver) Basename(ctx context.Context, obj *models.Folder) (string, error) {
+	return filepath.Base(obj.Path), nil
+}
+
 func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) (*models.Folder, error) {
 	if obj.ParentFolderID == nil {
 		return nil, nil
@@ -15,6 +20,17 @@ func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) (
 	return loaders.From(ctx).FolderByID.Load(*obj.ParentFolderID)
 }
 
+func (r *folderResolver) ParentFolders(ctx context.Context, obj *models.Folder) ([]*models.Folder, error) {
+	ids, err := loaders.From(ctx).FolderParentFolderIDs.Load(obj.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	var errs []error
+	ret, errs := loaders.From(ctx).FolderByID.LoadAll(ids)
+	return ret, firstError(errs)
+}
+
 func (r *folderResolver) ZipFile(ctx context.Context, obj *models.Folder) (*BasicFile, error) {
 	return zipFileResolver(ctx, obj.ZipFileID)
 }

pkg/file/zip.go

@@ -99,7 +99,9 @@ func (f *zipFS) rel(name string) (string, error) {
 
 	relName, err := filepath.Rel(f.zipPath, name)
 	if err != nil {
-		return "", fmt.Errorf("internal error getting relative path: %w", err)
+		// if the path is not relative to the zip path, then it's not found in the zip file,
+		// so treat this as a file not found
+		return "", fs.ErrNotExist
 	}
 
 	// convert relName to use slash, since zip files do so regardless

pkg/models/folder.go

@@ -18,10 +18,8 @@ type FolderQueryOptions struct {
 type FolderFilterType struct {
 	OperatorFilter[FolderFilterType]
 
-	Path     *StringCriterionInput `json:"path,omitempty"`
-	Basename *StringCriterionInput `json:"basename,omitempty"`
-	// Filter by parent directory path
-	Dir          *StringCriterionInput            `json:"dir,omitempty"`
+	Path         *StringCriterionInput            `json:"path,omitempty"`
+	Basename     *StringCriterionInput            `json:"basename,omitempty"`
 	ParentFolder *HierarchicalMultiCriterionInput `json:"parent_folder,omitempty"`
 	ZipFile      *MultiCriterionInput             `json:"zip_file,omitempty"`
 	// Filter by modification time

pkg/models/mocks/FolderReaderWriter.go

@@ -201,6 +201,29 @@ func (_m *FolderReaderWriter) FindMany(ctx context.Context, id []models.FolderID
 	return r0, r1
 }
 
+// GetManyParentFolderIDs provides a mock function with given fields: ctx, folderIDs
+func (_m *FolderReaderWriter) GetManyParentFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) {
+	ret := _m.Called(ctx, folderIDs)
+
+	var r0 [][]models.FolderID
+	if rf, ok := ret.Get(0).(func(context.Context, []models.FolderID) [][]models.FolderID); ok {
+		r0 = rf(ctx, folderIDs)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([][]models.FolderID)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, []models.FolderID) error); ok {
+		r1 = rf(ctx, folderIDs)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Query provides a mock function with given fields: ctx, options
 func (_m *FolderReaderWriter) Query(ctx context.Context, options models.FolderQueryOptions) (*models.FolderQueryResult, error) {
 	ret := _m.Called(ctx, options)

pkg/models/repository_folder.go

@@ -15,6 +15,7 @@ type FolderFinder interface {
 	FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error)
 	FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error)
 	FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error)
+	GetManyParentFolderIDs(ctx context.Context, folderIDs []FolderID) ([][]FolderID, error)
 }
 
 type FolderQueryer interface {

pkg/sqlite/database.go

@@ -34,7 +34,7 @@ const (
 	cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
 )
 
-var appSchemaVersion uint = 83
+var appSchemaVersion uint = 84
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/folder.go

@@ -20,6 +20,7 @@ const folderIDColumn = "folder_id"
 
 type folderRow struct {
 	ID             models.FolderID `db:"id" goqu:"skipinsert"`
+	Basename       string          `db:"basename"`
 	Path           string          `db:"path"`
 	ZipFileID      null.Int        `db:"zip_file_id"`
 	ParentFolderID null.Int        `db:"parent_folder_id"`
@@ -30,6 +31,8 @@ type folderRow struct {
 
 func (r *folderRow) fromFolder(o models.Folder) {
 	r.ID = o.ID
+	// derive basename from path
+	r.Basename = filepath.Base(o.Path)
 	r.Path = o.Path
 	r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID)
 	r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID)
@@ -322,6 +325,90 @@ func (qb *FolderStore) FindByParentFolderID(ctx context.Context, parentFolderID
 	return ret, nil
 }
 
+func (qb *FolderStore) GetManyParentFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) {
+	table := qb.table()
+
+	// SQL recursive query to get all parent folder IDs for each folder ID
+	/*
+		WITH RECURSIVE parent_folders AS (
+		    SELECT id, parent_folder_id
+		    FROM folders
+		    WHERE id IN (folderIDs)
+
+		    UNION ALL
+
+		    SELECT f.id, f.parent_folder_id
+		    FROM folders f
+		    INNER JOIN parent_folders pf ON f.id = pf.parent_folder_id
+		)
+		SELECT id, parent_folder_id FROM parent_folders;
+	*/
+	const parentFolders = "parent_folders"
+	const parentFolderID = "parent_folder_id"
+	const parentID = "parent_id"
+	const foldersAlias = "f"
+
+	const parentFoldersAlias = "pf"
+	foldersAliasedI := table.As(foldersAlias)
+	parentFoldersI := goqu.T(parentFolders).As(parentFoldersAlias)
+
+	q := dialect.From(parentFolders).Prepared(true).
+		WithRecursive(parentFolders,
+			dialect.From(table).Select(table.Col(idColumn), table.Col(parentFolderID).As(parentID)).
+				Where(table.Col(idColumn).In(folderIDs)).
+				Union(
+					dialect.From(foldersAliasedI).InnerJoin(
+						parentFoldersI,
+						goqu.On(foldersAliasedI.Col(idColumn).Eq(parentFoldersI.Col(parentID))),
+					).Select(foldersAliasedI.Col(idColumn), foldersAliasedI.Col(parentFolderID).As(parentID)),
+				),
+		).Select(idColumn, parentID)
+
+	type resultRow struct {
+		FolderID       models.FolderID `db:"id"`
+		ParentFolderID null.Int        `db:"parent_id"`
+	}
+
+	folderMap := make(map[models.FolderID]models.FolderID)
+
+	if err := queryFunc(ctx, q, false, func(r *sqlx.Rows) error {
+		var row resultRow
+		if err := r.StructScan(&row); err != nil {
+			return err
+		}
+
+		if row.ParentFolderID.Valid {
+			folderMap[row.FolderID] = models.FolderID(row.ParentFolderID.Int64)
+		} else {
+			folderMap[row.FolderID] = 0
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	ret := make([][]models.FolderID, len(folderIDs))
+
+	for i, folderID := range folderIDs {
+		var parents []models.FolderID
+		currentID := folderID
+
+		for {
+			parentID, exists := folderMap[currentID]
+			if !exists || parentID == 0 {
+				break
+			}
+			parents = append(parents, parentID)
+			currentID = parentID
+		}
+
+		ret[i] = parents
+	}
+
+	return ret, nil
+}
+
 func (qb *FolderStore) allInPaths(q *goqu.SelectDataset, p []string) *goqu.SelectDataset {
 	table := qb.table()
 

pkg/sqlite/folder_filter.go

@@ -65,6 +65,7 @@ func (qb *folderFilterHandler) criterionHandler() criterionHandler {
 	folderFilter := qb.folderFilter
 	return compoundHandler{
 		stringCriterionHandler(folderFilter.Path, qb.table.Col("path")),
+		stringCriterionHandler(folderFilter.Basename, qb.table.Col("basename")),
 		&timestampCriterionHandler{folderFilter.ModTime, qb.table.Col("mod_time"), nil},
 
 		qb.parentFolderCriterionHandler(folderFilter.ParentFolder),

pkg/sqlite/folder_filter_test.go

@@ -33,6 +33,17 @@ func TestFolderQuery(t *testing.T) {
 			includeIdxs: []int{folderIdxWithSubFolder, folderIdxWithParentFolder},
 			excludeIdxs: []int{folderIdxInZip},
 		},
+		{
+			name: "basename",
+			filter: &models.FolderFilterType{
+				Basename: &models.StringCriterionInput{
+					Value:    getFolderBasename(folderIdxWithParentFolder, nil),
+					Modifier: models.CriterionModifierIncludes,
+				},
+			},
+			includeIdxs: []int{folderIdxWithParentFolder},
+			excludeIdxs: []int{folderIdxInZip},
+		},
 		{
 			name: "parent folder",
 			filter: &models.FolderFilterType{

pkg/sqlite/folder_test.go

@@ -186,8 +186,6 @@ func Test_FolderStore_Update(t *testing.T) {
 			}
 
 			assert.Equal(copy, *s)
-
-			return
 		})
 	}
 }
@@ -239,3 +237,75 @@ func Test_FolderStore_FindByPath(t *testing.T) {
 		})
 	}
 }
+
+func Test_FolderStore_GetManyParentFolderIDs(t *testing.T) {
+	var empty []models.FolderID
+	emptyResult := [][]models.FolderID{empty}
+	tests := []struct {
+		name            string
+		parentFolderIDs []models.FolderID
+		want            [][]models.FolderID
+		wantErr         bool
+	}{
+		{
+			"valid with parent folders",
+			[]models.FolderID{folderIDs[folderIdxWithParentFolder]},
+			[][]models.FolderID{
+				{
+					folderIDs[folderIdxWithSubFolder],
+					folderIDs[folderIdxRoot],
+				},
+			},
+			false,
+		},
+		{
+			"valid multiple folders",
+			[]models.FolderID{
+				folderIDs[folderIdxWithParentFolder],
+				folderIDs[folderIdxWithSceneFiles],
+			},
+			[][]models.FolderID{
+				{
+					folderIDs[folderIdxWithSubFolder],
+					folderIDs[folderIdxRoot],
+				},
+				{
+					folderIDs[folderIdxForObjectFiles],
+					folderIDs[folderIdxRoot],
+				},
+			},
+			false,
+		},
+		{
+			"valid without parent folders",
+			[]models.FolderID{folderIDs[folderIdxRoot]},
+			emptyResult,
+			false,
+		},
+		{
+			"invalid folder id",
+			[]models.FolderID{invalidFolderID},
+			emptyResult,
+			// does not error, just returns empty result
+			false,
+		},
+	}
+
+	qb := db.Folder
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.GetManyParentFolderIDs(ctx, tt.parentFolderIDs)
+			if (err != nil) != tt.wantErr {
+				assert.Errorf(err, "FolderStore.GetManyParentFolderIDs() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if tt.wantErr {
+				return
+			}
+
+			assert.Equal(got, tt.want)
+		})
+	}
+}

pkg/sqlite/migrations/84_folder_basename.up.sql

@@ -0,0 +1,50 @@
+-- we cannot add basename column directly because we require it to be NOT NULL
+-- recreate folders table with basename column
+PRAGMA foreign_keys=OFF;
+
+CREATE TABLE `folders_new` (
+  `id` integer not null primary key autoincrement,
+  `basename` varchar(255) NOT NULL,
+  `path` varchar(255) NOT NULL,
+  `parent_folder_id` integer,
+  `zip_file_id` integer REFERENCES `files`(`id`),
+  `mod_time` datetime not null,
+  `created_at` datetime not null,
+  `updated_at` datetime not null,
+  foreign key(`parent_folder_id`) references `folders`(`id`) on delete SET NULL
+);
+
+-- copy data from old table to new table, setting basename to path temporarily
+INSERT INTO `folders_new` (
+    `id`, 
+    `basename`, 
+    `path`, 
+    `parent_folder_id`, 
+    `zip_file_id`,
+    `mod_time`, 
+    `created_at`, 
+    `updated_at`
+) SELECT 
+    `id`, 
+    `path`, 
+    `path`, 
+    `parent_folder_id`, 
+    `zip_file_id`,
+    `mod_time`, 
+    `created_at`, 
+    `updated_at`
+FROM `folders`;
+
+DROP INDEX IF EXISTS `index_folders_on_parent_folder_id`;
+DROP INDEX IF EXISTS `index_folders_on_path_unique`;
+DROP INDEX IF EXISTS `index_folders_on_zip_file_id`;
+DROP TABLE `folders`;
+
+ALTER TABLE `folders_new` RENAME TO `folders`;
+
+CREATE UNIQUE INDEX `index_folders_on_path_unique` on `folders` (`path`);
+CREATE UNIQUE INDEX `index_folders_on_parent_folder_id_basename_unique` on `folders` (`parent_folder_id`, `basename`);
+CREATE INDEX `index_folders_on_zip_file_id` on `folders` (`zip_file_id`) WHERE `zip_file_id` IS NOT NULL;
+CREATE INDEX `index_folders_on_basename` on `folders` (`basename`);
+
+PRAGMA foreign_keys=ON;

pkg/sqlite/migrations/84_postmigrate.go

@@ -0,0 +1,285 @@
+package migrations
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"slices"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stashapp/stash/internal/manager/config"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/sqlite"
+	"gopkg.in/guregu/null.v4"
+)
+
+func post84(ctx context.Context, db *sqlx.DB) error {
+	logger.Info("Running post-migration for schema version 76")
+
+	m := schema84Migrator{
+		migrator: migrator{
+			db: db,
+		},
+		folderCache: make(map[string]folderInfo),
+	}
+
+	rootPaths := config.GetInstance().GetStashPaths().Paths()
+
+	if err := m.createMissingFolderHierarchies(ctx, rootPaths); err != nil {
+		return fmt.Errorf("creating missing folder hierarchies: %w", err)
+	}
+
+	if err := m.migrateFolders(ctx); err != nil {
+		return fmt.Errorf("migrating folders: %w", err)
+	}
+
+	return nil
+}
+
+type schema84Migrator struct {
+	migrator
+	folderCache map[string]folderInfo
+}
+
+func (m *schema84Migrator) createMissingFolderHierarchies(ctx context.Context, rootPaths []string) error {
+	// before we set the basenames, we need to address any folders that are missing their
+	// parent folders.
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
+	lastID := 0
+	count := 0
+	logged := false
+
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` WHERE `folders`.`parent_folder_id` IS NULL "
+
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `folders`.`id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit)
+
+			rows, err := tx.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				// log once if we find any folders with missing parent folders
+				if !logged {
+					logger.Info("Migrating folders with missing parents...")
+					logged = true
+				}
+
+				var id int
+				var p string
+
+				err := rows.Scan(&id, &p)
+				if err != nil {
+					return err
+				}
+
+				lastID = id
+				gotSome = true
+				count++
+
+				// don't try to create parent folders for root paths
+				if slices.Contains(rootPaths, p) {
+					continue
+				}
+
+				parentDir := filepath.Dir(p)
+				if parentDir == p {
+					// this can happen if the path is something like "C:\", where the parent directory is the same as the current directory
+					continue
+				}
+
+				parentID, err := m.getOrCreateFolderHierarchy(tx, parentDir, rootPaths)
+				if err != nil {
+					return fmt.Errorf("error creating parent folder for folder %d %q: %w", id, p, err)
+				}
+
+				if parentID == nil {
+					continue
+				}
+
+				// now set the parent folder ID for the current folder
+				logger.Debugf("Migrating folder %d %q: setting parent folder ID to %d", id, p, *parentID)
+
+				_, err = tx.Exec("UPDATE `folders` SET `parent_folder_id` = ? WHERE `id` = ?", *parentID, id)
+				if err != nil {
+					return fmt.Errorf("error setting parent folder for folder %d %q: %w", id, p, err)
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
+			return err
+		}
+
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Migrated %d folders", count)
+		}
+	}
+
+	return nil
+}
+
+func (m *schema84Migrator) findFolderByPath(tx *sqlx.Tx, path string) (*int, error) {
+	query := "SELECT `folders`.`id` FROM `folders` WHERE `folders`.`path` = ?"
+
+	var id int
+	if err := tx.Get(&id, query, path); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, nil
+		}
+
+		return nil, err
+	}
+
+	return &id, nil
+}
+
+// this is a copy of the GetOrCreateFolderHierarchy function from pkg/file/folder.go,
+// but modified to use low-level SQL queries instead of the models.FolderFinderCreator interface, to avoid
+func (m *schema84Migrator) getOrCreateFolderHierarchy(tx *sqlx.Tx, path string, rootPaths []string) (*int, error) {
+	// get or create folder hierarchy
+	folderID, err := m.findFolderByPath(tx, path)
+	if err != nil {
+		return nil, err
+	}
+
+	if folderID == nil {
+		var parentID *int
+
+		if !slices.Contains(rootPaths, path) {
+			parentPath := filepath.Dir(path)
+
+			// it's possible that the parent path is the same as the current path, if there are folders outside
+			// of the root paths. In that case, we should just return nil for the parent ID.
+			if parentPath == path {
+				return nil, nil
+			}
+
+			parentID, err = m.getOrCreateFolderHierarchy(tx, parentPath, rootPaths)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		logger.Debugf("%s doesn't exist. Creating new folder entry...", path)
+
+		// we need to set basename to path, which will be addressed in the next step
+		const insertSQL = "INSERT INTO `folders` (`path`,`basename`,`parent_folder_id`,`mod_time`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?)"
+
+		var parentFolderID null.Int
+		if parentID != nil {
+			parentFolderID = null.IntFrom(int64(*parentID))
+		}
+
+		now := time.Now()
+		result, err := tx.Exec(insertSQL, path, path, parentFolderID, time.Time{}, now, now)
+		if err != nil {
+			return nil, fmt.Errorf("creating folder %s: %w", path, err)
+		}
+
+		id, err := result.LastInsertId()
+		if err != nil {
+			return nil, fmt.Errorf("creating folder %s: %w", path, err)
+		}
+
+		idInt := int(id)
+		folderID = &idInt
+	}
+
+	return folderID, nil
+}
+
+func (m *schema84Migrator) migrateFolders(ctx context.Context) error {
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
+	lastID := 0
+	count := 0
+	logged := false
+
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := "SELECT `folders`.`id`, `folders`.`path` FROM `folders` "
+
+			if lastID != 0 {
+				query += fmt.Sprintf("WHERE `folders`.`id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `folders`.`id` LIMIT %d", limit)
+
+			rows, err := tx.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				if !logged {
+					logger.Infof("Migrating folders to set basenames...")
+					logged = true
+				}
+
+				var id int
+				var p string
+
+				err := rows.Scan(&id, &p)
+				if err != nil {
+					return err
+				}
+
+				lastID = id
+				gotSome = true
+				count++
+
+				basename := filepath.Base(p)
+				logger.Debugf("Migrating folder %d %q: setting basename to %q", id, p, basename)
+				_, err = tx.Exec("UPDATE `folders` SET `basename` = ? WHERE `id` = ?", basename, id)
+				if err != nil {
+					return fmt.Errorf("error migrating folder %d %q: %w", id, p, err)
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
+			return err
+		}
+
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Migrated %d folders", count)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	sqlite.RegisterPostMigration(84, post84)
+}

pkg/sqlite/setup_test.go

@@ -31,7 +31,8 @@ const (
 )
 
 const (
-	folderIdxWithSubFolder = iota
+	folderIdxRoot = iota
+	folderIdxWithSubFolder
 	folderIdxWithParentFolder
 	folderIdxWithFiles
 	folderIdxInZip
@@ -359,6 +360,8 @@ func (m linkMap) reverseLookup(idx int) []int {
 
 var (
 	folderParentFolders = map[int]int{
+		folderIdxWithSubFolder:    folderIdxRoot,
+		folderIdxForObjectFiles:   folderIdxRoot,
 		folderIdxWithParentFolder: folderIdxWithSubFolder,
 		folderIdxWithSceneFiles:   folderIdxForObjectFiles,
 		folderIdxWithImageFiles:   folderIdxForObjectFiles,
@@ -785,6 +788,10 @@ func getFolderPath(index int, parentFolderIdx *int) string {
 	return path
 }
 
+func getFolderBasename(index int, parentFolderIdx *int) string {
+	return filepath.Base(getFolderPath(index, parentFolderIdx))
+}
+
 func getFolderModTime(index int) time.Time {
 	return time.Date(2000, 1, (index%10)+1, 0, 0, 0, 0, time.UTC)
 }