From 1d3bc40a6b4bf7287864fbde8d4c64e7ec1f6882 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:04:26 +1100 Subject: [PATCH] Import/export bug fixes (#5780) * Include parent tags in export if including dependencies * Handle uniqueness when sanitising filenames --- internal/manager/task_export.go | 53 +++++++++++++++++++++++++-------- pkg/fsutil/file.go | 9 +++++- pkg/fsutil/file_test.go | 14 ++++----- pkg/tag/export.go | 23 ++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index fe35c150d..5f2897670 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -1042,23 +1042,43 @@ func (t *ExportTask) ExportTags(ctx context.Context, workers int) { logger.Info("[tags] exporting") startTime := time.Now() - jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers - - for w := 0; w < workers; w++ { // create export Tag workers - tagsWg.Add(1) - go t.exportTag(ctx, &tagsWg, jobCh) + tagIdx := 0 + if t.tags != nil { + tagIdx = len(t.tags.IDs) } - for i, tag := range tags { - index := i + 1 - logger.Progressf("[tags] %d of %d", index, len(tags)) + for { + jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers - jobCh <- tag // feed workers + for w := 0; w < workers; w++ { // create export Tag workers + tagsWg.Add(1) + go t.exportTag(ctx, &tagsWg, jobCh) + } + + for i, tag := range tags { + index := i + 1 + tagIdx + logger.Progressf("[tags] %d of %d", index, len(tags)+tagIdx) + + jobCh <- tag // feed workers + } + + close(jobCh) + tagsWg.Wait() + + // if more tags were added, we need to export those too + if t.tags == nil || len(t.tags.IDs) == tagIdx { + break + } + + newTags, err := reader.FindMany(ctx, t.tags.IDs[tagIdx:]) + if err != nil { + logger.Errorf("[tags] failed to fetch tags: %v", err) + } + + tags = newTags + tagIdx = len(t.tags.IDs) } - close(jobCh) - tagsWg.Wait() - logger.Infof("[tags] export complete in %s. %d workers used.", time.Since(startTime), workers) } @@ -1075,6 +1095,15 @@ func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan continue } + if t.includeDependencies { + tagIDs, err := tag.GetDependentTagIDs(ctx, tagReader, thisTag) + if err != nil { + logger.Errorf("[tags] <%s> error getting dependent tags: %v", thisTag.Name, err) + continue + } + t.tags.IDs = sliceutil.AppendUniques(t.tags.IDs, tagIDs) + } + fn := newTagJSON.Filename() if err := t.json.saveTag(fn, newTagJSON); err != nil { diff --git a/pkg/fsutil/file.go b/pkg/fsutil/file.go index db5a13155..1d0c0c473 100644 --- a/pkg/fsutil/file.go +++ b/pkg/fsutil/file.go @@ -1,6 +1,8 @@ package fsutil import ( + "crypto/sha1" + "encoding/hex" "fmt" "io" "os" @@ -151,7 +153,12 @@ var ( ) // SanitiseBasename returns a file basename removing any characters that are illegal or problematic to use in the filesystem. +// It appends a short hash of the original string to ensure uniqueness. func SanitiseBasename(v string) string { + // Generate a short hash for uniqueness + hash := sha1.Sum([]byte(v)) + shortHash := hex.EncodeToString(hash[:4]) // Use the first 4 bytes of the hash + v = strings.TrimSpace(v) // replace illegal filename characters with - @@ -163,7 +170,7 @@ func SanitiseBasename(v string) string { // remove multiple hyphens v = multiHyphenRE.ReplaceAllString(v, "-") - return strings.TrimSpace(v) + return strings.TrimSpace(v) + "-" + shortHash } // GetExeName returns the name of the given executable for the current platform. diff --git a/pkg/fsutil/file_test.go b/pkg/fsutil/file_test.go index 393d3b420..4d84f8a47 100644 --- a/pkg/fsutil/file_test.go +++ b/pkg/fsutil/file_test.go @@ -8,13 +8,13 @@ func TestSanitiseBasename(t *testing.T) { v string want string }{ - {"basic", "basic", "basic"}, - {"spaces", `spaced name`, "spaced-name"}, - {"leading/trailing spaces", ` spaced name `, "spaced-name"}, - {"hyphen name", `hyphened-name`, "hyphened-name"}, - {"multi-hyphen", `hyphened--name`, "hyphened-name"}, - {"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g"}, - {"removed characters", `foo!!bar@@and, more`, "foobarand-more"}, + {"basic", "basic", "basic-61a7508e"}, + {"spaces", `spaced name`, "spaced-name-b297cf60"}, + {"leading/trailing spaces", ` spaced name `, "spaced-name-175433e9"}, + {"hyphen name", `hyphened-name`, "hyphened-name-789c55f2"}, + {"multi-hyphen", `hyphened--name`, "hyphened-name-2da2a58f"}, + {"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g-ffca6fb0"}, + {"removed characters", `foo!!bar@@and, more`, "foobarand-more-7cee02ab"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/tag/export.go b/pkg/tag/export.go index 58727f8f4..bd1573341 100644 --- a/pkg/tag/export.go +++ b/pkg/tag/export.go @@ -8,6 +8,7 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/json" "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil" "github.com/stashapp/stash/pkg/utils" ) @@ -55,6 +56,28 @@ func ToJSON(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag) return &newTagJSON, nil } +// GetDependentTagIDs returns a slice of unique tag IDs that this tag references. +func GetDependentTagIDs(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag) ([]int, error) { + var ret []int + + parents, err := reader.FindByChildTagID(ctx, tag.ID) + if err != nil { + return nil, fmt.Errorf("error getting parents: %v", err) + } + + for _, tt := range parents { + toAdd, err := GetDependentTagIDs(ctx, reader, tt) + if err != nil { + return nil, fmt.Errorf("error getting dependent tag IDs: %v", err) + } + + ret = sliceutil.AppendUniques(ret, toAdd) + ret = sliceutil.AppendUnique(ret, tt.ID) + } + + return ret, nil +} + func GetIDs(tags []*models.Tag) []int { var results []int for _, tag := range tags {