Import/export bug fixes (#5780)

* Include parent tags in export if including dependencies
* Handle uniqueness when sanitising filenames

internal/manager/task_export.go

@@ -1042,23 +1042,43 @@ func (t *ExportTask) ExportTags(ctx context.Context, workers int) {
 	logger.Info("[tags] exporting")
 	startTime := time.Now()
 
-	jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
+	tagIdx := 0
-
+	if t.tags != nil {
-	for w := 0; w < workers; w++ { // create export Tag workers
+		tagIdx = len(t.tags.IDs)
-		tagsWg.Add(1)
-		go t.exportTag(ctx, &tagsWg, jobCh)
 	}
 
-	for i, tag := range tags {
+	for {
-		index := i + 1
+		jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
-		logger.Progressf("[tags] %d of %d", index, len(tags))
 
-		jobCh <- tag // feed workers
+		for w := 0; w < workers; w++ { // create export Tag workers
+			tagsWg.Add(1)
+			go t.exportTag(ctx, &tagsWg, jobCh)
+		}
+
+		for i, tag := range tags {
+			index := i + 1 + tagIdx
+			logger.Progressf("[tags] %d of %d", index, len(tags)+tagIdx)
+
+			jobCh <- tag // feed workers
+		}
+
+		close(jobCh)
+		tagsWg.Wait()
+
+		// if more tags were added, we need to export those too
+		if t.tags == nil || len(t.tags.IDs) == tagIdx {
+			break
+		}
+
+		newTags, err := reader.FindMany(ctx, t.tags.IDs[tagIdx:])
+		if err != nil {
+			logger.Errorf("[tags] failed to fetch tags: %v", err)
+		}
+
+		tags = newTags
+		tagIdx = len(t.tags.IDs)
 	}
 
-	close(jobCh)
-	tagsWg.Wait()
-
 	logger.Infof("[tags] export complete in %s. %d workers used.", time.Since(startTime), workers)
 }
 
@@ -1075,6 +1095,15 @@ func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan
 			continue
 		}
 
+		if t.includeDependencies {
+			tagIDs, err := tag.GetDependentTagIDs(ctx, tagReader, thisTag)
+			if err != nil {
+				logger.Errorf("[tags] <%s> error getting dependent tags: %v", thisTag.Name, err)
+				continue
+			}
+			t.tags.IDs = sliceutil.AppendUniques(t.tags.IDs, tagIDs)
+		}
+
 		fn := newTagJSON.Filename()
 
 		if err := t.json.saveTag(fn, newTagJSON); err != nil {

pkg/fsutil/file.go

@@ -1,6 +1,8 @@
 package fsutil
 
 import (
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -151,7 +153,12 @@ var (
 )
 
 // SanitiseBasename returns a file basename removing any characters that are illegal or problematic to use in the filesystem.
+// It appends a short hash of the original string to ensure uniqueness.
 func SanitiseBasename(v string) string {
+	// Generate a short hash for uniqueness
+	hash := sha1.Sum([]byte(v))
+	shortHash := hex.EncodeToString(hash[:4]) // Use the first 4 bytes of the hash
+
 	v = strings.TrimSpace(v)
 
 	// replace illegal filename characters with -
@@ -163,7 +170,7 @@ func SanitiseBasename(v string) string {
 	// remove multiple hyphens
 	v = multiHyphenRE.ReplaceAllString(v, "-")
 
-	return strings.TrimSpace(v)
+	return strings.TrimSpace(v) + "-" + shortHash
 }
 
 // GetExeName returns the name of the given executable for the current platform.

pkg/fsutil/file_test.go

@@ -8,13 +8,13 @@ func TestSanitiseBasename(t *testing.T) {
 		v    string
 		want string
 	}{
-		{"basic", "basic", "basic"},
+		{"basic", "basic", "basic-61a7508e"},
-		{"spaces", `spaced name`, "spaced-name"},
+		{"spaces", `spaced name`, "spaced-name-b297cf60"},
-		{"leading/trailing spaces", `  spaced name  `, "spaced-name"},
+		{"leading/trailing spaces", `  spaced name  `, "spaced-name-175433e9"},
-		{"hyphen name", `hyphened-name`, "hyphened-name"},
+		{"hyphen name", `hyphened-name`, "hyphened-name-789c55f2"},
-		{"multi-hyphen", `hyphened--name`, "hyphened-name"},
+		{"multi-hyphen", `hyphened--name`, "hyphened-name-2da2a58f"},
-		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g"},
+		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g-ffca6fb0"},
-		{"removed characters", `foo!!bar@@and, more`, "foobarand-more"},
+		{"removed characters", `foo!!bar@@and, more`, "foobarand-more-7cee02ab"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/tag/export.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/json"
 	"github.com/stashapp/stash/pkg/models/jsonschema"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -55,6 +56,28 @@ func ToJSON(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag)
 	return &newTagJSON, nil
 }
 
+// GetDependentTagIDs returns a slice of unique tag IDs that this tag references.
+func GetDependentTagIDs(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag) ([]int, error) {
+	var ret []int
+
+	parents, err := reader.FindByChildTagID(ctx, tag.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting parents: %v", err)
+	}
+
+	for _, tt := range parents {
+		toAdd, err := GetDependentTagIDs(ctx, reader, tt)
+		if err != nil {
+			return nil, fmt.Errorf("error getting dependent tag IDs: %v", err)
+		}
+
+		ret = sliceutil.AppendUniques(ret, toAdd)
+		ret = sliceutil.AppendUnique(ret, tt.ID)
+	}
+
+	return ret, nil
+}
+
 func GetIDs(tags []*models.Tag) []int {
 	var results []int
 	for _, tag := range tags {