diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 9b502c9e0..6765167a2 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -81,6 +81,43 @@ fragment ConfigScrapingData on ConfigScrapingResult { excludeTagPatterns } +fragment IdentifyFieldOptionsData on IdentifyFieldOptions { + field + strategy + createMissing +} + +fragment IdentifyMetadataOptionsData on IdentifyMetadataOptions { + fieldOptions { + ...IdentifyFieldOptionsData + } + setCoverImage + setOrganized + includeMalePerformers +} + +fragment ScraperSourceData on ScraperSource { + stash_box_index + stash_box_endpoint + scraper_id +} + +fragment ConfigDefaultSettingsData on ConfigDefaultSettingsResult { + identify { + sources { + source { + ...ScraperSourceData + } + options { + ...IdentifyMetadataOptionsData + } + } + options { + ...IdentifyMetadataOptionsData + } + } +} + fragment ConfigData on ConfigResult { general { ...ConfigGeneralData @@ -94,4 +131,7 @@ fragment ConfigData on ConfigResult { scraping { ...ConfigScrapingData } + defaults { + ...ConfigDefaultSettingsData + } } diff --git a/graphql/documents/mutations/config.graphql b/graphql/documents/mutations/config.graphql index 149d9bf28..fff7dbeca 100644 --- a/graphql/documents/mutations/config.graphql +++ b/graphql/documents/mutations/config.graphql @@ -30,6 +30,12 @@ mutation ConfigureScraping($input: ConfigScrapingInput!) { } } +mutation ConfigureDefaults($input: ConfigDefaultSettingsInput!) { + configureDefaults(input: $input) { + ...ConfigDefaultSettingsData + } +} + mutation GenerateAPIKey($input: GenerateAPIKeyInput!) { generateAPIKey(input: $input) } diff --git a/graphql/documents/mutations/metadata.graphql b/graphql/documents/mutations/metadata.graphql index 710a9aac9..068665d9f 100644 --- a/graphql/documents/mutations/metadata.graphql +++ b/graphql/documents/mutations/metadata.graphql @@ -26,6 +26,10 @@ mutation MetadataAutoTag($input: AutoTagMetadataInput!) { metadataAutoTag(input: $input) } +mutation MetadataIdentify($input: IdentifyMetadataInput!) { + metadataIdentify(input: $input) +} + mutation MetadataClean($input: CleanMetadataInput!) { metadataClean(input: $input) } diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 774876e7a..e6f435e98 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -237,6 +237,7 @@ type Mutation { configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult! configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult! configureScraping(input: ConfigScrapingInput!): ConfigScrapingResult! + configureDefaults(input: ConfigDefaultSettingsInput!): ConfigDefaultSettingsResult! """Generate and set (or clear) API key""" generateAPIKey(input: GenerateAPIKeyInput!): String! @@ -259,6 +260,8 @@ type Mutation { metadataAutoTag(input: AutoTagMetadataInput!): ID! """Clean metadata. Returns the job ID""" metadataClean(input: CleanMetadataInput!): ID! + """Identifies scenes using scrapers. Returns the job ID""" + metadataIdentify(input: IdentifyMetadataInput!): ID! """Migrate generated files for the current hash naming""" migrateHashNaming: ID! diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 1f65a5dc0..3ce410d60 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -302,12 +302,21 @@ type ConfigScrapingResult { excludeTagPatterns: [String!]! } +type ConfigDefaultSettingsResult { + identify: IdentifyMetadataTaskOptions +} + +input ConfigDefaultSettingsInput { + identify: IdentifyMetadataInput +} + """All configuration settings""" type ConfigResult { general: ConfigGeneralResult! interface: ConfigInterfaceResult! dlna: ConfigDLNAResult! scraping: ConfigScrapingResult! + defaults: ConfigDefaultSettingsResult! } """Directory structure of a path""" diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index cfc366ccc..bb2f5643f 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -67,6 +67,88 @@ input AutoTagMetadataInput { tags: [String!] } +enum IdentifyFieldStrategy { + """Never sets the field value""" + IGNORE + """ + For multi-value fields, merge with existing. + For single-value fields, ignore if already set + """ + MERGE + """Always replaces the value if a value is found. + For multi-value fields, any existing values are removed and replaced with the + scraped values. + """ + OVERWRITE +} + +input IdentifyFieldOptionsInput { + field: String! + strategy: IdentifyFieldStrategy! + """creates missing objects if needed - only applicable for performers, tags and studios""" + createMissing: Boolean +} + +input IdentifyMetadataOptionsInput { + """any fields missing from here are defaulted to MERGE and createMissing false""" + fieldOptions: [IdentifyFieldOptionsInput!] + """defaults to true if not provided""" + setCoverImage: Boolean + setOrganized: Boolean + """defaults to true if not provided""" + includeMalePerformers: Boolean +} + +input IdentifySourceInput { + source: ScraperSourceInput! + """Options defined for a source override the defaults""" + options: IdentifyMetadataOptionsInput +} + +input IdentifyMetadataInput { + """An ordered list of sources to identify items with. Only the first source that finds a match is used.""" + sources: [IdentifySourceInput!]! + """Options defined here override the configured defaults""" + options: IdentifyMetadataOptionsInput + + """scene ids to identify""" + sceneIDs: [ID!] + + """paths of scenes to identify - ignored if scene ids are set""" + paths: [String!] +} + +# types for default options +type IdentifyFieldOptions { + field: String! + strategy: IdentifyFieldStrategy! + """creates missing objects if needed - only applicable for performers, tags and studios""" + createMissing: Boolean +} + +type IdentifyMetadataOptions { + """any fields missing from here are defaulted to MERGE and createMissing false""" + fieldOptions: [IdentifyFieldOptions!] + """defaults to true if not provided""" + setCoverImage: Boolean + setOrganized: Boolean + """defaults to true if not provided""" + includeMalePerformers: Boolean +} + +type IdentifySource { + source: ScraperSource! + """Options defined for a source override the defaults""" + options: IdentifyMetadataOptions +} + +type IdentifyMetadataTaskOptions { + """An ordered list of sources to identify items with. Only the first source that finds a match is used.""" + sources: [IdentifySource!]! + """Options defined here override the configured defaults""" + options: IdentifyMetadataOptions +} + input ExportObjectTypeInput { ids: [String!] all: Boolean diff --git a/graphql/schema/types/scraper.graphql b/graphql/schema/types/scraper.graphql index 81925a6dc..ebe338e1c 100644 --- a/graphql/schema/types/scraper.graphql +++ b/graphql/schema/types/scraper.graphql @@ -96,7 +96,18 @@ input ScrapedGalleryInput { input ScraperSourceInput { """Index of the configured stash-box instance to use. Should be unset if scraper_id is set""" - stash_box_index: Int + stash_box_index: Int @deprecated(reason: "use stash_box_endpoint") + """Stash-box endpoint""" + stash_box_endpoint: String + """Scraper ID to scrape with. Should be unset if stash_box_index is set""" + scraper_id: ID +} + +type ScraperSource { + """Index of the configured stash-box instance to use. Should be unset if scraper_id is set""" + stash_box_index: Int @deprecated(reason: "use stash_box_endpoint") + """Stash-box endpoint""" + stash_box_endpoint: String """Scraper ID to scrape with. Should be unset if stash_box_index is set""" scraper_id: ID } diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index aa9ecf710..cd9e010f2 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -352,6 +352,20 @@ func (r *mutationResolver) ConfigureScraping(ctx context.Context, input models.C return makeConfigScrapingResult(), nil } +func (r *mutationResolver) ConfigureDefaults(ctx context.Context, input models.ConfigDefaultSettingsInput) (*models.ConfigDefaultSettingsResult, error) { + c := config.GetInstance() + + if input.Identify != nil { + c.Set(config.DefaultIdentifySettings, input.Identify) + } + + if err := c.Write(); err != nil { + return makeConfigDefaultsResult(), err + } + + return makeConfigDefaultsResult(), nil +} + func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) { c := config.GetInstance() diff --git a/pkg/api/resolver_mutation_metadata.go b/pkg/api/resolver_mutation_metadata.go index aee27005e..ff49347a3 100644 --- a/pkg/api/resolver_mutation_metadata.go +++ b/pkg/api/resolver_mutation_metadata.go @@ -90,6 +90,13 @@ func (r *mutationResolver) MetadataAutoTag(ctx context.Context, input models.Aut return strconv.Itoa(jobID), nil } +func (r *mutationResolver) MetadataIdentify(ctx context.Context, input models.IdentifyMetadataInput) (string, error) { + t := manager.CreateIdentifyJob(input) + jobID := manager.GetInstance().JobManager.Add(ctx, "Identifying...", t) + + return strconv.Itoa(jobID), nil +} + func (r *mutationResolver) MetadataClean(ctx context.Context, input models.CleanMetadataInput) (string, error) { jobID := manager.GetInstance().Clean(ctx, input) return strconv.Itoa(jobID), nil diff --git a/pkg/api/resolver_mutation_scene.go b/pkg/api/resolver_mutation_scene.go index 82339df3c..411867fc2 100644 --- a/pkg/api/resolver_mutation_scene.go +++ b/pkg/api/resolver_mutation_scene.go @@ -11,6 +11,7 @@ import ( "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/utils" ) @@ -119,7 +120,7 @@ func (r *mutationResolver) sceneUpdate(ctx context.Context, input models.SceneUp } qb := repo.Scene() - scene, err := qb.Update(updatedScene) + s, err := qb.Update(updatedScene) if err != nil { return nil, err } @@ -169,13 +170,13 @@ func (r *mutationResolver) sceneUpdate(ctx context.Context, input models.SceneUp // only update the cover image if provided and everything else was successful if coverImageData != nil { - err = manager.SetSceneScreenshot(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), coverImageData) + err = scene.SetScreenshot(manager.GetInstance().Paths, s.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), coverImageData) if err != nil { return nil, err } } - return scene, nil + return s, nil } func (r *mutationResolver) updateScenePerformers(qb models.SceneReaderWriter, sceneID int, performerIDs []string) error { diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 4401a68a6..2c28476cc 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -43,6 +43,7 @@ func makeConfigResult() *models.ConfigResult { Interface: makeConfigInterfaceResult(), Dlna: makeConfigDLNAResult(), Scraping: makeConfigScrapingResult(), + Defaults: makeConfigDefaultsResult(), } } @@ -159,3 +160,11 @@ func makeConfigScrapingResult() *models.ConfigScrapingResult { ExcludeTagPatterns: config.GetScraperExcludeTagPatterns(), } } + +func makeConfigDefaultsResult() *models.ConfigDefaultSettingsResult { + config := config.GetInstance() + + return &models.ConfigDefaultSettingsResult{ + Identify: config.GetDefaultIdentifySettings(), + } +} diff --git a/pkg/identify/identify.go b/pkg/identify/identify.go new file mode 100644 index 000000000..da3be4ac5 --- /dev/null +++ b/pkg/identify/identify.go @@ -0,0 +1,264 @@ +package identify + +import ( + "context" + "database/sql" + "fmt" + + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/utils" +) + +type SceneScraper interface { + ScrapeScene(sceneID int) (*models.ScrapedScene, error) +} + +type SceneUpdatePostHookExecutor interface { + ExecuteSceneUpdatePostHooks(ctx context.Context, input models.SceneUpdateInput, inputFields []string) +} + +type ScraperSource struct { + Name string + Options *models.IdentifyMetadataOptionsInput + Scraper SceneScraper + RemoteSite string +} + +type SceneIdentifier struct { + DefaultOptions *models.IdentifyMetadataOptionsInput + Sources []ScraperSource + ScreenshotSetter scene.ScreenshotSetter + SceneUpdatePostHookExecutor SceneUpdatePostHookExecutor +} + +func (t *SceneIdentifier) Identify(ctx context.Context, repo models.Repository, scene *models.Scene) error { + result, err := t.scrapeScene(scene) + if err != nil { + return err + } + + if result == nil { + logger.Infof("Unable to identify %s", scene.Path) + return nil + } + + // results were found, modify the scene + if err := t.modifyScene(ctx, repo, scene, result); err != nil { + return fmt.Errorf("error modifying scene: %v", err) + } + + return nil +} + +type scrapeResult struct { + result *models.ScrapedScene + source ScraperSource +} + +func (t *SceneIdentifier) scrapeScene(scene *models.Scene) (*scrapeResult, error) { + // iterate through the input sources + for _, source := range t.Sources { + // scrape using the source + scraped, err := source.Scraper.ScrapeScene(scene.ID) + if err != nil { + return nil, fmt.Errorf("error scraping from %v: %v", source.Scraper, err) + } + + // if results were found then return + if scraped != nil { + return &scrapeResult{ + result: scraped, + source: source, + }, nil + } + } + + return nil, nil +} + +func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, result *scrapeResult, repo models.Repository) (*scene.UpdateSet, error) { + ret := &scene.UpdateSet{ + ID: s.ID, + } + + options := []models.IdentifyMetadataOptionsInput{} + if result.source.Options != nil { + options = append(options, *result.source.Options) + } + if t.DefaultOptions != nil { + options = append(options, *t.DefaultOptions) + } + + fieldOptions := getFieldOptions(options) + + setOrganized := false + for _, o := range options { + if o.SetOrganized != nil { + setOrganized = *o.SetOrganized + break + } + } + + scraped := result.result + + rel := sceneRelationships{ + repo: repo, + scene: s, + result: result, + fieldOptions: fieldOptions, + } + + ret.Partial = getScenePartial(s, scraped, fieldOptions, setOrganized) + + studioID, err := rel.studio() + if err != nil { + return nil, fmt.Errorf("error getting studio: %w", err) + } + + if studioID != nil { + ret.Partial.StudioID = &sql.NullInt64{ + Int64: *studioID, + Valid: true, + } + } + + ignoreMale := false + for _, o := range options { + if o.IncludeMalePerformers != nil { + ignoreMale = !*o.IncludeMalePerformers + break + } + } + + ret.PerformerIDs, err = rel.performers(ignoreMale) + if err != nil { + return nil, err + } + + ret.TagIDs, err = rel.tags() + if err != nil { + return nil, err + } + + ret.StashIDs, err = rel.stashIDs() + if err != nil { + return nil, err + } + + setCoverImage := false + for _, o := range options { + if o.SetCoverImage != nil { + setCoverImage = *o.SetCoverImage + break + } + } + + if setCoverImage { + ret.CoverImage, err = rel.cover(ctx) + if err != nil { + return nil, err + } + } + + return ret, nil +} + +func (t *SceneIdentifier) modifyScene(ctx context.Context, repo models.Repository, scene *models.Scene, result *scrapeResult) error { + updater, err := t.getSceneUpdater(ctx, scene, result, repo) + if err != nil { + return err + } + + // don't update anything if nothing was set + if updater.IsEmpty() { + logger.Infof("Nothing to set for %s", scene.Path) + return nil + } + + _, err = updater.Update(repo.Scene(), t.ScreenshotSetter) + if err != nil { + return fmt.Errorf("error updating scene: %w", err) + } + + // fire post-update hooks + updateInput := updater.UpdateInput() + fields := utils.NotNilFields(updateInput, "json") + t.SceneUpdatePostHookExecutor.ExecuteSceneUpdatePostHooks(ctx, updateInput, fields) + + as := "" + title := updater.Partial.Title + if title != nil { + as = fmt.Sprintf(" as %s", title.String) + } + logger.Infof("Successfully identified %s%s using %s", scene.Path, as, result.source.Name) + + return nil +} + +func getFieldOptions(options []models.IdentifyMetadataOptionsInput) map[string]*models.IdentifyFieldOptionsInput { + // prefer source-specific field strategies, then the defaults + ret := make(map[string]*models.IdentifyFieldOptionsInput) + for _, oo := range options { + for _, f := range oo.FieldOptions { + if _, found := ret[f.Field]; !found { + ret[f.Field] = f + } + } + } + + return ret +} + +func getScenePartial(scene *models.Scene, scraped *models.ScrapedScene, fieldOptions map[string]*models.IdentifyFieldOptionsInput, setOrganized bool) models.ScenePartial { + partial := models.ScenePartial{ + ID: scene.ID, + } + + if scraped.Title != nil && scene.Title.String != *scraped.Title { + if shouldSetSingleValueField(fieldOptions["title"], scene.Title.String != "") { + partial.Title = models.NullStringPtr(*scraped.Title) + } + } + if scraped.Date != nil && scene.Date.String != *scraped.Date { + if shouldSetSingleValueField(fieldOptions["date"], scene.Date.Valid) { + partial.Date = &models.SQLiteDate{ + String: *scraped.Date, + Valid: true, + } + } + } + if scraped.Details != nil && scene.Details.String != *scraped.Details { + if shouldSetSingleValueField(fieldOptions["details"], scene.Details.String != "") { + partial.Details = models.NullStringPtr(*scraped.Details) + } + } + if scraped.URL != nil && scene.URL.String != *scraped.URL { + if shouldSetSingleValueField(fieldOptions["url"], scene.URL.String != "") { + partial.URL = models.NullStringPtr(*scraped.URL) + } + } + + if setOrganized && !scene.Organized { + // just reuse the boolean since we know it's true + partial.Organized = &setOrganized + } + + return partial +} + +func shouldSetSingleValueField(strategy *models.IdentifyFieldOptionsInput, hasExistingValue bool) bool { + // if unset then default to MERGE + fs := models.IdentifyFieldStrategyMerge + + if strategy != nil && strategy.Strategy.IsValid() { + fs = strategy.Strategy + } + + if fs == models.IdentifyFieldStrategyIgnore { + return false + } + + return !hasExistingValue || fs == models.IdentifyFieldStrategyOverwrite +} diff --git a/pkg/identify/identify_test.go b/pkg/identify/identify_test.go new file mode 100644 index 000000000..a598c04bb --- /dev/null +++ b/pkg/identify/identify_test.go @@ -0,0 +1,502 @@ +package identify + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stashapp/stash/pkg/utils" + "github.com/stretchr/testify/mock" +) + +type mockSceneScraper struct { + errIDs []int + results map[int]*models.ScrapedScene +} + +func (s mockSceneScraper) ScrapeScene(sceneID int) (*models.ScrapedScene, error) { + if utils.IntInclude(s.errIDs, sceneID) { + return nil, errors.New("scrape scene error") + } + return s.results[sceneID], nil +} + +type mockHookExecutor struct { +} + +func (s mockHookExecutor) ExecuteSceneUpdatePostHooks(ctx context.Context, input models.SceneUpdateInput, inputFields []string) { +} + +func TestSceneIdentifier_Identify(t *testing.T) { + const ( + errID1 = iota + errID2 + missingID + found1ID + found2ID + errUpdateID + ) + + var scrapedTitle = "scrapedTitle" + + defaultOptions := &models.IdentifyMetadataOptionsInput{} + sources := []ScraperSource{ + { + Scraper: mockSceneScraper{ + errIDs: []int{errID1}, + results: map[int]*models.ScrapedScene{ + found1ID: { + Title: &scrapedTitle, + }, + }, + }, + }, + { + Scraper: mockSceneScraper{ + errIDs: []int{errID2}, + results: map[int]*models.ScrapedScene{ + found2ID: { + Title: &scrapedTitle, + }, + errUpdateID: { + Title: &scrapedTitle, + }, + }, + }, + }, + } + + repo := mocks.NewTransactionManager() + repo.Scene().(*mocks.SceneReaderWriter).On("Update", mock.MatchedBy(func(partial models.ScenePartial) bool { + return partial.ID != errUpdateID + })).Return(nil, nil) + repo.Scene().(*mocks.SceneReaderWriter).On("Update", mock.MatchedBy(func(partial models.ScenePartial) bool { + return partial.ID == errUpdateID + })).Return(nil, errors.New("update error")) + + tests := []struct { + name string + sceneID int + wantErr bool + }{ + { + "error scraping", + errID1, + true, + }, + { + "error scraping from second", + errID2, + true, + }, + { + "found in first scraper", + found1ID, + false, + }, + { + "found in second scraper", + found2ID, + false, + }, + { + "not found", + missingID, + false, + }, + { + "error modifying", + errUpdateID, + true, + }, + } + + identifier := SceneIdentifier{ + DefaultOptions: defaultOptions, + Sources: sources, + SceneUpdatePostHookExecutor: mockHookExecutor{}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scene := &models.Scene{ + ID: tt.sceneID, + } + if err := identifier.Identify(context.TODO(), repo, scene); (err != nil) != tt.wantErr { + t.Errorf("SceneIdentifier.Identify() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestSceneIdentifier_modifyScene(t *testing.T) { + repo := mocks.NewTransactionManager() + tr := &SceneIdentifier{} + + type args struct { + scene *models.Scene + result *scrapeResult + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + "empty update", + args{ + &models.Scene{}, + &scrapeResult{ + result: &models.ScrapedScene{}, + }, + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tr.modifyScene(context.TODO(), repo, tt.args.scene, tt.args.result); (err != nil) != tt.wantErr { + t.Errorf("SceneIdentifier.modifyScene() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_getFieldOptions(t *testing.T) { + const ( + inFirst = "inFirst" + inSecond = "inSecond" + inBoth = "inBoth" + ) + + type args struct { + options []models.IdentifyMetadataOptionsInput + } + tests := []struct { + name string + args args + want map[string]*models.IdentifyFieldOptionsInput + }{ + { + "simple", + args{ + []models.IdentifyMetadataOptionsInput{ + { + FieldOptions: []*models.IdentifyFieldOptionsInput{ + { + Field: inFirst, + Strategy: models.IdentifyFieldStrategyIgnore, + }, + { + Field: inBoth, + Strategy: models.IdentifyFieldStrategyIgnore, + }, + }, + }, + { + FieldOptions: []*models.IdentifyFieldOptionsInput{ + { + Field: inSecond, + Strategy: models.IdentifyFieldStrategyMerge, + }, + { + Field: inBoth, + Strategy: models.IdentifyFieldStrategyMerge, + }, + }, + }, + }, + }, + map[string]*models.IdentifyFieldOptionsInput{ + inFirst: { + Field: inFirst, + Strategy: models.IdentifyFieldStrategyIgnore, + }, + inSecond: { + Field: inSecond, + Strategy: models.IdentifyFieldStrategyMerge, + }, + inBoth: { + Field: inBoth, + Strategy: models.IdentifyFieldStrategyIgnore, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getFieldOptions(tt.args.options); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getFieldOptions() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getScenePartial(t *testing.T) { + var ( + originalTitle = "originalTitle" + originalDate = "originalDate" + originalDetails = "originalDetails" + originalURL = "originalURL" + ) + + var ( + scrapedTitle = "scrapedTitle" + scrapedDate = "scrapedDate" + scrapedDetails = "scrapedDetails" + scrapedURL = "scrapedURL" + ) + + originalScene := &models.Scene{ + Title: models.NullString(originalTitle), + Date: models.SQLiteDate{ + String: originalDate, + Valid: true, + }, + Details: models.NullString(originalDetails), + URL: models.NullString(originalURL), + } + + organisedScene := *originalScene + organisedScene.Organized = true + + emptyScene := &models.Scene{} + + postPartial := models.ScenePartial{ + Title: models.NullStringPtr(scrapedTitle), + Date: &models.SQLiteDate{ + String: scrapedDate, + Valid: true, + }, + Details: models.NullStringPtr(scrapedDetails), + URL: models.NullStringPtr(scrapedURL), + } + + scrapedScene := &models.ScrapedScene{ + Title: &scrapedTitle, + Date: &scrapedDate, + Details: &scrapedDetails, + URL: &scrapedURL, + } + + scrapedUnchangedScene := &models.ScrapedScene{ + Title: &originalTitle, + Date: &originalDate, + Details: &originalDetails, + URL: &originalURL, + } + + makeFieldOptions := func(input *models.IdentifyFieldOptionsInput) map[string]*models.IdentifyFieldOptionsInput { + return map[string]*models.IdentifyFieldOptionsInput{ + "title": input, + "date": input, + "details": input, + "url": input, + } + } + + overwriteAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }) + ignoreAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }) + mergeAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + }) + + setOrganised := true + + type args struct { + scene *models.Scene + scraped *models.ScrapedScene + fieldOptions map[string]*models.IdentifyFieldOptionsInput + setOrganized bool + } + tests := []struct { + name string + args args + want models.ScenePartial + }{ + { + "overwrite all", + args{ + originalScene, + scrapedScene, + overwriteAll, + false, + }, + postPartial, + }, + { + "ignore all", + args{ + originalScene, + scrapedScene, + ignoreAll, + false, + }, + models.ScenePartial{}, + }, + { + "merge (existing values)", + args{ + originalScene, + scrapedScene, + mergeAll, + false, + }, + models.ScenePartial{}, + }, + { + "merge (empty values)", + args{ + emptyScene, + scrapedScene, + mergeAll, + false, + }, + postPartial, + }, + { + "unchanged", + args{ + originalScene, + scrapedUnchangedScene, + overwriteAll, + false, + }, + models.ScenePartial{}, + }, + { + "set organized", + args{ + originalScene, + scrapedUnchangedScene, + overwriteAll, + true, + }, + models.ScenePartial{ + Organized: &setOrganised, + }, + }, + { + "set organized unchanged", + args{ + &organisedScene, + scrapedUnchangedScene, + overwriteAll, + true, + }, + models.ScenePartial{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getScenePartial() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_shouldSetSingleValueField(t *testing.T) { + const invalid = "invalid" + + type args struct { + strategy *models.IdentifyFieldOptionsInput + hasExistingValue bool + } + tests := []struct { + name string + args args + want bool + }{ + { + "ignore", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }, + false, + }, + false, + }, + { + "merge existing", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + }, + true, + }, + false, + }, + { + "merge absent", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + }, + false, + }, + true, + }, + { + "overwrite", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + true, + }, + true, + }, + { + "nil (merge) existing", + args{ + &models.IdentifyFieldOptionsInput{}, + true, + }, + false, + }, + { + "nil (merge) absent", + args{ + &models.IdentifyFieldOptionsInput{}, + false, + }, + true, + }, + { + "invalid (merge) existing", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: invalid, + }, + true, + }, + false, + }, + { + "invalid (merge) absent", + args{ + &models.IdentifyFieldOptionsInput{ + Strategy: invalid, + }, + false, + }, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shouldSetSingleValueField(tt.args.strategy, tt.args.hasExistingValue); got != tt.want { + t.Errorf("shouldSetSingleValueField() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/identify/performer.go b/pkg/identify/performer.go new file mode 100644 index 000000000..4d0855388 --- /dev/null +++ b/pkg/identify/performer.go @@ -0,0 +1,108 @@ +package identify + +import ( + "database/sql" + "fmt" + "strconv" + "time" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" +) + +func getPerformerID(endpoint string, r models.Repository, p *models.ScrapedPerformer, createMissing bool) (*int, error) { + if p.StoredID != nil { + // existing performer, just add it + performerID, err := strconv.Atoi(*p.StoredID) + if err != nil { + return nil, fmt.Errorf("error converting performer ID %s: %w", *p.StoredID, err) + } + + return &performerID, nil + } else if createMissing && p.Name != nil { // name is mandatory + return createMissingPerformer(endpoint, r, p) + } + + return nil, nil +} + +func createMissingPerformer(endpoint string, r models.Repository, p *models.ScrapedPerformer) (*int, error) { + created, err := r.Performer().Create(scrapedToPerformerInput(p)) + if err != nil { + return nil, fmt.Errorf("error creating performer: %w", err) + } + + if endpoint != "" && p.RemoteSiteID != nil { + if err := r.Performer().UpdateStashIDs(created.ID, []models.StashID{ + { + Endpoint: endpoint, + StashID: *p.RemoteSiteID, + }, + }); err != nil { + return nil, fmt.Errorf("error setting performer stash id: %w", err) + } + } + + return &created.ID, nil +} + +func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performer { + currentTime := time.Now() + ret := models.Performer{ + Name: sql.NullString{String: *performer.Name, Valid: true}, + Checksum: utils.MD5FromString(*performer.Name), + CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, + UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, + Favorite: sql.NullBool{Bool: false, Valid: true}, + } + if performer.Birthdate != nil { + ret.Birthdate = models.SQLiteDate{String: *performer.Birthdate, Valid: true} + } + if performer.DeathDate != nil { + ret.DeathDate = models.SQLiteDate{String: *performer.DeathDate, Valid: true} + } + if performer.Gender != nil { + ret.Gender = sql.NullString{String: *performer.Gender, Valid: true} + } + if performer.Ethnicity != nil { + ret.Ethnicity = sql.NullString{String: *performer.Ethnicity, Valid: true} + } + if performer.Country != nil { + ret.Country = sql.NullString{String: *performer.Country, Valid: true} + } + if performer.EyeColor != nil { + ret.EyeColor = sql.NullString{String: *performer.EyeColor, Valid: true} + } + if performer.HairColor != nil { + ret.HairColor = sql.NullString{String: *performer.HairColor, Valid: true} + } + if performer.Height != nil { + ret.Height = sql.NullString{String: *performer.Height, Valid: true} + } + if performer.Measurements != nil { + ret.Measurements = sql.NullString{String: *performer.Measurements, Valid: true} + } + if performer.FakeTits != nil { + ret.FakeTits = sql.NullString{String: *performer.FakeTits, Valid: true} + } + if performer.CareerLength != nil { + ret.CareerLength = sql.NullString{String: *performer.CareerLength, Valid: true} + } + if performer.Tattoos != nil { + ret.Tattoos = sql.NullString{String: *performer.Tattoos, Valid: true} + } + if performer.Piercings != nil { + ret.Piercings = sql.NullString{String: *performer.Piercings, Valid: true} + } + if performer.Aliases != nil { + ret.Aliases = sql.NullString{String: *performer.Aliases, Valid: true} + } + if performer.Twitter != nil { + ret.Twitter = sql.NullString{String: *performer.Twitter, Valid: true} + } + if performer.Instagram != nil { + ret.Instagram = sql.NullString{String: *performer.Instagram, Valid: true} + } + + return ret +} diff --git a/pkg/identify/performer_test.go b/pkg/identify/performer_test.go new file mode 100644 index 000000000..ebe8e49fe --- /dev/null +++ b/pkg/identify/performer_test.go @@ -0,0 +1,329 @@ +package identify + +import ( + "database/sql" + "errors" + "reflect" + "strconv" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + + "github.com/stretchr/testify/mock" +) + +func Test_getPerformerID(t *testing.T) { + const ( + emptyEndpoint = "" + endpoint = "endpoint" + ) + invalidStoredID := "invalidStoredID" + validStoredIDStr := "1" + validStoredID := 1 + name := "name" + + repo := mocks.NewTransactionManager() + repo.PerformerMock().On("Create", mock.Anything).Return(&models.Performer{ + ID: validStoredID, + }, nil) + + type args struct { + endpoint string + p *models.ScrapedPerformer + createMissing bool + } + tests := []struct { + name string + args args + want *int + wantErr bool + }{ + { + "no performer", + args{ + emptyEndpoint, + &models.ScrapedPerformer{}, + false, + }, + nil, + false, + }, + { + "invalid stored id", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + StoredID: &invalidStoredID, + }, + false, + }, + nil, + true, + }, + { + "valid stored id", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + StoredID: &validStoredIDStr, + }, + false, + }, + &validStoredID, + false, + }, + { + "nil stored not creating", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + Name: &name, + }, + false, + }, + nil, + false, + }, + { + "nil name creating", + args{ + emptyEndpoint, + &models.ScrapedPerformer{}, + true, + }, + nil, + false, + }, + { + "valid name creating", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + Name: &name, + }, + true, + }, + &validStoredID, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getPerformerID(tt.args.endpoint, repo, tt.args.p, tt.args.createMissing) + if (err != nil) != tt.wantErr { + t.Errorf("getPerformerID() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getPerformerID() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createMissingPerformer(t *testing.T) { + emptyEndpoint := "" + validEndpoint := "validEndpoint" + invalidEndpoint := "invalidEndpoint" + remoteSiteID := "remoteSiteID" + validName := "validName" + invalidName := "invalidName" + performerID := 1 + + repo := mocks.NewTransactionManager() + repo.PerformerMock().On("Create", mock.MatchedBy(func(p models.Performer) bool { + return p.Name.String == validName + })).Return(&models.Performer{ + ID: performerID, + }, nil) + repo.PerformerMock().On("Create", mock.MatchedBy(func(p models.Performer) bool { + return p.Name.String == invalidName + })).Return(nil, errors.New("error creating performer")) + + repo.PerformerMock().On("UpdateStashIDs", performerID, []models.StashID{ + { + Endpoint: invalidEndpoint, + StashID: remoteSiteID, + }, + }).Return(errors.New("error updating stash ids")) + repo.PerformerMock().On("UpdateStashIDs", performerID, []models.StashID{ + { + Endpoint: validEndpoint, + StashID: remoteSiteID, + }, + }).Return(nil) + + type args struct { + endpoint string + p *models.ScrapedPerformer + } + tests := []struct { + name string + args args + want *int + wantErr bool + }{ + { + "simple", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + Name: &validName, + }, + }, + &performerID, + false, + }, + { + "error creating", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + Name: &invalidName, + }, + }, + nil, + true, + }, + { + "valid stash id", + args{ + validEndpoint, + &models.ScrapedPerformer{ + Name: &validName, + RemoteSiteID: &remoteSiteID, + }, + }, + &performerID, + false, + }, + { + "invalid stash id", + args{ + invalidEndpoint, + &models.ScrapedPerformer{ + Name: &validName, + RemoteSiteID: &remoteSiteID, + }, + }, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := createMissingPerformer(tt.args.endpoint, repo, tt.args.p) + if (err != nil) != tt.wantErr { + t.Errorf("createMissingPerformer() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("createMissingPerformer() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_scrapedToPerformerInput(t *testing.T) { + name := "name" + md5 := "b068931cc450442b63f5b3d276ea4297" + + var stringValues []string + for i := 0; i < 16; i++ { + stringValues = append(stringValues, strconv.Itoa(i)) + } + + upTo := 0 + nextVal := func() *string { + ret := stringValues[upTo] + upTo = (upTo + 1) % len(stringValues) + return &ret + } + + tests := []struct { + name string + performer *models.ScrapedPerformer + want models.Performer + }{ + { + "set all", + &models.ScrapedPerformer{ + Name: &name, + Birthdate: nextVal(), + DeathDate: nextVal(), + Gender: nextVal(), + Ethnicity: nextVal(), + Country: nextVal(), + EyeColor: nextVal(), + HairColor: nextVal(), + Height: nextVal(), + Measurements: nextVal(), + FakeTits: nextVal(), + CareerLength: nextVal(), + Tattoos: nextVal(), + Piercings: nextVal(), + Aliases: nextVal(), + Twitter: nextVal(), + Instagram: nextVal(), + }, + models.Performer{ + Name: models.NullString(name), + Checksum: md5, + Favorite: sql.NullBool{ + Bool: false, + Valid: true, + }, + Birthdate: models.SQLiteDate{ + String: *nextVal(), + Valid: true, + }, + DeathDate: models.SQLiteDate{ + String: *nextVal(), + Valid: true, + }, + Gender: models.NullString(*nextVal()), + Ethnicity: models.NullString(*nextVal()), + Country: models.NullString(*nextVal()), + EyeColor: models.NullString(*nextVal()), + HairColor: models.NullString(*nextVal()), + Height: models.NullString(*nextVal()), + Measurements: models.NullString(*nextVal()), + FakeTits: models.NullString(*nextVal()), + CareerLength: models.NullString(*nextVal()), + Tattoos: models.NullString(*nextVal()), + Piercings: models.NullString(*nextVal()), + Aliases: models.NullString(*nextVal()), + Twitter: models.NullString(*nextVal()), + Instagram: models.NullString(*nextVal()), + }, + }, + { + "set none", + &models.ScrapedPerformer{ + Name: &name, + }, + models.Performer{ + Name: models.NullString(name), + Checksum: md5, + Favorite: sql.NullBool{ + Bool: false, + Valid: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := scrapedToPerformerInput(tt.performer) + + // clear created/updated dates + got.CreatedAt = models.SQLiteTimestamp{} + got.UpdatedAt = got.CreatedAt + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("scrapedToPerformerInput() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/identify/scene.go b/pkg/identify/scene.go new file mode 100644 index 000000000..a8b5d4cff --- /dev/null +++ b/pkg/identify/scene.go @@ -0,0 +1,251 @@ +package identify + +import ( + "bytes" + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" +) + +type sceneRelationships struct { + repo models.Repository + scene *models.Scene + result *scrapeResult + fieldOptions map[string]*models.IdentifyFieldOptionsInput +} + +func (g sceneRelationships) studio() (*int64, error) { + existingID := g.scene.StudioID + fieldStrategy := g.fieldOptions["studio"] + createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing) + + scraped := g.result.result.Studio + endpoint := g.result.source.RemoteSite + + if scraped == nil || !shouldSetSingleValueField(fieldStrategy, existingID.Valid) { + return nil, nil + } + + if scraped.StoredID != nil { + // existing studio, just set it + studioID, err := strconv.ParseInt(*scraped.StoredID, 10, 64) + if err != nil { + return nil, fmt.Errorf("error converting studio ID %s: %w", *scraped.StoredID, err) + } + + // only return value if different to current + if existingID.Int64 != studioID { + return &studioID, nil + } + } else if createMissing { + return createMissingStudio(endpoint, g.repo, scraped) + } + + return nil, nil +} + +func (g sceneRelationships) performers(ignoreMale bool) ([]int, error) { + fieldStrategy := g.fieldOptions["performers"] + scraped := g.result.result.Performers + + // just check if ignored + if len(scraped) == 0 || !shouldSetSingleValueField(fieldStrategy, false) { + return nil, nil + } + + createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing) + strategy := models.IdentifyFieldStrategyMerge + if fieldStrategy != nil { + strategy = fieldStrategy.Strategy + } + + repo := g.repo + endpoint := g.result.source.RemoteSite + + var performerIDs []int + originalPerformerIDs, err := repo.Scene().GetPerformerIDs(g.scene.ID) + if err != nil { + return nil, fmt.Errorf("error getting scene performers: %w", err) + } + + if strategy == models.IdentifyFieldStrategyMerge { + // add to existing + performerIDs = originalPerformerIDs + } + + for _, p := range scraped { + if ignoreMale && p.Gender != nil && strings.EqualFold(*p.Gender, models.GenderEnumMale.String()) { + continue + } + + performerID, err := getPerformerID(endpoint, repo, p, createMissing) + if err != nil { + return nil, err + } + + if performerID != nil { + performerIDs = utils.IntAppendUnique(performerIDs, *performerID) + } + } + + // don't return if nothing was added + if utils.SliceSame(originalPerformerIDs, performerIDs) { + return nil, nil + } + + return performerIDs, nil +} + +func (g sceneRelationships) tags() ([]int, error) { + fieldStrategy := g.fieldOptions["tags"] + scraped := g.result.result.Tags + target := g.scene + r := g.repo + + // just check if ignored + if len(scraped) == 0 || !shouldSetSingleValueField(fieldStrategy, false) { + return nil, nil + } + + createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing) + strategy := models.IdentifyFieldStrategyMerge + if fieldStrategy != nil { + strategy = fieldStrategy.Strategy + } + + var tagIDs []int + originalTagIDs, err := r.Scene().GetTagIDs(target.ID) + if err != nil { + return nil, fmt.Errorf("error getting scene tags: %w", err) + } + + if strategy == models.IdentifyFieldStrategyMerge { + // add to existing + tagIDs = originalTagIDs + } + + for _, t := range scraped { + if t.StoredID != nil { + // existing tag, just add it + tagID, err := strconv.ParseInt(*t.StoredID, 10, 64) + if err != nil { + return nil, fmt.Errorf("error converting tag ID %s: %w", *t.StoredID, err) + } + + tagIDs = utils.IntAppendUnique(tagIDs, int(tagID)) + } else if createMissing { + now := time.Now() + created, err := r.Tag().Create(models.Tag{ + Name: t.Name, + CreatedAt: models.SQLiteTimestamp{Timestamp: now}, + UpdatedAt: models.SQLiteTimestamp{Timestamp: now}, + }) + if err != nil { + return nil, fmt.Errorf("error creating tag: %w", err) + } + + tagIDs = append(tagIDs, created.ID) + } + } + + // don't return if nothing was added + if utils.SliceSame(originalTagIDs, tagIDs) { + return nil, nil + } + + return tagIDs, nil +} + +func (g sceneRelationships) stashIDs() ([]models.StashID, error) { + remoteSiteID := g.result.result.RemoteSiteID + fieldStrategy := g.fieldOptions["stash_ids"] + target := g.scene + r := g.repo + + endpoint := g.result.source.RemoteSite + + // just check if ignored + if remoteSiteID == nil || endpoint == "" || !shouldSetSingleValueField(fieldStrategy, false) { + return nil, nil + } + + strategy := models.IdentifyFieldStrategyMerge + if fieldStrategy != nil { + strategy = fieldStrategy.Strategy + } + + var originalStashIDs []models.StashID + var stashIDs []models.StashID + stashIDPtrs, err := r.Scene().GetStashIDs(target.ID) + if err != nil { + return nil, fmt.Errorf("error getting scene tag: %w", err) + } + + // convert existing to non-pointer types + for _, stashID := range stashIDPtrs { + originalStashIDs = append(originalStashIDs, *stashID) + } + + if strategy == models.IdentifyFieldStrategyMerge { + // add to existing + stashIDs = originalStashIDs + } + + for i, stashID := range stashIDs { + if endpoint == stashID.Endpoint { + // if stashID is the same, then don't set + if stashID.StashID == *remoteSiteID { + return nil, nil + } + + // replace the stash id and return + stashID.StashID = *remoteSiteID + stashIDs[i] = stashID + return stashIDs, nil + } + } + + // not found, create new entry + stashIDs = append(stashIDs, models.StashID{ + StashID: *remoteSiteID, + Endpoint: endpoint, + }) + + if utils.SliceSame(originalStashIDs, stashIDs) { + return nil, nil + } + + return stashIDs, nil +} + +func (g sceneRelationships) cover(ctx context.Context) ([]byte, error) { + scraped := g.result.result.Image + r := g.repo + + if scraped == nil { + return nil, nil + } + + // always overwrite if present + existingCover, err := r.Scene().GetCover(g.scene.ID) + if err != nil { + return nil, fmt.Errorf("error getting scene cover: %w", err) + } + + data, err := utils.ProcessImageInput(ctx, *scraped) + if err != nil { + return nil, fmt.Errorf("error processing image input: %w", err) + } + + // only return if different + if !bytes.Equal(existingCover, data) { + return data, nil + } + + return nil, nil +} diff --git a/pkg/identify/scene_test.go b/pkg/identify/scene_test.go new file mode 100644 index 000000000..f0ba7da17 --- /dev/null +++ b/pkg/identify/scene_test.go @@ -0,0 +1,782 @@ +package identify + +import ( + "context" + "errors" + "reflect" + "strconv" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stashapp/stash/pkg/utils" + "github.com/stretchr/testify/mock" +) + +func Test_sceneRelationships_studio(t *testing.T) { + validStoredID := "1" + var validStoredIDInt int64 = 1 + invalidStoredID := "invalidStoredID" + createMissing := true + + defaultOptions := &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + } + + repo := mocks.NewTransactionManager() + repo.StudioMock().On("Create", mock.Anything).Return(&models.Studio{ + ID: int(validStoredIDInt), + }, nil) + + tr := sceneRelationships{ + repo: repo, + fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput), + } + + tests := []struct { + name string + scene *models.Scene + fieldOptions *models.IdentifyFieldOptionsInput + result *models.ScrapedStudio + want *int64 + wantErr bool + }{ + { + "nil studio", + &models.Scene{}, + defaultOptions, + nil, + nil, + false, + }, + { + "ignore", + &models.Scene{}, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }, + &models.ScrapedStudio{ + StoredID: &validStoredID, + }, + nil, + false, + }, + { + "invalid stored id", + &models.Scene{}, + defaultOptions, + &models.ScrapedStudio{ + StoredID: &invalidStoredID, + }, + nil, + true, + }, + { + "same stored id", + &models.Scene{ + StudioID: models.NullInt64(validStoredIDInt), + }, + defaultOptions, + &models.ScrapedStudio{ + StoredID: &validStoredID, + }, + nil, + false, + }, + { + "different stored id", + &models.Scene{}, + defaultOptions, + &models.ScrapedStudio{ + StoredID: &validStoredID, + }, + &validStoredIDInt, + false, + }, + { + "no create missing", + &models.Scene{}, + defaultOptions, + &models.ScrapedStudio{}, + nil, + false, + }, + { + "create missing", + &models.Scene{}, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + CreateMissing: &createMissing, + }, + &models.ScrapedStudio{}, + &validStoredIDInt, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr.scene = tt.scene + tr.fieldOptions["studio"] = tt.fieldOptions + tr.result = &scrapeResult{ + result: &models.ScrapedScene{ + Studio: tt.result, + }, + } + + got, err := tr.studio() + if (err != nil) != tt.wantErr { + t.Errorf("sceneRelationships.studio() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sceneRelationships.studio() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sceneRelationships_performers(t *testing.T) { + const ( + sceneID = iota + sceneWithPerformerID + errSceneID + existingPerformerID + validStoredIDInt + ) + validStoredID := strconv.Itoa(validStoredIDInt) + invalidStoredID := "invalidStoredID" + createMissing := true + existingPerformerStr := strconv.Itoa(existingPerformerID) + validName := "validName" + female := models.GenderEnumFemale.String() + male := models.GenderEnumMale.String() + + defaultOptions := &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + } + + repo := mocks.NewTransactionManager() + repo.SceneMock().On("GetPerformerIDs", sceneID).Return(nil, nil) + repo.SceneMock().On("GetPerformerIDs", sceneWithPerformerID).Return([]int{existingPerformerID}, nil) + repo.SceneMock().On("GetPerformerIDs", errSceneID).Return(nil, errors.New("error getting IDs")) + + tr := sceneRelationships{ + repo: repo, + fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput), + } + + tests := []struct { + name string + sceneID int + fieldOptions *models.IdentifyFieldOptionsInput + scraped []*models.ScrapedPerformer + ignoreMale bool + want []int + wantErr bool + }{ + { + "ignore", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }, + []*models.ScrapedPerformer{ + { + StoredID: &validStoredID, + }, + }, + false, + nil, + false, + }, + { + "none", + sceneID, + defaultOptions, + []*models.ScrapedPerformer{}, + false, + nil, + false, + }, + { + "error getting ids", + errSceneID, + defaultOptions, + []*models.ScrapedPerformer{ + {}, + }, + false, + nil, + true, + }, + { + "merge existing", + sceneWithPerformerID, + defaultOptions, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &existingPerformerStr, + }, + }, + false, + nil, + false, + }, + { + "merge add", + sceneWithPerformerID, + defaultOptions, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &validStoredID, + }, + }, + false, + []int{existingPerformerID, validStoredIDInt}, + false, + }, + { + "ignore male", + sceneID, + defaultOptions, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &validStoredID, + Gender: &male, + }, + }, + true, + nil, + false, + }, + { + "overwrite", + sceneWithPerformerID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &validStoredID, + }, + }, + false, + []int{validStoredIDInt}, + false, + }, + { + "ignore male (not male)", + sceneWithPerformerID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &validStoredID, + Gender: &female, + }, + }, + true, + []int{validStoredIDInt}, + false, + }, + { + "error getting tag ID", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + CreateMissing: &createMissing, + }, + []*models.ScrapedPerformer{ + { + Name: &validName, + StoredID: &invalidStoredID, + }, + }, + false, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr.scene = &models.Scene{ + ID: tt.sceneID, + } + tr.fieldOptions["performers"] = tt.fieldOptions + tr.result = &scrapeResult{ + result: &models.ScrapedScene{ + Performers: tt.scraped, + }, + } + + got, err := tr.performers(tt.ignoreMale) + if (err != nil) != tt.wantErr { + t.Errorf("sceneRelationships.performers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sceneRelationships.performers() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sceneRelationships_tags(t *testing.T) { + const ( + sceneID = iota + sceneWithTagID + errSceneID + existingID + validStoredIDInt + ) + validStoredID := strconv.Itoa(validStoredIDInt) + invalidStoredID := "invalidStoredID" + createMissing := true + existingIDStr := strconv.Itoa(existingID) + validName := "validName" + invalidName := "invalidName" + + defaultOptions := &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + } + + repo := mocks.NewTransactionManager() + repo.SceneMock().On("GetTagIDs", sceneID).Return(nil, nil) + repo.SceneMock().On("GetTagIDs", sceneWithTagID).Return([]int{existingID}, nil) + repo.SceneMock().On("GetTagIDs", errSceneID).Return(nil, errors.New("error getting IDs")) + + repo.TagMock().On("Create", mock.MatchedBy(func(p models.Tag) bool { + return p.Name == validName + })).Return(&models.Tag{ + ID: validStoredIDInt, + }, nil) + repo.TagMock().On("Create", mock.MatchedBy(func(p models.Tag) bool { + return p.Name == invalidName + })).Return(nil, errors.New("error creating tag")) + + tr := sceneRelationships{ + repo: repo, + fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput), + } + + tests := []struct { + name string + sceneID int + fieldOptions *models.IdentifyFieldOptionsInput + scraped []*models.ScrapedTag + want []int + wantErr bool + }{ + { + "ignore", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }, + []*models.ScrapedTag{ + { + StoredID: &validStoredID, + }, + }, + nil, + false, + }, + { + "none", + sceneID, + defaultOptions, + []*models.ScrapedTag{}, + nil, + false, + }, + { + "error getting ids", + errSceneID, + defaultOptions, + []*models.ScrapedTag{ + {}, + }, + nil, + true, + }, + { + "merge existing", + sceneWithTagID, + defaultOptions, + []*models.ScrapedTag{ + { + Name: validName, + StoredID: &existingIDStr, + }, + }, + nil, + false, + }, + { + "merge add", + sceneWithTagID, + defaultOptions, + []*models.ScrapedTag{ + { + Name: validName, + StoredID: &validStoredID, + }, + }, + []int{existingID, validStoredIDInt}, + false, + }, + { + "overwrite", + sceneWithTagID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + []*models.ScrapedTag{ + { + Name: validName, + StoredID: &validStoredID, + }, + }, + []int{validStoredIDInt}, + false, + }, + { + "error getting tag ID", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + []*models.ScrapedTag{ + { + Name: validName, + StoredID: &invalidStoredID, + }, + }, + nil, + true, + }, + { + "create missing", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + CreateMissing: &createMissing, + }, + []*models.ScrapedTag{ + { + Name: validName, + }, + }, + []int{validStoredIDInt}, + false, + }, + { + "error creating", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + CreateMissing: &createMissing, + }, + []*models.ScrapedTag{ + { + Name: invalidName, + }, + }, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr.scene = &models.Scene{ + ID: tt.sceneID, + } + tr.fieldOptions["tags"] = tt.fieldOptions + tr.result = &scrapeResult{ + result: &models.ScrapedScene{ + Tags: tt.scraped, + }, + } + + got, err := tr.tags() + if (err != nil) != tt.wantErr { + t.Errorf("sceneRelationships.tags() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sceneRelationships.tags() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sceneRelationships_stashIDs(t *testing.T) { + const ( + sceneID = iota + sceneWithStashID + errSceneID + existingID + validStoredIDInt + ) + existingEndpoint := "existingEndpoint" + newEndpoint := "newEndpoint" + remoteSiteID := "remoteSiteID" + newRemoteSiteID := "newRemoteSiteID" + + defaultOptions := &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyMerge, + } + + repo := mocks.NewTransactionManager() + repo.SceneMock().On("GetStashIDs", sceneID).Return(nil, nil) + repo.SceneMock().On("GetStashIDs", sceneWithStashID).Return([]*models.StashID{ + { + StashID: remoteSiteID, + Endpoint: existingEndpoint, + }, + }, nil) + repo.SceneMock().On("GetStashIDs", errSceneID).Return(nil, errors.New("error getting IDs")) + + tr := sceneRelationships{ + repo: repo, + fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput), + } + + tests := []struct { + name string + sceneID int + fieldOptions *models.IdentifyFieldOptionsInput + endpoint string + remoteSiteID *string + want []models.StashID + wantErr bool + }{ + { + "ignore", + sceneID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyIgnore, + }, + newEndpoint, + &remoteSiteID, + nil, + false, + }, + { + "no endpoint", + sceneID, + defaultOptions, + "", + &remoteSiteID, + nil, + false, + }, + { + "no site id", + sceneID, + defaultOptions, + newEndpoint, + nil, + nil, + false, + }, + { + "error getting ids", + errSceneID, + defaultOptions, + newEndpoint, + &remoteSiteID, + nil, + true, + }, + { + "merge existing", + sceneWithStashID, + defaultOptions, + existingEndpoint, + &remoteSiteID, + nil, + false, + }, + { + "merge existing new value", + sceneWithStashID, + defaultOptions, + existingEndpoint, + &newRemoteSiteID, + []models.StashID{ + { + StashID: newRemoteSiteID, + Endpoint: existingEndpoint, + }, + }, + false, + }, + { + "merge add", + sceneWithStashID, + defaultOptions, + newEndpoint, + &newRemoteSiteID, + []models.StashID{ + { + StashID: remoteSiteID, + Endpoint: existingEndpoint, + }, + { + StashID: newRemoteSiteID, + Endpoint: newEndpoint, + }, + }, + false, + }, + { + "overwrite", + sceneWithStashID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + newEndpoint, + &newRemoteSiteID, + []models.StashID{ + { + StashID: newRemoteSiteID, + Endpoint: newEndpoint, + }, + }, + false, + }, + { + "overwrite same", + sceneWithStashID, + &models.IdentifyFieldOptionsInput{ + Strategy: models.IdentifyFieldStrategyOverwrite, + }, + existingEndpoint, + &remoteSiteID, + nil, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr.scene = &models.Scene{ + ID: tt.sceneID, + } + tr.fieldOptions["stash_ids"] = tt.fieldOptions + tr.result = &scrapeResult{ + source: ScraperSource{ + RemoteSite: tt.endpoint, + }, + result: &models.ScrapedScene{ + RemoteSiteID: tt.remoteSiteID, + }, + } + + got, err := tr.stashIDs() + if (err != nil) != tt.wantErr { + t.Errorf("sceneRelationships.stashIDs() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sceneRelationships.stashIDs() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sceneRelationships_cover(t *testing.T) { + const ( + sceneID = iota + sceneWithStashID + errSceneID + existingID + validStoredIDInt + ) + existingData := []byte("existingData") + newData := []byte("newData") + const base64Prefix = "data:image/png;base64," + existingDataEncoded := base64Prefix + utils.GetBase64StringFromData(existingData) + newDataEncoded := base64Prefix + utils.GetBase64StringFromData(newData) + invalidData := newDataEncoded + "!!!" + + repo := mocks.NewTransactionManager() + repo.SceneMock().On("GetCover", sceneID).Return(existingData, nil) + repo.SceneMock().On("GetCover", errSceneID).Return(nil, errors.New("error getting cover")) + + tr := sceneRelationships{ + repo: repo, + fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput), + } + + tests := []struct { + name string + sceneID int + image *string + want []byte + wantErr bool + }{ + { + "nil image", + sceneID, + nil, + nil, + false, + }, + { + "different image", + sceneID, + &newDataEncoded, + newData, + false, + }, + { + "same image", + sceneID, + &existingDataEncoded, + nil, + false, + }, + { + "error getting scene cover", + errSceneID, + &newDataEncoded, + nil, + true, + }, + { + "invalid data", + sceneID, + &invalidData, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr.scene = &models.Scene{ + ID: tt.sceneID, + } + tr.result = &scrapeResult{ + result: &models.ScrapedScene{ + Image: tt.image, + }, + } + + got, err := tr.cover(context.TODO()) + if (err != nil) != tt.wantErr { + t.Errorf("sceneRelationships.cover() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sceneRelationships.cover() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/identify/studio.go b/pkg/identify/studio.go new file mode 100644 index 000000000..4a4c2924b --- /dev/null +++ b/pkg/identify/studio.go @@ -0,0 +1,47 @@ +package identify + +import ( + "database/sql" + "fmt" + "time" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" +) + +func createMissingStudio(endpoint string, repo models.Repository, studio *models.ScrapedStudio) (*int64, error) { + created, err := repo.Studio().Create(scrapedToStudioInput(studio)) + if err != nil { + return nil, fmt.Errorf("error creating studio: %w", err) + } + + if endpoint != "" && studio.RemoteSiteID != nil { + if err := repo.Studio().UpdateStashIDs(created.ID, []models.StashID{ + { + Endpoint: endpoint, + StashID: *studio.RemoteSiteID, + }, + }); err != nil { + return nil, fmt.Errorf("error setting studio stash id: %w", err) + } + } + + createdID := int64(created.ID) + return &createdID, nil +} + +func scrapedToStudioInput(studio *models.ScrapedStudio) models.Studio { + currentTime := time.Now() + ret := models.Studio{ + Name: sql.NullString{String: studio.Name, Valid: true}, + Checksum: utils.MD5FromString(studio.Name), + CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, + UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, + } + + if studio.URL != nil { + ret.URL = sql.NullString{String: *studio.URL, Valid: true} + } + + return ret +} diff --git a/pkg/identify/studio_test.go b/pkg/identify/studio_test.go new file mode 100644 index 000000000..2ba0b840e --- /dev/null +++ b/pkg/identify/studio_test.go @@ -0,0 +1,163 @@ +package identify + +import ( + "errors" + "reflect" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stretchr/testify/mock" +) + +func Test_createMissingStudio(t *testing.T) { + emptyEndpoint := "" + validEndpoint := "validEndpoint" + invalidEndpoint := "invalidEndpoint" + remoteSiteID := "remoteSiteID" + validName := "validName" + invalidName := "invalidName" + createdID := 1 + createdID64 := int64(createdID) + + repo := mocks.NewTransactionManager() + repo.StudioMock().On("Create", mock.MatchedBy(func(p models.Studio) bool { + return p.Name.String == validName + })).Return(&models.Studio{ + ID: createdID, + }, nil) + repo.StudioMock().On("Create", mock.MatchedBy(func(p models.Studio) bool { + return p.Name.String == invalidName + })).Return(nil, errors.New("error creating performer")) + + repo.StudioMock().On("UpdateStashIDs", createdID, []models.StashID{ + { + Endpoint: invalidEndpoint, + StashID: remoteSiteID, + }, + }).Return(errors.New("error updating stash ids")) + repo.StudioMock().On("UpdateStashIDs", createdID, []models.StashID{ + { + Endpoint: validEndpoint, + StashID: remoteSiteID, + }, + }).Return(nil) + + type args struct { + endpoint string + studio *models.ScrapedStudio + } + tests := []struct { + name string + args args + want *int64 + wantErr bool + }{ + { + "simple", + args{ + emptyEndpoint, + &models.ScrapedStudio{ + Name: validName, + }, + }, + &createdID64, + false, + }, + { + "error creating", + args{ + emptyEndpoint, + &models.ScrapedStudio{ + Name: invalidName, + }, + }, + nil, + true, + }, + { + "valid stash id", + args{ + validEndpoint, + &models.ScrapedStudio{ + Name: validName, + RemoteSiteID: &remoteSiteID, + }, + }, + &createdID64, + false, + }, + { + "invalid stash id", + args{ + invalidEndpoint, + &models.ScrapedStudio{ + Name: validName, + RemoteSiteID: &remoteSiteID, + }, + }, + nil, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := createMissingStudio(tt.args.endpoint, repo, tt.args.studio) + if (err != nil) != tt.wantErr { + t.Errorf("createMissingStudio() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("createMissingStudio() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_scrapedToStudioInput(t *testing.T) { + const name = "name" + const md5 = "b068931cc450442b63f5b3d276ea4297" + url := "url" + + tests := []struct { + name string + studio *models.ScrapedStudio + want models.Studio + }{ + { + "set all", + &models.ScrapedStudio{ + Name: name, + URL: &url, + }, + models.Studio{ + Name: models.NullString(name), + Checksum: md5, + URL: models.NullString(url), + }, + }, + { + "set none", + &models.ScrapedStudio{ + Name: name, + }, + models.Studio{ + Name: models.NullString(name), + Checksum: md5, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := scrapedToStudioInput(tt.studio) + + // clear created/updated dates + got.CreatedAt = models.SQLiteTimestamp{} + got.UpdatedAt = got.CreatedAt + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("scrapedToStudioInput() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index be63a4c0d..aff7fc752 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -144,6 +144,11 @@ const ( const HandyKey = "handy_key" const FunscriptOffset = "funscript_offset" +// Default settings +const ( + DefaultIdentifySettings = "defaults.identify_task" +) + // Security const TrustedProxies = "trusted_proxies" const dangerousAllowPublicWithoutAuth = "dangerous_allow_public_without_auth" @@ -476,10 +481,10 @@ func (i *Instance) GetScraperExcludeTagPatterns() []string { return ret } -func (i *Instance) GetStashBoxes() []*models.StashBox { +func (i *Instance) GetStashBoxes() models.StashBoxes { i.RLock() defer i.RUnlock() - var boxes []*models.StashBox + var boxes models.StashBoxes if err := viper.UnmarshalKey(StashBoxes, &boxes); err != nil { logger.Warnf("error in unmarshalkey: %v", err) } @@ -869,10 +874,30 @@ func (i *Instance) GetHandyKey() string { } func (i *Instance) GetFunscriptOffset() int { + i.Lock() + defer i.Unlock() viper.SetDefault(FunscriptOffset, 0) return viper.GetInt(FunscriptOffset) } +// GetDefaultIdentifySettings returns the default Identify task settings. +// Returns nil if the settings could not be unmarshalled, or if it +// has not been set. +func (i *Instance) GetDefaultIdentifySettings() *models.IdentifyMetadataTaskOptions { + i.RLock() + defer i.RUnlock() + + if viper.IsSet(DefaultIdentifySettings) { + var ret models.IdentifyMetadataTaskOptions + if err := viper.UnmarshalKey(DefaultIdentifySettings, &ret); err != nil { + return nil + } + return &ret + } + + return nil +} + // GetTrustedProxies returns a comma separated list of ip addresses that should allow proxying. // When empty, allow from any private network func (i *Instance) GetTrustedProxies() []string { diff --git a/pkg/manager/config/config_concurrency_test.go b/pkg/manager/config/config_concurrency_test.go index efac016ab..7447a581a 100644 --- a/pkg/manager/config/config_concurrency_test.go +++ b/pkg/manager/config/config_concurrency_test.go @@ -92,6 +92,8 @@ func TestConcurrentConfigAccess(t *testing.T) { i.Set(LogLevel, i.GetLogLevel()) i.Set(LogAccess, i.GetLogAccess()) i.Set(MaxUploadSize, i.GetMaxUploadSize()) + i.Set(FunscriptOffset, i.GetFunscriptOffset()) + i.Set(DefaultIdentifySettings, i.GetDefaultIdentifySettings()) } wg.Done() }(k) diff --git a/pkg/manager/scene_screenshot.go b/pkg/manager/scene_screenshot.go deleted file mode 100644 index 0e410ae5d..000000000 --- a/pkg/manager/scene_screenshot.go +++ /dev/null @@ -1,61 +0,0 @@ -package manager - -import ( - "bytes" - "image" - "image/jpeg" - "os" - - "github.com/disintegration/imaging" - - // needed to decode other image formats - _ "image/gif" - _ "image/png" -) - -func writeImage(path string, imageData []byte) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - _, err = f.Write(imageData) - return err -} - -func writeThumbnail(path string, thumbnail image.Image) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - return jpeg.Encode(f, thumbnail, nil) -} - -func SetSceneScreenshot(checksum string, imageData []byte) error { - thumbPath := instance.Paths.Scene.GetThumbnailScreenshotPath(checksum) - normalPath := instance.Paths.Scene.GetScreenshotPath(checksum) - - img, _, err := image.Decode(bytes.NewReader(imageData)) - if err != nil { - return err - } - - // resize to 320 width maintaining aspect ratio, for the thumbnail - const width = 320 - origWidth := img.Bounds().Max.X - origHeight := img.Bounds().Max.Y - height := width / origWidth * origHeight - - thumbnail := imaging.Resize(img, width, height, imaging.Lanczos) - err = writeThumbnail(thumbPath, thumbnail) - if err != nil { - return err - } - - err = writeImage(normalPath, imageData) - - return err -} diff --git a/pkg/manager/task_autotag.go b/pkg/manager/task_autotag.go index 627a8f3d4..421eff709 100644 --- a/pkg/manager/task_autotag.go +++ b/pkg/manager/task_autotag.go @@ -326,28 +326,7 @@ type autoTagFilesTask struct { } func (t *autoTagFilesTask) makeSceneFilter() *models.SceneFilterType { - ret := &models.SceneFilterType{} - or := ret - sep := string(filepath.Separator) - - for _, p := range t.paths { - if !strings.HasSuffix(p, sep) { - p += sep - } - - if ret.Path == nil { - or = ret - } else { - newOr := &models.SceneFilterType{} - or.Or = newOr - or = newOr - } - - or.Path = &models.StringCriterionInput{ - Modifier: models.CriterionModifierEquals, - Value: p + "%", - } - } + ret := scene.FilterFromPaths(t.paths) organized := false ret.Organized = &organized diff --git a/pkg/manager/task_generate_screenshot.go b/pkg/manager/task_generate_screenshot.go index f6c0f19db..baa3ab107 100644 --- a/pkg/manager/task_generate_screenshot.go +++ b/pkg/manager/task_generate_screenshot.go @@ -9,6 +9,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/scene" ) type GenerateScreenshotTask struct { @@ -66,7 +67,7 @@ func (t *GenerateScreenshotTask) Start(ctx context.Context) { UpdatedAt: &models.SQLiteTimestamp{Timestamp: updatedTime}, } - if err := SetSceneScreenshot(checksum, coverImageData); err != nil { + if err := scene.SetScreenshot(instance.Paths, checksum, coverImageData); err != nil { return fmt.Errorf("error writing screenshot: %v", err) } diff --git a/pkg/manager/task_identify.go b/pkg/manager/task_identify.go new file mode 100644 index 000000000..b8d2bfe3d --- /dev/null +++ b/pkg/manager/task_identify.go @@ -0,0 +1,244 @@ +package manager + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/stashapp/stash/pkg/identify" + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/scraper" + "github.com/stashapp/stash/pkg/scraper/stashbox" + "github.com/stashapp/stash/pkg/utils" +) + +var ErrInput = errors.New("invalid request input") + +type IdentifyJob struct { + txnManager models.TransactionManager + postHookExecutor identify.SceneUpdatePostHookExecutor + input models.IdentifyMetadataInput + + stashBoxes models.StashBoxes + progress *job.Progress +} + +func CreateIdentifyJob(input models.IdentifyMetadataInput) *IdentifyJob { + return &IdentifyJob{ + txnManager: instance.TxnManager, + postHookExecutor: instance.PluginCache, + input: input, + stashBoxes: instance.Config.GetStashBoxes(), + } +} + +func (j *IdentifyJob) Execute(ctx context.Context, progress *job.Progress) { + j.progress = progress + + // if no sources provided - just return + if len(j.input.Sources) == 0 { + return + } + + sources, err := j.getSources() + if err != nil { + logger.Error(err) + return + } + + // if scene ids provided, use those + // otherwise, batch query for all scenes - ordering by path + if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { + if len(j.input.SceneIDs) == 0 { + return j.identifyAllScenes(ctx, r, sources) + } + + sceneIDs, err := utils.StringSliceToIntSlice(j.input.SceneIDs) + if err != nil { + return fmt.Errorf("invalid scene IDs: %w", err) + } + + progress.SetTotal(len(sceneIDs)) + for _, id := range sceneIDs { + if job.IsCancelled(ctx) { + break + } + + // find the scene + var err error + scene, err := r.Scene().Find(id) + if err != nil { + return fmt.Errorf("error finding scene with id %d: %w", id, err) + } + + if scene == nil { + return fmt.Errorf("%w: scene with id %d", models.ErrNotFound, id) + } + + j.identifyScene(ctx, scene, sources) + } + + return nil + }); err != nil { + logger.Errorf("Error encountered while identifying scenes: %v", err) + } +} + +func (j *IdentifyJob) identifyAllScenes(ctx context.Context, r models.ReaderRepository, sources []identify.ScraperSource) error { + // exclude organised + organised := false + sceneFilter := scene.FilterFromPaths(j.input.Paths) + sceneFilter.Organized = &organised + + sort := "path" + findFilter := &models.FindFilterType{ + Sort: &sort, + } + + // get the count + pp := 0 + findFilter.PerPage = &pp + countResult, err := r.Scene().Query(models.SceneQueryOptions{ + QueryOptions: models.QueryOptions{ + FindFilter: findFilter, + Count: true, + }, + SceneFilter: sceneFilter, + }) + if err != nil { + return fmt.Errorf("error getting scene count: %w", err) + } + + j.progress.SetTotal(countResult.Count) + + return scene.BatchProcess(ctx, r.Scene(), sceneFilter, findFilter, func(scene *models.Scene) error { + if job.IsCancelled(ctx) { + return nil + } + + j.identifyScene(ctx, scene, sources) + return nil + }) +} + +func (j *IdentifyJob) identifyScene(ctx context.Context, s *models.Scene, sources []identify.ScraperSource) { + if job.IsCancelled(ctx) { + return + } + + if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { + var taskError error + j.progress.ExecuteTask("Identifying "+s.Path, func() { + task := identify.SceneIdentifier{ + DefaultOptions: j.input.Options, + Sources: sources, + ScreenshotSetter: &scene.PathsScreenshotSetter{ + Paths: instance.Paths, + FileNamingAlgorithm: instance.Config.GetVideoFileNamingAlgorithm(), + }, + SceneUpdatePostHookExecutor: j.postHookExecutor, + } + + taskError = task.Identify(ctx, r, s) + }) + + return taskError + }); err != nil { + logger.Errorf("Error encountered identifying %s: %v", s.Path, err) + } + + j.progress.Increment() +} + +func (j *IdentifyJob) getSources() ([]identify.ScraperSource, error) { + var ret []identify.ScraperSource + for _, source := range j.input.Sources { + // get scraper source + stashBox, err := j.getStashBox(source.Source) + if err != nil { + return nil, err + } + + var src identify.ScraperSource + if stashBox != nil { + src = identify.ScraperSource{ + Name: "stash-box: " + stashBox.Endpoint, + Scraper: stashboxSource{ + stashbox.NewClient(*stashBox, j.txnManager), + stashBox.Endpoint, + }, + RemoteSite: stashBox.Endpoint, + } + } else { + scraperID := *source.Source.ScraperID + s := instance.ScraperCache.GetScraper(scraperID) + if s == nil { + return nil, fmt.Errorf("%w: scraper with id %q", models.ErrNotFound, scraperID) + } + src = identify.ScraperSource{ + Name: s.Name, + Scraper: scraperSource{ + cache: instance.ScraperCache, + scraperID: scraperID, + }, + } + } + + src.Options = source.Options + ret = append(ret, src) + } + + return ret, nil +} + +func (j *IdentifyJob) getStashBox(src *models.ScraperSourceInput) (*models.StashBox, error) { + if src.ScraperID != nil { + return nil, nil + } + + // must be stash-box + if src.StashBoxIndex == nil && src.StashBoxEndpoint == nil { + return nil, fmt.Errorf("%w: stash_box_index or stash_box_endpoint or scraper_id must be set", ErrInput) + } + + return j.stashBoxes.ResolveStashBox(*src) +} + +type stashboxSource struct { + *stashbox.Client + endpoint string +} + +func (s stashboxSource) ScrapeScene(sceneID int) (*models.ScrapedScene, error) { + results, err := s.FindStashBoxScenesByFingerprintsFlat([]string{strconv.Itoa(sceneID)}) + if err != nil { + return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err) + } + + if len(results) > 0 { + return results[0], nil + } + + return nil, nil +} + +func (s stashboxSource) String() string { + return fmt.Sprintf("stash-box %s", s.endpoint) +} + +type scraperSource struct { + cache *scraper.Cache + scraperID string +} + +func (s scraperSource) ScrapeScene(sceneID int) (*models.ScrapedScene, error) { + return s.cache.ScrapeScene(s.scraperID, sceneID) +} + +func (s scraperSource) String() string { + return fmt.Sprintf("scraper %s", s.scraperID) +} diff --git a/pkg/models/errors.go b/pkg/models/errors.go new file mode 100644 index 000000000..54f5e1d00 --- /dev/null +++ b/pkg/models/errors.go @@ -0,0 +1,5 @@ +package models + +import "errors" + +var ErrNotFound = errors.New("not found") diff --git a/pkg/models/mocks/transaction.go b/pkg/models/mocks/transaction.go index da6e2e333..886fef7d6 100644 --- a/pkg/models/mocks/transaction.go +++ b/pkg/models/mocks/transaction.go @@ -7,16 +7,16 @@ import ( ) type TransactionManager struct { - gallery models.GalleryReaderWriter - image models.ImageReaderWriter - movie models.MovieReaderWriter - performer models.PerformerReaderWriter - scene models.SceneReaderWriter - sceneMarker models.SceneMarkerReaderWriter - scrapedItem models.ScrapedItemReaderWriter - studio models.StudioReaderWriter - tag models.TagReaderWriter - savedFilter models.SavedFilterReaderWriter + gallery *GalleryReaderWriter + image *ImageReaderWriter + movie *MovieReaderWriter + performer *PerformerReaderWriter + scene *SceneReaderWriter + sceneMarker *SceneMarkerReaderWriter + scrapedItem *ScrapedItemReaderWriter + studio *StudioReaderWriter + tag *TagReaderWriter + savedFilter *SavedFilterReaderWriter } func NewTransactionManager() *TransactionManager { @@ -38,90 +38,130 @@ func (t *TransactionManager) WithTxn(ctx context.Context, fn func(r models.Repos return fn(t) } -func (t *TransactionManager) Gallery() models.GalleryReaderWriter { +func (t *TransactionManager) GalleryMock() *GalleryReaderWriter { return t.gallery } -func (t *TransactionManager) Image() models.ImageReaderWriter { +func (t *TransactionManager) ImageMock() *ImageReaderWriter { return t.image } -func (t *TransactionManager) Movie() models.MovieReaderWriter { +func (t *TransactionManager) MovieMock() *MovieReaderWriter { return t.movie } -func (t *TransactionManager) Performer() models.PerformerReaderWriter { +func (t *TransactionManager) PerformerMock() *PerformerReaderWriter { return t.performer } -func (t *TransactionManager) SceneMarker() models.SceneMarkerReaderWriter { +func (t *TransactionManager) SceneMarkerMock() *SceneMarkerReaderWriter { return t.sceneMarker } -func (t *TransactionManager) Scene() models.SceneReaderWriter { +func (t *TransactionManager) SceneMock() *SceneReaderWriter { return t.scene } -func (t *TransactionManager) ScrapedItem() models.ScrapedItemReaderWriter { +func (t *TransactionManager) ScrapedItemMock() *ScrapedItemReaderWriter { return t.scrapedItem } -func (t *TransactionManager) Studio() models.StudioReaderWriter { +func (t *TransactionManager) StudioMock() *StudioReaderWriter { return t.studio } -func (t *TransactionManager) Tag() models.TagReaderWriter { +func (t *TransactionManager) TagMock() *TagReaderWriter { return t.tag } -func (t *TransactionManager) SavedFilter() models.SavedFilterReaderWriter { +func (t *TransactionManager) SavedFilterMock() *SavedFilterReaderWriter { return t.savedFilter } +func (t *TransactionManager) Gallery() models.GalleryReaderWriter { + return t.GalleryMock() +} + +func (t *TransactionManager) Image() models.ImageReaderWriter { + return t.ImageMock() +} + +func (t *TransactionManager) Movie() models.MovieReaderWriter { + return t.MovieMock() +} + +func (t *TransactionManager) Performer() models.PerformerReaderWriter { + return t.PerformerMock() +} + +func (t *TransactionManager) SceneMarker() models.SceneMarkerReaderWriter { + return t.SceneMarkerMock() +} + +func (t *TransactionManager) Scene() models.SceneReaderWriter { + return t.SceneMock() +} + +func (t *TransactionManager) ScrapedItem() models.ScrapedItemReaderWriter { + return t.ScrapedItemMock() +} + +func (t *TransactionManager) Studio() models.StudioReaderWriter { + return t.StudioMock() +} + +func (t *TransactionManager) Tag() models.TagReaderWriter { + return t.TagMock() +} + +func (t *TransactionManager) SavedFilter() models.SavedFilterReaderWriter { + return t.SavedFilterMock() +} + type ReadTransaction struct { - t *TransactionManager + *TransactionManager } func (t *TransactionManager) WithReadTxn(ctx context.Context, fn func(r models.ReaderRepository) error) error { - return fn(&ReadTransaction{t: t}) + return fn(&ReadTransaction{t}) } func (r *ReadTransaction) Gallery() models.GalleryReader { - return r.t.gallery + return r.GalleryMock() } func (r *ReadTransaction) Image() models.ImageReader { - return r.t.image + return r.ImageMock() } func (r *ReadTransaction) Movie() models.MovieReader { - return r.t.movie + return r.MovieMock() } func (r *ReadTransaction) Performer() models.PerformerReader { - return r.t.performer + return r.PerformerMock() } func (r *ReadTransaction) SceneMarker() models.SceneMarkerReader { - return r.t.sceneMarker + return r.SceneMarkerMock() } func (r *ReadTransaction) Scene() models.SceneReader { - return r.t.scene + return r.SceneMock() } func (r *ReadTransaction) ScrapedItem() models.ScrapedItemReader { - return r.t.scrapedItem + return r.ScrapedItemMock() } func (r *ReadTransaction) Studio() models.StudioReader { - return r.t.studio + return r.StudioMock() } func (r *ReadTransaction) Tag() models.TagReader { - return r.t.tag + return r.TagMock() } func (r *ReadTransaction) SavedFilter() models.SavedFilterReader { - return r.t.savedFilter + return r.SavedFilterMock() } diff --git a/pkg/models/model_joins.go b/pkg/models/model_joins.go index 802651e21..1eebcd2f1 100644 --- a/pkg/models/model_joins.go +++ b/pkg/models/model_joins.go @@ -12,3 +12,10 @@ type StashID struct { StashID string `db:"stash_id" json:"stash_id"` Endpoint string `db:"endpoint" json:"endpoint"` } + +func (s StashID) StashIDInput() StashIDInput { + return StashIDInput{ + Endpoint: s.Endpoint, + StashID: s.StashID, + } +} diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index 6ffc914c1..6d1c37e3f 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -3,6 +3,7 @@ package models import ( "database/sql" "path/filepath" + "strconv" "time" ) @@ -119,6 +120,29 @@ type ScenePartial struct { Interactive *bool `db:"interactive" json:"interactive"` } +// UpdateInput constructs a SceneUpdateInput using the populated fields in the ScenePartial object. +func (s ScenePartial) UpdateInput() SceneUpdateInput { + boolPtrCopy := func(v *bool) *bool { + if v == nil { + return nil + } + + vv := *v + return &vv + } + + return SceneUpdateInput{ + ID: strconv.Itoa(s.ID), + Title: nullStringPtrToStringPtr(s.Title), + Details: nullStringPtrToStringPtr(s.Details), + URL: nullStringPtrToStringPtr(s.URL), + Date: s.Date.StringPtr(), + Rating: nullInt64PtrToIntPtr(s.Rating), + Organized: boolPtrCopy(s.Organized), + StudioID: nullInt64PtrToStringPtr(s.StudioID), + } +} + func (s *ScenePartial) SetFile(f File) { path := f.Path s.Path = &path diff --git a/pkg/models/model_scene_test.go b/pkg/models/model_scene_test.go new file mode 100644 index 000000000..43216e539 --- /dev/null +++ b/pkg/models/model_scene_test.go @@ -0,0 +1,80 @@ +package models + +import ( + "database/sql" + "reflect" + "testing" +) + +func TestScenePartial_UpdateInput(t *testing.T) { + const ( + id = 1 + idStr = "1" + ) + + var ( + title = "title" + details = "details" + url = "url" + date = "2001-02-03" + rating = 4 + organized = true + studioID = 2 + studioIDStr = "2" + ) + + tests := []struct { + name string + s ScenePartial + want SceneUpdateInput + }{ + { + "full", + ScenePartial{ + ID: id, + Title: NullStringPtr(title), + Details: NullStringPtr(details), + URL: NullStringPtr(url), + Date: &SQLiteDate{ + String: date, + Valid: true, + }, + Rating: &sql.NullInt64{ + Int64: int64(rating), + Valid: true, + }, + Organized: &organized, + StudioID: &sql.NullInt64{ + Int64: int64(studioID), + Valid: true, + }, + }, + SceneUpdateInput{ + ID: idStr, + Title: &title, + Details: &details, + URL: &url, + Date: &date, + Rating: &rating, + Organized: &organized, + StudioID: &studioIDStr, + }, + }, + { + "empty", + ScenePartial{ + ID: id, + }, + SceneUpdateInput{ + ID: idStr, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.s.UpdateInput(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ScenePartial.UpdateInput() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/models/scraped.go b/pkg/models/scraped.go index ecbddf68a..f57a8409a 100644 --- a/pkg/models/scraped.go +++ b/pkg/models/scraped.go @@ -1,5 +1,9 @@ package models +import "errors" + +var ErrScraperSource = errors.New("invalid ScraperSource") + type ScrapedItemReader interface { All() ([]*ScrapedItem, error) } diff --git a/pkg/models/sql.go b/pkg/models/sql.go index ea33f3245..f4960d84b 100644 --- a/pkg/models/sql.go +++ b/pkg/models/sql.go @@ -1,6 +1,9 @@ package models -import "database/sql" +import ( + "database/sql" + "strconv" +) func NullString(v string) sql.NullString { return sql.NullString{ @@ -9,9 +12,43 @@ func NullString(v string) sql.NullString { } } +func NullStringPtr(v string) *sql.NullString { + return &sql.NullString{ + String: v, + Valid: true, + } +} + func NullInt64(v int64) sql.NullInt64 { return sql.NullInt64{ Int64: v, Valid: true, } } + +func nullStringPtrToStringPtr(v *sql.NullString) *string { + if v == nil || !v.Valid { + return nil + } + + vv := v.String + return &vv +} + +func nullInt64PtrToIntPtr(v *sql.NullInt64) *int { + if v == nil || !v.Valid { + return nil + } + + vv := int(v.Int64) + return &vv +} + +func nullInt64PtrToStringPtr(v *sql.NullInt64) *string { + if v == nil || !v.Valid { + return nil + } + + vv := strconv.FormatInt(v.Int64, 10) + return &vv +} diff --git a/pkg/models/sqlite_date.go b/pkg/models/sqlite_date.go index bd9ebf8cd..e11bf462c 100644 --- a/pkg/models/sqlite_date.go +++ b/pkg/models/sqlite_date.go @@ -44,3 +44,12 @@ func (t SQLiteDate) Value() (driver.Value, error) { } return result, nil } + +func (t *SQLiteDate) StringPtr() *string { + if t == nil || !t.Valid { + return nil + } + + vv := t.String + return &vv +} diff --git a/pkg/models/stash_box.go b/pkg/models/stash_box.go new file mode 100644 index 000000000..3e981484b --- /dev/null +++ b/pkg/models/stash_box.go @@ -0,0 +1,39 @@ +package models + +import ( + "fmt" + "strings" +) + +type StashBoxes []*StashBox + +func (sb StashBoxes) ResolveStashBox(source ScraperSourceInput) (*StashBox, error) { + if source.StashBoxIndex != nil { + index := source.StashBoxIndex + if *index < 0 || *index >= len(sb) { + return nil, fmt.Errorf("%w: invalid stash_box_index: %d", ErrScraperSource, index) + } + + return sb[*index], nil + } + + if source.StashBoxEndpoint != nil { + var ret *StashBox + endpoint := *source.StashBoxEndpoint + for _, b := range sb { + if strings.EqualFold(endpoint, b.Endpoint) { + ret = b + } + } + + if ret == nil { + return nil, fmt.Errorf(`%w: stash-box with endpoint "%s"`, ErrNotFound, endpoint) + } + + return ret, nil + } + + // neither stash-box inputs were provided, so assume it is a scraper + + return nil, nil +} diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 7322221a9..8f88e3d06 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "strconv" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" @@ -179,6 +180,15 @@ func (c Cache) ExecutePostHooks(ctx context.Context, id int, hookType HookTrigge } } +func (c Cache) ExecuteSceneUpdatePostHooks(ctx context.Context, input models.SceneUpdateInput, inputFields []string) { + id, err := strconv.Atoi(input.ID) + if err != nil { + logger.Errorf("error converting id in SceneUpdatePostHooks: %v", err) + return + } + c.ExecutePostHooks(ctx, id, SceneUpdatePost, input, inputFields) +} + func (c Cache) executePostHooks(ctx context.Context, hookType HookTriggerEnum, hookContext common.HookContext) error { visitedPlugins := session.GetVisitedPlugins(ctx) diff --git a/pkg/scene/export_test.go b/pkg/scene/export_test.go index 258b17a11..3911b03ca 100644 --- a/pkg/scene/export_test.go +++ b/pkg/scene/export_test.go @@ -85,7 +85,7 @@ var names = []string{ var imageBytes = []byte("imageBytes") -const image = "aW1hZ2VCeXRlcw==" +const imageBase64 = "aW1hZ2VCeXRlcw==" var ( createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) @@ -198,7 +198,7 @@ type basicTestScenario struct { var scenarios = []basicTestScenario{ { createFullScene(sceneID), - createFullJSONScene(image), + createFullJSONScene(imageBase64), false, }, { diff --git a/pkg/scene/import_test.go b/pkg/scene/import_test.go index 3f59c3cf3..2a8122551 100644 --- a/pkg/scene/import_test.go +++ b/pkg/scene/import_test.go @@ -75,7 +75,7 @@ func TestImporterPreImport(t *testing.T) { err := i.PreImport() assert.NotNil(t, err) - i.Input.Cover = image + i.Input.Cover = imageBase64 err = i.PreImport() assert.Nil(t, err) diff --git a/pkg/scene/query.go b/pkg/scene/query.go index 6671eac0d..f560430c3 100644 --- a/pkg/scene/query.go +++ b/pkg/scene/query.go @@ -1,6 +1,14 @@ package scene -import "github.com/stashapp/stash/pkg/models" +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/models" +) type Queryer interface { Query(options models.SceneQueryOptions) (*models.SceneQueryResult, error) @@ -48,3 +56,70 @@ func Query(qb Queryer, sceneFilter *models.SceneFilterType, findFilter *models.F return scenes, nil } + +func BatchProcess(ctx context.Context, reader models.SceneReader, sceneFilter *models.SceneFilterType, findFilter *models.FindFilterType, fn func(scene *models.Scene) error) error { + const batchSize = 1000 + + if findFilter == nil { + findFilter = &models.FindFilterType{} + } + + page := 1 + perPage := batchSize + findFilter.Page = &page + findFilter.PerPage = &perPage + + for more := true; more; { + if job.IsCancelled(ctx) { + return nil + } + + scenes, err := Query(reader, sceneFilter, findFilter) + if err != nil { + return fmt.Errorf("error querying for scenes: %w", err) + } + + for _, scene := range scenes { + if err := fn(scene); err != nil { + return err + } + } + + if len(scenes) != batchSize { + more = false + } else { + *findFilter.Page++ + } + } + + return nil +} + +// FilterFromPaths creates a SceneFilterType that filters using the provided +// paths. +func FilterFromPaths(paths []string) *models.SceneFilterType { + ret := &models.SceneFilterType{} + or := ret + sep := string(filepath.Separator) + + for _, p := range paths { + if !strings.HasSuffix(p, sep) { + p += sep + } + + if ret.Path == nil { + or = ret + } else { + newOr := &models.SceneFilterType{} + or.Or = newOr + or = newOr + } + + or.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierEquals, + Value: p + "%", + } + } + + return ret +} diff --git a/pkg/scene/screenshot.go b/pkg/scene/screenshot.go index 9ee0ccab0..7af8ca3e4 100644 --- a/pkg/scene/screenshot.go +++ b/pkg/scene/screenshot.go @@ -1,8 +1,21 @@ package scene import ( + "bytes" + "image" + "image/jpeg" + "os" + "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/models" + + "github.com/disintegration/imaging" + + // needed to decode other image formats + _ "image/gif" + _ "image/png" ) type screenshotter interface { @@ -21,3 +34,64 @@ func makeScreenshot(encoder screenshotter, probeResult ffmpeg.VideoFile, outputP logger.Warnf("[encoder] failure to generate screenshot: %v", err) } } + +type ScreenshotSetter interface { + SetScreenshot(scene *models.Scene, imageData []byte) error +} + +type PathsScreenshotSetter struct { + Paths *paths.Paths + FileNamingAlgorithm models.HashAlgorithm +} + +func (ss *PathsScreenshotSetter) SetScreenshot(scene *models.Scene, imageData []byte) error { + checksum := scene.GetHash(ss.FileNamingAlgorithm) + return SetScreenshot(ss.Paths, checksum, imageData) +} + +func writeImage(path string, imageData []byte) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(imageData) + return err +} + +func writeThumbnail(path string, thumbnail image.Image) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return jpeg.Encode(f, thumbnail, nil) +} + +func SetScreenshot(paths *paths.Paths, checksum string, imageData []byte) error { + thumbPath := paths.Scene.GetThumbnailScreenshotPath(checksum) + normalPath := paths.Scene.GetScreenshotPath(checksum) + + img, _, err := image.Decode(bytes.NewReader(imageData)) + if err != nil { + return err + } + + // resize to 320 width maintaining aspect ratio, for the thumbnail + const width = 320 + origWidth := img.Bounds().Max.X + origHeight := img.Bounds().Max.Y + height := width / origWidth * origHeight + + thumbnail := imaging.Resize(img, width, height, imaging.Lanczos) + err = writeThumbnail(thumbPath, thumbnail) + if err != nil { + return err + } + + err = writeImage(normalPath, imageData) + + return err +} diff --git a/pkg/scene/update.go b/pkg/scene/update.go index 7dec4f415..d351b13eb 100644 --- a/pkg/scene/update.go +++ b/pkg/scene/update.go @@ -2,11 +2,127 @@ package scene import ( "database/sql" + "errors" + "fmt" + "time" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) +var ErrEmptyUpdater = errors.New("no fields have been set") + +// UpdateSet is used to update a scene and its relationships. +type UpdateSet struct { + ID int + + Partial models.ScenePartial + + // in future these could be moved into a separate struct and reused + // for a Creator struct + + // Not set if nil. Set to []int{} to clear existing + PerformerIDs []int + // Not set if nil. Set to []int{} to clear existing + TagIDs []int + // Not set if nil. Set to []int{} to clear existing + StashIDs []models.StashID + // Not set if nil. Set to []byte{} to clear existing + CoverImage []byte +} + +// IsEmpty returns true if there is nothing to update. +func (u *UpdateSet) IsEmpty() bool { + withoutID := u.Partial + withoutID.ID = 0 + + return withoutID == models.ScenePartial{} && + u.PerformerIDs == nil && + u.TagIDs == nil && + u.StashIDs == nil && + u.CoverImage == nil +} + +// Update updates a scene by updating the fields in the Partial field, then +// updates non-nil relationships. Returns an error if there is no work to +// be done. +func (u *UpdateSet) Update(qb models.SceneWriter, screenshotSetter ScreenshotSetter) (*models.Scene, error) { + if u.IsEmpty() { + return nil, ErrEmptyUpdater + } + + partial := u.Partial + partial.ID = u.ID + partial.UpdatedAt = &models.SQLiteTimestamp{ + Timestamp: time.Now(), + } + + ret, err := qb.Update(partial) + if err != nil { + return nil, fmt.Errorf("error updating scene: %w", err) + } + + if u.PerformerIDs != nil { + if err := qb.UpdatePerformers(u.ID, u.PerformerIDs); err != nil { + return nil, fmt.Errorf("error updating scene performers: %w", err) + } + } + + if u.TagIDs != nil { + if err := qb.UpdateTags(u.ID, u.TagIDs); err != nil { + return nil, fmt.Errorf("error updating scene tags: %w", err) + } + } + + if u.StashIDs != nil { + if err := qb.UpdateStashIDs(u.ID, u.StashIDs); err != nil { + return nil, fmt.Errorf("error updating scene stash_ids: %w", err) + } + } + + if u.CoverImage != nil { + if err := qb.UpdateCover(u.ID, u.CoverImage); err != nil { + return nil, fmt.Errorf("error updating scene cover: %w", err) + } + + if err := screenshotSetter.SetScreenshot(ret, u.CoverImage); err != nil { + return nil, fmt.Errorf("error setting scene screenshot: %w", err) + } + } + + return ret, nil +} + +// UpdateInput converts the UpdateSet into SceneUpdateInput for hook firing purposes. +func (u UpdateSet) UpdateInput() models.SceneUpdateInput { + // ensure the partial ID is set + u.Partial.ID = u.ID + ret := u.Partial.UpdateInput() + + if u.PerformerIDs != nil { + ret.PerformerIds = utils.IntSliceToStringSlice(u.PerformerIDs) + } + + if u.TagIDs != nil { + ret.TagIds = utils.IntSliceToStringSlice(u.TagIDs) + } + + if u.StashIDs != nil { + for _, s := range u.StashIDs { + ss := s.StashIDInput() + ret.StashIds = append(ret.StashIds, &ss) + } + } + + if u.CoverImage != nil { + // convert back to base64 + data := utils.GetBase64StringFromData(u.CoverImage) + ret.CoverImage = &data + } + + return ret +} + func UpdateFormat(qb models.SceneWriter, id int, format string) (*models.Scene, error) { return qb.Update(models.ScenePartial{ ID: id, diff --git a/pkg/scene/update_test.go b/pkg/scene/update_test.go new file mode 100644 index 000000000..587258b93 --- /dev/null +++ b/pkg/scene/update_test.go @@ -0,0 +1,337 @@ +package scene + +import ( + "errors" + "strconv" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stashapp/stash/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestUpdater_IsEmpty(t *testing.T) { + organized := true + ids := []int{1} + stashIDs := []models.StashID{ + {}, + } + cover := []byte{1} + + tests := []struct { + name string + u *UpdateSet + want bool + }{ + { + "empty", + &UpdateSet{}, + true, + }, + { + "id only", + &UpdateSet{ + Partial: models.ScenePartial{ + ID: 1, + }, + }, + true, + }, + { + "partial set", + &UpdateSet{ + Partial: models.ScenePartial{ + Organized: &organized, + }, + }, + false, + }, + { + "performer set", + &UpdateSet{ + PerformerIDs: ids, + }, + false, + }, + { + "tags set", + &UpdateSet{ + TagIDs: ids, + }, + false, + }, + { + "performer set", + &UpdateSet{ + StashIDs: stashIDs, + }, + false, + }, + { + "cover set", + &UpdateSet{ + CoverImage: cover, + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.u.IsEmpty(); got != tt.want { + t.Errorf("Updater.IsEmpty() = %v, want %v", got, tt.want) + } + }) + } +} + +type mockScreenshotSetter struct{} + +func (s *mockScreenshotSetter) SetScreenshot(scene *models.Scene, imageData []byte) error { + return nil +} + +func TestUpdater_Update(t *testing.T) { + const ( + sceneID = iota + 1 + badUpdateID + badPerformersID + badTagsID + badStashIDsID + badCoverID + performerID + tagID + ) + + performerIDs := []int{performerID} + tagIDs := []int{tagID} + stashID := "stashID" + endpoint := "endpoint" + stashIDs := []models.StashID{ + { + StashID: stashID, + Endpoint: endpoint, + }, + } + + title := "title" + cover := []byte("cover") + + validScene := &models.Scene{} + + updateErr := errors.New("error updating") + + qb := mocks.SceneReaderWriter{} + qb.On("Update", mock.MatchedBy(func(s models.ScenePartial) bool { + return s.ID != badUpdateID + })).Return(validScene, nil) + qb.On("Update", mock.MatchedBy(func(s models.ScenePartial) bool { + return s.ID == badUpdateID + })).Return(nil, updateErr) + + qb.On("UpdatePerformers", sceneID, performerIDs).Return(nil).Once() + qb.On("UpdateTags", sceneID, tagIDs).Return(nil).Once() + qb.On("UpdateStashIDs", sceneID, stashIDs).Return(nil).Once() + qb.On("UpdateCover", sceneID, cover).Return(nil).Once() + + qb.On("UpdatePerformers", badPerformersID, performerIDs).Return(updateErr).Once() + qb.On("UpdateTags", badTagsID, tagIDs).Return(updateErr).Once() + qb.On("UpdateStashIDs", badStashIDsID, stashIDs).Return(updateErr).Once() + qb.On("UpdateCover", badCoverID, cover).Return(updateErr).Once() + + tests := []struct { + name string + u *UpdateSet + wantNil bool + wantErr bool + }{ + { + "empty", + &UpdateSet{ + ID: sceneID, + }, + true, + true, + }, + { + "update all", + &UpdateSet{ + ID: sceneID, + PerformerIDs: performerIDs, + TagIDs: tagIDs, + StashIDs: []models.StashID{ + { + StashID: stashID, + Endpoint: endpoint, + }, + }, + CoverImage: cover, + }, + false, + false, + }, + { + "update fields only", + &UpdateSet{ + ID: sceneID, + Partial: models.ScenePartial{ + Title: models.NullStringPtr(title), + }, + }, + false, + false, + }, + { + "error updating scene", + &UpdateSet{ + ID: badUpdateID, + Partial: models.ScenePartial{ + Title: models.NullStringPtr(title), + }, + }, + true, + true, + }, + { + "error updating performers", + &UpdateSet{ + ID: badPerformersID, + PerformerIDs: performerIDs, + }, + true, + true, + }, + { + "error updating tags", + &UpdateSet{ + ID: badTagsID, + TagIDs: tagIDs, + }, + true, + true, + }, + { + "error updating stash IDs", + &UpdateSet{ + ID: badStashIDsID, + StashIDs: stashIDs, + }, + true, + true, + }, + { + "error updating cover", + &UpdateSet{ + ID: badCoverID, + CoverImage: cover, + }, + true, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.u.Update(&qb, &mockScreenshotSetter{}) + if (err != nil) != tt.wantErr { + t.Errorf("Updater.Update() error = %v, wantErr %v", err, tt.wantErr) + return + } + if (got == nil) != tt.wantNil { + t.Errorf("Updater.Update() = %v, want %v", got, tt.wantNil) + } + }) + } + + qb.AssertExpectations(t) +} + +func TestUpdateSet_UpdateInput(t *testing.T) { + const ( + sceneID = iota + 1 + badUpdateID + badPerformersID + badTagsID + badStashIDsID + badCoverID + performerID + tagID + ) + + sceneIDStr := strconv.Itoa(sceneID) + + performerIDs := []int{performerID} + performerIDStrs := utils.IntSliceToStringSlice(performerIDs) + tagIDs := []int{tagID} + tagIDStrs := utils.IntSliceToStringSlice(tagIDs) + stashID := "stashID" + endpoint := "endpoint" + stashIDs := []models.StashID{ + { + StashID: stashID, + Endpoint: endpoint, + }, + } + stashIDInputs := []*models.StashIDInput{ + { + StashID: stashID, + Endpoint: endpoint, + }, + } + + title := "title" + cover := []byte("cover") + coverB64 := "Y292ZXI=" + + tests := []struct { + name string + u UpdateSet + want models.SceneUpdateInput + }{ + { + "empty", + UpdateSet{ + ID: sceneID, + }, + models.SceneUpdateInput{ + ID: sceneIDStr, + }, + }, + { + "update all", + UpdateSet{ + ID: sceneID, + PerformerIDs: performerIDs, + TagIDs: tagIDs, + StashIDs: stashIDs, + CoverImage: cover, + }, + models.SceneUpdateInput{ + ID: sceneIDStr, + PerformerIds: performerIDStrs, + TagIds: tagIDStrs, + StashIds: stashIDInputs, + CoverImage: &coverB64, + }, + }, + { + "update fields only", + UpdateSet{ + ID: sceneID, + Partial: models.ScenePartial{ + Title: models.NullStringPtr(title), + }, + }, + models.SceneUpdateInput{ + ID: sceneIDStr, + Title: &title, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.u.UpdateInput() + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go index 6c3d16d9c..2a1fc8efc 100644 --- a/pkg/scraper/scrapers.go +++ b/pkg/scraper/scrapers.go @@ -212,6 +212,16 @@ func (c Cache) ListMovieScrapers() []*models.Scraper { return ret } +// GetScraper returns the scraper matching the provided id. +func (c Cache) GetScraper(scraperID string) *models.Scraper { + ret := c.findScraper(scraperID) + if ret != nil { + return ret.Spec + } + + return nil +} + func (c Cache) findScraper(scraperID string) *scraper { for _, s := range c.scrapers { if s.ID == scraperID { diff --git a/pkg/utils/collections.go b/pkg/utils/collections.go new file mode 100644 index 000000000..06bc9f1f5 --- /dev/null +++ b/pkg/utils/collections.go @@ -0,0 +1,60 @@ +package utils + +import "reflect" + +// SliceSame returns true if the two provided lists have the same elements, +// regardless of order. Panics if either parameter is not a slice. +func SliceSame(a, b interface{}) bool { + v1 := reflect.ValueOf(a) + v2 := reflect.ValueOf(b) + + if (v1.IsValid() && v1.Kind() != reflect.Slice) || (v2.IsValid() && v2.Kind() != reflect.Slice) { + panic("not a slice") + } + + v1Len := 0 + v2Len := 0 + + v1Valid := v1.IsValid() + v2Valid := v2.IsValid() + + if v1Valid { + v1Len = v1.Len() + } + if v2Valid { + v2Len = v2.Len() + } + + if !v1Valid || !v2Valid { + return v1Len == v2Len + } + + if v1Len != v2Len { + return false + } + + if v1.Type() != v2.Type() { + return false + } + + visited := make(map[int]bool) + for i := 0; i < v1.Len(); i++ { + found := false + for j := 0; j < v2.Len(); j++ { + if visited[j] { + continue + } + if reflect.DeepEqual(v1.Index(i).Interface(), v2.Index(j).Interface()) { + found = true + visited[j] = true + break + } + } + + if !found { + return false + } + } + + return true +} diff --git a/pkg/utils/collections_test.go b/pkg/utils/collections_test.go new file mode 100644 index 000000000..359b9ad10 --- /dev/null +++ b/pkg/utils/collections_test.go @@ -0,0 +1,92 @@ +package utils + +import "testing" + +func TestSliceSame(t *testing.T) { + objs := []struct { + a string + b int + }{ + {"1", 2}, + {"1", 2}, + {"2", 1}, + } + + tests := []struct { + name string + a interface{} + b interface{} + want bool + }{ + {"nil values", nil, nil, true}, + {"empty", []int{}, []int{}, true}, + {"nil and empty", nil, []int{}, true}, + { + "different type", + []string{"1"}, + []int{1}, + false, + }, + { + "different length", + []int{1, 2, 3}, + []int{1, 2}, + false, + }, + { + "equal", + []int{1, 2, 3, 4, 5}, + []int{1, 2, 3, 4, 5}, + true, + }, + { + "different order", + []int{5, 4, 3, 2, 1}, + []int{1, 2, 3, 4, 5}, + true, + }, + { + "different", + []int{5, 4, 3, 2, 6}, + []int{1, 2, 3, 4, 5}, + false, + }, + { + "same with duplicates", + []int{1, 1, 2, 3, 4}, + []int{1, 2, 3, 4, 1}, + true, + }, + { + "subset", + []int{1, 1, 2, 2, 3}, + []int{1, 2, 3, 4, 5}, + false, + }, + { + "superset", + []int{1, 2, 3, 4, 5}, + []int{1, 1, 2, 2, 3}, + false, + }, + { + "structs equal", + objs[0:1], + objs[0:1], + true, + }, + { + "structs not equal", + objs[0:2], + objs[1:3], + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SliceSame(tt.a, tt.b); got != tt.want { + t.Errorf("SliceSame() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/utils/int_collections.go b/pkg/utils/int_collections.go index 41daa5185..9f2d3e6bc 100644 --- a/pkg/utils/int_collections.go +++ b/pkg/utils/int_collections.go @@ -1,5 +1,7 @@ package utils +import "strconv" + // IntIndex returns the first index of the provided int value in the provided // int slice. It returns -1 if it is not found. func IntIndex(vs []int, t int) int { @@ -50,3 +52,13 @@ func IntExclude(vs []int, toExclude []int) []int { return ret } + +// IntSliceToStringSlice converts a slice of ints to a slice of strings. +func IntSliceToStringSlice(ss []int) []string { + ret := make([]string, len(ss)) + for i, v := range ss { + ret[i] = strconv.Itoa(v) + } + + return ret +} diff --git a/pkg/utils/reflect.go b/pkg/utils/reflect.go new file mode 100644 index 000000000..65b0903b6 --- /dev/null +++ b/pkg/utils/reflect.go @@ -0,0 +1,30 @@ +package utils + +import "reflect" + +// NotNilFields returns the matching tag values of fields from an object that are not nil. +// Panics if the provided object is not a struct. +func NotNilFields(subject interface{}, tag string) []string { + value := reflect.ValueOf(subject) + structType := value.Type() + + if structType.Kind() != reflect.Struct { + panic("subject must be struct") + } + + var ret []string + + for i := 0; i < value.NumField(); i++ { + field := value.Field(i) + + kind := field.Type().Kind() + if (kind == reflect.Ptr || kind == reflect.Slice) && !field.IsNil() { + tagValue := structType.Field(i).Tag.Get(tag) + if tagValue != "" { + ret = append(ret, tagValue) + } + } + } + + return ret +} diff --git a/pkg/utils/reflect_test.go b/pkg/utils/reflect_test.go new file mode 100644 index 000000000..87757e0e1 --- /dev/null +++ b/pkg/utils/reflect_test.go @@ -0,0 +1,83 @@ +package utils + +import ( + "reflect" + "testing" +) + +func TestNotNilFields(t *testing.T) { + v := "value" + var zeroStr string + + type testObject struct { + ptrField *string `tag:"ptrField"` + noTagField *string + otherTagField *string `otherTag:"otherTagField"` + sliceField []string `tag:"sliceField"` + } + + type args struct { + subject interface{} + tag string + } + tests := []struct { + name string + args args + want []string + }{ + { + "basic", + args{ + testObject{ + ptrField: &v, + noTagField: &v, + otherTagField: &v, + sliceField: []string{v}, + }, + "tag", + }, + []string{"ptrField", "sliceField"}, + }, + { + "empty", + args{ + testObject{}, + "tag", + }, + nil, + }, + { + "zero values", + args{ + testObject{ + ptrField: &zeroStr, + noTagField: &zeroStr, + otherTagField: &zeroStr, + sliceField: []string{}, + }, + "tag", + }, + []string{"ptrField", "sliceField"}, + }, + { + "other tag", + args{ + testObject{ + ptrField: &v, + noTagField: &v, + otherTagField: &v, + sliceField: []string{v}, + }, + "otherTag", + }, + []string{"otherTagField"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NotNilFields(tt.args.subject, tt.args.tag); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NotNilFields() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/ui/v2.5/src/components/Changelog/versions/v0110.md b/ui/v2.5/src/components/Changelog/versions/v0110.md index f01eaa3c9..ead8e66b6 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0110.md +++ b/ui/v2.5/src/components/Changelog/versions/v0110.md @@ -1,4 +1,5 @@ ### ✨ New Features +* Added Identify task to automatically identify scenes from stash-box/scraper sources. See manual entry for details. ([#1839](https://github.com/stashapp/stash/pull/1839)) * Added support for matching scenes using perceptual hashes when querying stash-box. ([#1858](https://github.com/stashapp/stash/pull/1858)) * Generalised Tagger view to support tagging using supported scene scrapers. ([#1812](https://github.com/stashapp/stash/pull/1812)) * Added built-in `Auto Tag` scene scraper to match performers, studio and tags from filename - using AutoTag logic. ([#1817](https://github.com/stashapp/stash/pull/1817)) diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx new file mode 100644 index 000000000..142023e65 --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx @@ -0,0 +1,338 @@ +import React, { useState, useEffect, useCallback } from "react"; +import { Form, Button, Table } from "react-bootstrap"; +import { Icon } from "src/components/Shared"; +import * as GQL from "src/core/generated-graphql"; +import { FormattedMessage, useIntl } from "react-intl"; +import { multiValueSceneFields, SceneField, sceneFields } from "./constants"; +import { ThreeStateBoolean } from "./ThreeStateBoolean"; + +interface IFieldOptionsEditor { + options: GQL.IdentifyFieldOptions | undefined; + field: string; + editField: () => void; + editOptions: (o?: GQL.IdentifyFieldOptions | null) => void; + editing: boolean; + allowSetDefault: boolean; + defaultOptions?: GQL.IdentifyMetadataOptionsInput; +} + +interface IFieldOptions { + field: string; + strategy: GQL.IdentifyFieldStrategy | undefined; + createMissing?: GQL.Maybe | undefined; +} + +const FieldOptionsEditor: React.FC = ({ + options, + field, + editField, + editOptions, + editing, + allowSetDefault, + defaultOptions, +}) => { + const intl = useIntl(); + + const [localOptions, setLocalOptions] = useState(); + + const resetOptions = useCallback(() => { + let toSet: IFieldOptions; + if (!options) { + // unset - use default values + toSet = { + field, + strategy: undefined, + createMissing: undefined, + }; + } else { + toSet = { + field, + strategy: options.strategy, + createMissing: options.createMissing, + }; + } + setLocalOptions(toSet); + }, [options, field]); + + useEffect(() => { + resetOptions(); + }, [resetOptions]); + + function renderField() { + return intl.formatMessage({ id: field }); + } + + function renderStrategy() { + if (!localOptions) { + return; + } + + const strategies = Object.entries(GQL.IdentifyFieldStrategy); + let { strategy } = localOptions; + if (strategy === undefined) { + if (!allowSetDefault) { + strategy = GQL.IdentifyFieldStrategy.Merge; + } + } + + if (!editing) { + if (strategy === undefined) { + return intl.formatMessage({ id: "use_default" }); + } + + const f = strategies.find((s) => s[1] === strategy); + return intl.formatMessage({ + id: `config.tasks.identify.field_strategies.${f![0].toLowerCase()}`, + }); + } + + if (!localOptions) { + return <>; + } + + return ( + + {allowSetDefault ? ( + + setLocalOptions({ + ...localOptions, + strategy: undefined, + }) + } + disabled={!editing} + label={intl.formatMessage({ id: "use_default" })} + /> + ) : undefined} + {strategies.map((f) => ( + + setLocalOptions({ + ...localOptions, + strategy: f[1], + }) + } + disabled={!editing} + label={intl.formatMessage({ + id: `config.tasks.identify.field_strategies.${f[0].toLowerCase()}`, + })} + /> + ))} + + ); + } + + function maybeRenderCreateMissing() { + if (!localOptions) { + return; + } + + if ( + multiValueSceneFields.includes(localOptions.field as SceneField) && + localOptions.strategy !== GQL.IdentifyFieldStrategy.Ignore + ) { + const value = + localOptions.createMissing === null + ? undefined + : localOptions.createMissing; + + if (!editing) { + if (value === undefined && allowSetDefault) { + return intl.formatMessage({ id: "use_default" }); + } + if (value) { + return ; + } + + return ; + } + + const defaultVal = defaultOptions?.fieldOptions?.find( + (f) => f.field === localOptions.field + )?.createMissing; + + if (localOptions.strategy === undefined) { + return; + } + + return ( + + setLocalOptions({ ...localOptions, createMissing: v }) + } + defaultValue={defaultVal ?? undefined} + /> + ); + } + } + + function onEditOptions() { + if (!localOptions) { + return; + } + + // send null if strategy is undefined + if (localOptions.strategy === undefined) { + editOptions(null); + resetOptions(); + } else { + let { createMissing } = localOptions; + if (createMissing === undefined && !allowSetDefault) { + createMissing = false; + } + + editOptions({ + ...localOptions, + strategy: localOptions.strategy, + createMissing, + }); + } + } + + return ( + + {renderField()} + {renderStrategy()} + {maybeRenderCreateMissing()} + + {editing ? ( + <> + + + + ) : ( + <> + + + )} + + + ); +}; + +interface IFieldOptionsList { + fieldOptions?: GQL.IdentifyFieldOptions[]; + setFieldOptions: (o: GQL.IdentifyFieldOptions[]) => void; + setEditingField: (v: boolean) => void; + allowSetDefault?: boolean; + defaultOptions?: GQL.IdentifyMetadataOptionsInput; +} + +export const FieldOptionsList: React.FC = ({ + fieldOptions, + setFieldOptions, + setEditingField, + allowSetDefault = true, + defaultOptions, +}) => { + const [localFieldOptions, setLocalFieldOptions] = useState< + GQL.IdentifyFieldOptions[] + >(); + const [editField, setEditField] = useState(); + + useEffect(() => { + if (fieldOptions) { + setLocalFieldOptions([...fieldOptions]); + } else { + setLocalFieldOptions([]); + } + }, [fieldOptions]); + + function handleEditOptions(o?: GQL.IdentifyFieldOptions | null) { + if (!localFieldOptions) { + return; + } + + if (o !== undefined) { + const newOptions = [...localFieldOptions]; + const index = newOptions.findIndex( + (option) => option.field === editField + ); + if (index !== -1) { + // if null, then we're removing + if (o === null) { + newOptions.splice(index, 1); + } else { + // replace in list + newOptions.splice(index, 1, o); + } + } else if (o !== null) { + // don't add if null + newOptions.push(o); + } + + setFieldOptions(newOptions); + } + + setEditField(undefined); + setEditingField(false); + } + + function onEditField(field: string) { + setEditField(field); + setEditingField(true); + } + + if (!localFieldOptions) { + return <>; + } + + return ( + +
+ +
+ + + + + + + {/* eslint-disable-next-line jsx-a11y/control-has-associated-label */} + + + + {sceneFields.map((f) => ( + o.field === f)} + editField={() => onEditField(f)} + editOptions={handleEditOptions} + editing={f === editField} + defaultOptions={defaultOptions} + /> + ))} + +
FieldStrategyCreate missing +
+ + ); +}; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx new file mode 100644 index 000000000..f4e91b4ea --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx @@ -0,0 +1,451 @@ +import React, { useState, useEffect, useMemo } from "react"; +import { Button, Form, Spinner } from "react-bootstrap"; +import { + mutateMetadataIdentify, + useConfiguration, + useConfigureDefaults, + useListSceneScrapers, +} from "src/core/StashService"; +import { Icon, Modal } from "src/components/Shared"; +import { useToast } from "src/hooks"; +import * as GQL from "src/core/generated-graphql"; +import { FormattedMessage, useIntl } from "react-intl"; +import { withoutTypename } from "src/utils"; +import { + SCRAPER_PREFIX, + STASH_BOX_PREFIX, +} from "src/components/Tagger/constants"; +import { DirectorySelectionDialog } from "src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog"; +import { Manual } from "src/components/Help/Manual"; +import { IScraperSource } from "./constants"; +import { OptionsEditor } from "./Options"; +import { SourcesEditor, SourcesList } from "./Sources"; + +const autoTagScraperID = "builtin_autotag"; + +interface IIdentifyDialogProps { + selectedIds?: string[]; + onClose: () => void; +} + +export const IdentifyDialog: React.FC = ({ + selectedIds, + onClose, +}) => { + function getDefaultOptions(): GQL.IdentifyMetadataOptionsInput { + return { + fieldOptions: [ + { + field: "title", + strategy: GQL.IdentifyFieldStrategy.Overwrite, + }, + ], + includeMalePerformers: true, + setCoverImage: true, + setOrganized: false, + }; + } + + const [configureDefaults] = useConfigureDefaults(); + + const [options, setOptions] = useState( + getDefaultOptions() + ); + const [sources, setSources] = useState([]); + const [editingSource, setEditingSource] = useState< + IScraperSource | undefined + >(); + const [paths, setPaths] = useState([]); + const [showManual, setShowManual] = useState(false); + const [settingPaths, setSettingPaths] = useState(false); + const [animation, setAnimation] = useState(true); + const [editingField, setEditingField] = useState(false); + const [savingDefaults, setSavingDefaults] = useState(false); + + const intl = useIntl(); + const Toast = useToast(); + + const { data: configData, error: configError } = useConfiguration(); + const { data: scraperData, error: scraperError } = useListSceneScrapers(); + + const allSources = useMemo(() => { + if (!configData || !scraperData) return; + + const ret: IScraperSource[] = []; + + ret.push( + ...configData.configuration.general.stashBoxes.map((b, i) => { + return { + id: `${STASH_BOX_PREFIX}${i}`, + displayName: `stash-box: ${b.name}`, + stash_box_endpoint: b.endpoint, + }; + }) + ); + + const scrapers = scraperData.listSceneScrapers; + + const fragmentScrapers = scrapers.filter((s) => + s.scene?.supported_scrapes.includes(GQL.ScrapeType.Fragment) + ); + + ret.push( + ...fragmentScrapers.map((s) => { + return { + id: `${SCRAPER_PREFIX}${s.id}`, + displayName: s.name, + scraper_id: s.id, + }; + }) + ); + + return ret; + }, [configData, scraperData]); + + const selectionStatus = useMemo(() => { + if (selectedIds) { + return ( + + + . + + ); + } + const message = paths.length ? ( +
+ : +
    + {paths.map((p) => ( +
  • {p}
    • + ))} +
+
+ ) : ( + + + . + + ); + + function onClick() { + setAnimation(false); + setSettingPaths(true); + } + + return ( + +
+ {message} +
+ +
+
+ + ); + }, [selectedIds, intl, paths]); + + useEffect(() => { + if (!configData || !allSources) return; + + const { identify: identifyDefaults } = configData.configuration.defaults; + + if (identifyDefaults) { + const mappedSources = identifyDefaults.sources + .map((s) => { + const found = allSources.find( + (ss) => + ss.scraper_id === s.source.scraper_id || + ss.stash_box_endpoint === s.source.stash_box_endpoint + ); + + if (!found) return; + + const ret: IScraperSource = { + ...found, + }; + + if (s.options) { + const sourceOptions = withoutTypename(s.options); + sourceOptions.fieldOptions = sourceOptions.fieldOptions?.map( + withoutTypename + ); + ret.options = sourceOptions; + } + + return ret; + }) + .filter((s) => s) as IScraperSource[]; + + setSources(mappedSources); + if (identifyDefaults.options) { + const defaultOptions = withoutTypename(identifyDefaults.options); + defaultOptions.fieldOptions = defaultOptions.fieldOptions?.map( + withoutTypename + ); + setOptions(defaultOptions); + } + } else { + // default to first stash-box instance only + const stashBox = allSources.find((s) => s.stash_box_endpoint); + + // add auto-tag as well + const autoTag = allSources.find( + (s) => s.id === `${SCRAPER_PREFIX}${autoTagScraperID}` + ); + + const newSources: IScraperSource[] = []; + if (stashBox) { + newSources.push(stashBox); + } + + // sanity check - this should always be true + if (autoTag) { + // don't set organised by default + const autoTagCopy = { ...autoTag }; + autoTagCopy.options = { + setOrganized: false, + }; + newSources.push(autoTagCopy); + } + + setSources(newSources); + } + }, [allSources, configData]); + + if (configError || scraperError) + return
{configError ?? scraperError}
; + if (!allSources || !configData) return
; + + function makeIdentifyInput(): GQL.IdentifyMetadataInput { + return { + sources: sources.map((s) => { + return { + source: { + scraper_id: s.scraper_id, + stash_box_endpoint: s.stash_box_endpoint, + }, + options: s.options, + }; + }), + options, + sceneIDs: selectedIds, + paths, + }; + } + + function makeDefaultIdentifyInput() { + const ret = makeIdentifyInput(); + const { sceneIDs, paths: _paths, ...withoutSpecifics } = ret; + return withoutSpecifics; + } + + async function onIdentify() { + try { + await mutateMetadataIdentify(makeIdentifyInput()); + + Toast.success({ + content: intl.formatMessage( + { id: "config.tasks.added_job_to_queue" }, + { operation_name: intl.formatMessage({ id: "actions.identify" }) } + ), + }); + } catch (e) { + Toast.error(e); + } finally { + onClose(); + } + } + + function getAvailableSources() { + // only include scrapers not already present + return !editingSource?.id === undefined + ? [] + : allSources?.filter((s) => { + return !sources.some((ss) => ss.id === s.id); + }) ?? []; + } + + function onEditSource(s?: IScraperSource) { + setAnimation(false); + + // if undefined, then set a dummy source to create a new one + if (!s) { + setEditingSource(getAvailableSources()[0]); + } else { + setEditingSource(s); + } + } + + function onShowManual() { + setAnimation(false); + setShowManual(true); + } + + function isNewSource() { + return !!editingSource && !sources.includes(editingSource); + } + + function onSaveSource(s?: IScraperSource) { + if (s) { + let found = false; + const newSources = sources.map((ss) => { + if (ss.id === s.id) { + found = true; + return s; + } + return ss; + }); + + if (!found) { + newSources.push(s); + } + + setSources(newSources); + } + setEditingSource(undefined); + } + + async function setAsDefault() { + try { + setSavingDefaults(true); + await configureDefaults({ + variables: { + input: { + identify: makeDefaultIdentifyInput(), + }, + }, + }); + } catch (e) { + Toast.error(e); + } finally { + setSavingDefaults(false); + } + } + + if (editingSource) { + return ( + + ); + } + + if (settingPaths) { + return ( + { + if (p) { + setPaths(p); + } + setSettingPaths(false); + }} + /> + ); + } + + if (showManual) { + return ( + setShowManual(false)} + defaultActiveTab="Identify.md" + /> + ); + } + + return ( + onClose(), + text: intl.formatMessage({ id: "actions.cancel" }), + variant: "secondary", + }} + disabled={editingField || savingDefaults || sources.length === 0} + footerButtons={ + + } + leftFooterButtons={ + + } + > +
+ {selectionStatus} + setSources(s)} + editSource={onEditSource} + canAdd={sources.length < allSources.length} + /> + setOptions(o)} + setEditingField={(v) => setEditingField(v)} + /> + +
+ ); +}; + +export default IdentifyDialog; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx new file mode 100644 index 000000000..88655c860 --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx @@ -0,0 +1,117 @@ +import React from "react"; +import { Form } from "react-bootstrap"; +import * as GQL from "src/core/generated-graphql"; +import { FormattedMessage, useIntl } from "react-intl"; +import { IScraperSource } from "./constants"; +import { FieldOptionsList } from "./FieldOptions"; +import { ThreeStateBoolean } from "./ThreeStateBoolean"; + +interface IOptionsEditor { + options: GQL.IdentifyMetadataOptionsInput; + setOptions: (s: GQL.IdentifyMetadataOptionsInput) => void; + source?: IScraperSource; + defaultOptions?: GQL.IdentifyMetadataOptionsInput; + setEditingField: (v: boolean) => void; +} + +export const OptionsEditor: React.FC = ({ + options, + setOptions: setOptionsState, + source, + setEditingField, + defaultOptions, +}) => { + const intl = useIntl(); + + function setOptions(v: Partial) { + setOptionsState({ ...options, ...v }); + } + + const headingID = !source + ? "config.tasks.identify.default_options" + : "config.tasks.identify.source_options"; + const checkboxProps = { + allowUndefined: !!source, + indeterminateClassname: "text-muted", + }; + + return ( + + +
+ +
+ {!source && ( + + {intl.formatMessage({ + id: "config.tasks.identify.explicit_set_description", + })} + + )} + + + + setOptions({ + includeMalePerformers: v, + }) + } + label={intl.formatMessage({ + id: "config.tasks.identify.include_male_performers", + })} + defaultValue={defaultOptions?.includeMalePerformers ?? undefined} + {...checkboxProps} + /> + + setOptions({ + setCoverImage: v, + }) + } + label={intl.formatMessage({ + id: "config.tasks.identify.set_cover_images", + })} + defaultValue={defaultOptions?.setCoverImage ?? undefined} + {...checkboxProps} + /> + + setOptions({ + setOrganized: v, + }) + } + label={intl.formatMessage({ + id: "config.tasks.identify.set_organized", + })} + defaultValue={defaultOptions?.setOrganized ?? undefined} + {...checkboxProps} + /> + + + setOptions({ fieldOptions: o })} + setEditingField={setEditingField} + allowSetDefault={!!source} + defaultOptions={defaultOptions} + /> + + ); +}; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/Sources.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Sources.tsx new file mode 100644 index 000000000..127adbdc4 --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Sources.tsx @@ -0,0 +1,216 @@ +import React, { useState, useEffect } from "react"; +import { Form, Button, ListGroup } from "react-bootstrap"; +import { Modal, Icon } from "src/components/Shared"; +import { FormattedMessage, useIntl } from "react-intl"; +import * as GQL from "src/core/generated-graphql"; +import { IScraperSource } from "./constants"; +import { OptionsEditor } from "./Options"; + +interface ISourceEditor { + isNew: boolean; + availableSources: IScraperSource[]; + source: IScraperSource; + saveSource: (s?: IScraperSource) => void; + defaultOptions: GQL.IdentifyMetadataOptionsInput; +} + +export const SourcesEditor: React.FC = ({ + isNew, + availableSources, + source: initialSource, + saveSource, + defaultOptions, +}) => { + const [source, setSource] = useState(initialSource); + const [editingField, setEditingField] = useState(false); + + const intl = useIntl(); + + // if id is empty, then we are adding a new source + const headerMsgId = isNew ? "actions.add" : "dialogs.edit_entity_title"; + const acceptMsgId = isNew ? "actions.add" : "actions.confirm"; + + function handleSourceSelect(e: React.ChangeEvent) { + const selectedSource = availableSources.find( + (s) => s.id === e.currentTarget.value + ); + if (!selectedSource) return; + + setSource({ + ...source, + id: selectedSource.id, + displayName: selectedSource.displayName, + scraper_id: selectedSource.scraper_id, + stash_box_endpoint: selectedSource.stash_box_endpoint, + }); + } + + return ( + saveSource(source), + text: intl.formatMessage({ id: acceptMsgId }), + }} + cancel={{ + onClick: () => saveSource(), + text: intl.formatMessage({ id: "actions.cancel" }), + variant: "secondary", + }} + disabled={ + (!source.scraper_id && !source.stash_box_endpoint) || editingField + } + > +
+ {isNew && ( + +
+ +
+ + {availableSources.map((i) => ( + + ))} + + + )} + setSource({ ...source, options: o })} + source={source} + setEditingField={(v) => setEditingField(v)} + defaultOptions={defaultOptions} + /> + +
+ ); +}; + +interface ISourcesList { + sources: IScraperSource[]; + setSources: (s: IScraperSource[]) => void; + editSource: (s?: IScraperSource) => void; + canAdd: boolean; +} + +export const SourcesList: React.FC = ({ + sources, + setSources, + editSource, + canAdd, +}) => { + const [tempSources, setTempSources] = useState(sources); + const [dragIndex, setDragIndex] = useState(); + const [mouseOverIndex, setMouseOverIndex] = useState(); + + useEffect(() => { + setTempSources([...sources]); + }, [sources]); + + function removeSource(index: number) { + const newSources = [...sources]; + newSources.splice(index, 1); + setSources(newSources); + } + + function onDragStart(event: React.DragEvent, index: number) { + event.dataTransfer.effectAllowed = "move"; + setDragIndex(index); + } + + function onDragOver(event: React.DragEvent, index?: number) { + if (dragIndex !== undefined && index !== undefined && index !== dragIndex) { + const newSources = [...tempSources]; + const moved = newSources.splice(dragIndex, 1); + newSources.splice(index, 0, moved[0]); + setTempSources(newSources); + setDragIndex(index); + } + + event.dataTransfer.dropEffect = "move"; + event.preventDefault(); + } + + function onDragOverDefault(event: React.DragEvent) { + event.dataTransfer.dropEffect = "move"; + event.preventDefault(); + } + + function onDrop() { + // assume we've already set the temp source list + // feed it up + setSources(tempSources); + setDragIndex(undefined); + setMouseOverIndex(undefined); + } + + return ( + +
+ +
+ + {tempSources.map((s, index) => ( + onDragStart(e, index)} + onDragEnter={(e) => onDragOver(e, index)} + onDrop={() => onDrop()} + > +
+
setMouseOverIndex(index)} + onMouseLeave={() => setMouseOverIndex(undefined)} + > + +
+ {s.displayName} +
+
+ + +
+ + ))} + + {canAdd && ( +
+ +
+ )} + + ); +}; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/ThreeStateBoolean.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/ThreeStateBoolean.tsx new file mode 100644 index 000000000..636b613b6 --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/ThreeStateBoolean.tsx @@ -0,0 +1,90 @@ +import React from "react"; +import { Form } from "react-bootstrap"; +import { useIntl } from "react-intl"; + +interface IThreeStateBoolean { + id: string; + value: boolean | undefined; + setValue: (v: boolean | undefined) => void; + allowUndefined?: boolean; + label?: React.ReactNode; + disabled?: boolean; + defaultValue?: boolean; +} + +export const ThreeStateBoolean: React.FC = ({ + id, + value, + setValue, + allowUndefined = true, + label, + disabled, + defaultValue, +}) => { + const intl = useIntl(); + + if (!allowUndefined) { + return ( + setValue(!value)} + /> + ); + } + + function getBooleanText(v: boolean) { + if (v) { + return intl.formatMessage({ id: "true" }); + } + return intl.formatMessage({ id: "false" }); + } + + function getButtonText(v: boolean | undefined) { + if (v === undefined) { + const defaultVal = + defaultValue !== undefined ? ( + + {" "} + ({getBooleanText(defaultValue)}) + + ) : ( + "" + ); + return ( + + {intl.formatMessage({ id: "use_default" })} + {defaultVal} + + ); + } + + return getBooleanText(v); + } + + function renderModeButton(v: boolean | undefined) { + return ( + setValue(v)} + disabled={disabled} + label={getButtonText(v)} + /> + ); + } + + return ( + +
{label}
+ + {renderModeButton(undefined)} + {renderModeButton(false)} + {renderModeButton(true)} + + + ); +}; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/constants.ts b/ui/v2.5/src/components/Dialogs/IdentifyDialog/constants.ts new file mode 100644 index 000000000..11c7fe6e8 --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/constants.ts @@ -0,0 +1,27 @@ +import * as GQL from "src/core/generated-graphql"; + +export interface IScraperSource { + id: string; + displayName: string; + stash_box_endpoint?: string; + scraper_id?: string; + options?: GQL.IdentifyMetadataOptionsInput; +} + +export const sceneFields = [ + "title", + "date", + "details", + "url", + "studio", + "performers", + "tags", + "stash_ids", +] as const; +export type SceneField = typeof sceneFields[number]; + +export const multiValueSceneFields: SceneField[] = [ + "studio", + "performers", + "tags", +]; diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/styles.scss b/ui/v2.5/src/components/Dialogs/IdentifyDialog/styles.scss new file mode 100644 index 000000000..9257d0b1d --- /dev/null +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/styles.scss @@ -0,0 +1,45 @@ +.identify-source-editor { + .default-value { + color: #bfccd6; + } +} + +.scraper-source-list { + .list-group-item { + background-color: $textfield-bg; + padding: 0.25em; + + .drag-handle { + cursor: move; + display: inline-block; + margin: -0.25em 0.25em -0.25em -0.25em; + padding: 0.25em 0.5em 0.25em; + } + + .drag-handle:hover, + .drag-handle:active, + .drag-handle:focus, + .drag-handle:focus:active { + background-color: initial; + border-color: initial; + box-shadow: initial; + } + } +} + +.scraper-sources { + .add-scraper-source-button { + margin-right: 0.25em; + } +} + +.field-options-table td:first-child { + padding-left: 0.75rem; +} + +#selected-identify-folders { + & > div { + display: flex; + justify-content: space-between; + } +} diff --git a/ui/v2.5/src/components/Help/Manual.tsx b/ui/v2.5/src/components/Help/Manual.tsx index 7455f79b5..af6b2c03a 100644 --- a/ui/v2.5/src/components/Help/Manual.tsx +++ b/ui/v2.5/src/components/Help/Manual.tsx @@ -18,15 +18,18 @@ import KeyboardShortcuts from "src/docs/en/KeyboardShortcuts.md"; import Help from "src/docs/en/Help.md"; import Deduplication from "src/docs/en/Deduplication.md"; import Interactive from "src/docs/en/Interactive.md"; +import Identify from "src/docs/en/Identify.md"; import { MarkdownPage } from "../Shared/MarkdownPage"; interface IManualProps { + animation?: boolean; show: boolean; onClose: () => void; defaultActiveTab?: string; } export const Manual: React.FC = ({ + animation, show, onClose, defaultActiveTab, @@ -52,6 +55,12 @@ export const Manual: React.FC = ({ title: "Tasks", content: Tasks, }, + { + key: "Identify.md", + title: "Identify", + content: Identify, + className: "indent-1", + }, { key: "AutoTagging.md", title: "Auto Tagging", @@ -152,6 +161,7 @@ export const Manual: React.FC = ({ return ( ListFilterModel; @@ -38,6 +39,7 @@ export const SceneList: React.FC = ({ const intl = useIntl(); const history = useHistory(); const [isGenerateDialogOpen, setIsGenerateDialogOpen] = useState(false); + const [isIdentifyDialogOpen, setIsIdentifyDialogOpen] = useState(false); const [isExportDialogOpen, setIsExportDialogOpen] = useState(false); const [isExportAll, setIsExportAll] = useState(false); @@ -53,10 +55,15 @@ export const SceneList: React.FC = ({ onClick: playRandom, }, { - text: intl.formatMessage({ id: "actions.generate" }), + text: `${intl.formatMessage({ id: "actions.generate" })}…`, onClick: generate, isDisplayed: showWhenSelected, }, + { + text: `${intl.formatMessage({ id: "actions.identify" })}…`, + onClick: identify, + isDisplayed: showWhenSelected, + }, { text: intl.formatMessage({ id: "actions.export" }), onClick: onExport, @@ -138,6 +145,10 @@ export const SceneList: React.FC = ({ setIsGenerateDialogOpen(true); } + async function identify() { + setIsIdentifyDialogOpen(true); + } + async function onExport() { setIsExportAll(false); setIsExportDialogOpen(true); @@ -163,6 +174,21 @@ export const SceneList: React.FC = ({ } } + function maybeRenderSceneIdentifyDialog(selectedIds: Set) { + if (isIdentifyDialogOpen) { + return ( + <> + { + setIsIdentifyDialogOpen(false); + }} + /> + + ); + } + } + function maybeRenderSceneExportDialog(selectedIds: Set) { if (isExportDialogOpen) { return ( @@ -248,6 +274,7 @@ export const SceneList: React.FC = ({ return ( <> {maybeRenderSceneGenerateDialog(selectedIds)} + {maybeRenderSceneIdentifyDialog(selectedIds)} {maybeRenderSceneExportDialog(selectedIds)} {renderScenes(result, filter, selectedIds)} diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx index 0da409d19..a3ba44c03 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx @@ -6,18 +6,24 @@ import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect"; import { ConfigurationContext } from "src/hooks/Config"; interface IDirectorySelectionDialogProps { + animation?: boolean; + initialPaths?: string[]; + allowEmpty?: boolean; onClose: (paths?: string[]) => void; } -export const DirectorySelectionDialog: React.FC = ( - props: IDirectorySelectionDialogProps -) => { +export const DirectorySelectionDialog: React.FC = ({ + animation, + allowEmpty = false, + initialPaths = [], + onClose, +}) => { const intl = useIntl(); const { configuration } = React.useContext(ConfigurationContext); const libraryPaths = configuration?.general.stashes.map((s) => s.path); - const [paths, setPaths] = useState([]); + const [paths, setPaths] = useState(initialPaths); const [currentDirectory, setCurrentDirectory] = useState(""); function removePath(p: string) { @@ -33,17 +39,18 @@ export const DirectorySelectionDialog: React.FC return ( { - props.onClose(paths); + onClose(paths); }, text: intl.formatMessage({ id: "actions.confirm" }), }} cancel={{ - onClick: () => props.onClose(), + onClick: () => onClose(), text: intl.formatMessage({ id: "actions.cancel" }), variant: "secondary", }} diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index 8ae9a62ef..f7f53f387 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -16,6 +16,7 @@ import { useToast } from "src/hooks"; import * as GQL from "src/core/generated-graphql"; import { LoadingIndicator, Modal } from "src/components/Shared"; import { downloadFile } from "src/utils"; +import IdentifyDialog from "src/components/Dialogs/IdentifyDialog/IdentifyDialog"; import { GenerateButton } from "./GenerateButton"; import { ImportDialog } from "./ImportDialog"; import { DirectorySelectionDialog } from "./DirectorySelectionDialog"; @@ -27,13 +28,18 @@ type PluginTask = Pick; export const SettingsTasksPanel: React.FC = () => { const intl = useIntl(); const Toast = useToast(); - const [isImportAlertOpen, setIsImportAlertOpen] = useState(false); - const [isCleanAlertOpen, setIsCleanAlertOpen] = useState(false); - const [isImportDialogOpen, setIsImportDialogOpen] = useState(false); - const [isScanDialogOpen, setIsScanDialogOpen] = useState(false); - const [isAutoTagDialogOpen, setIsAutoTagDialogOpen] = useState( - false - ); + const [dialogOpen, setDialogOpenState] = useState({ + importAlert: false, + cleanAlert: false, + import: false, + clean: false, + scan: false, + autoTag: false, + identify: false, + }); + + type DialogOpenState = typeof dialogOpen; + const [isBackupRunning, setIsBackupRunning] = useState(false); const [useFileMetadata, setUseFileMetadata] = useState(false); const [stripFileExtension, setStripFileExtension] = useState(false); @@ -61,8 +67,14 @@ export const SettingsTasksPanel: React.FC = () => { const plugins = usePlugins(); + function setDialogOpen(s: Partial) { + setDialogOpenState((v) => { + return { ...v, ...s }; + }); + } + async function onImport() { - setIsImportAlertOpen(false); + setDialogOpen({ importAlert: false }); try { await mutateMetadataImport(); Toast.success({ @@ -79,14 +91,14 @@ export const SettingsTasksPanel: React.FC = () => { function renderImportAlert() { return ( setIsImportAlertOpen(false) }} + cancel={{ onClick: () => setDialogOpen({ importAlert: false }) }} >

{intl.formatMessage({ id: "actions.tasks.import_warning" })}

 @@ -94,7 +106,7 @@ export const SettingsTasksPanel: React.FC = () => { } function onClean() { - setIsCleanAlertOpen(false); + setDialogOpen({ cleanAlert: false }); mutateMetadataClean({ dryRun: cleanDryRun, }); @@ -116,14 +128,14 @@ export const SettingsTasksPanel: React.FC = () => { return ( setIsCleanAlertOpen(false) }} + cancel={{ onClick: () => setDialogOpen({ cleanAlert: false }) }} > {msg} @@ -131,15 +143,15 @@ export const SettingsTasksPanel: React.FC = () => { } function renderImportDialog() { - if (!isImportDialogOpen) { + if (!dialogOpen.import) { return; } - return setIsImportDialogOpen(false)} />; + return setDialogOpen({ import: false })} />; } function renderScanDialog() { - if (!isScanDialogOpen) { + if (!dialogOpen.scan) { return; } @@ -151,7 +163,7 @@ export const SettingsTasksPanel: React.FC = () => { onScan(paths); } - setIsScanDialogOpen(false); + setDialogOpen({ scan: false }); } async function onScan(paths?: string[]) { @@ -178,19 +190,27 @@ export const SettingsTasksPanel: React.FC = () => { } function renderAutoTagDialog() { - if (!isAutoTagDialogOpen) { + if (!dialogOpen.autoTag) { return; } return ; } + function maybeRenderIdentifyDialog() { + if (!dialogOpen.identify) return; + + return ( + setDialogOpen({ identify: false })} /> + ); + } + function onAutoTagDialogClosed(paths?: string[]) { if (paths) { onAutoTag(paths); } - setIsAutoTagDialogOpen(false); + setDialogOpen({ autoTag: false }); } function getAutoTagInput(paths?: string[]) { @@ -343,6 +363,7 @@ export const SettingsTasksPanel: React.FC = () => { {renderImportDialog()} {renderScanDialog()} {renderAutoTagDialog()} + {maybeRenderIdentifyDialog()}

{intl.formatMessage({ id: "config.tasks.job_queue" })}

@@ -350,138 +371,159 @@ export const SettingsTasksPanel: React.FC = () => {
-
{intl.formatMessage({ id: "library" })}
- setUseFileMetadata(!useFileMetadata)} - /> - setStripFileExtension(!stripFileExtension)} - /> - setScanGeneratePreviews(!scanGeneratePreviews)} - /> -
-
↳
+
{intl.formatMessage({ id: "library" })}
+ +
{intl.formatMessage({ id: "actions.scan" })}
- setScanGenerateImagePreviews(!scanGenerateImagePreviews) - } - className="ml-2 flex-grow" + onChange={() => setUseFileMetadata(!useFileMetadata)} /> -
- setScanGenerateSprites(!scanGenerateSprites)} - /> - setScanGeneratePhashes(!scanGeneratePhashes)} - /> - setScanGenerateThumbnails(!scanGenerateThumbnails)} - /> - - - - - - {intl.formatMessage({ id: "config.tasks.scan_for_content_desc" })} - - + setStripFileExtension(!stripFileExtension)} + /> + setScanGeneratePreviews(!scanGeneratePreviews)} + /> +
+
↳
+ + setScanGenerateImagePreviews(!scanGenerateImagePreviews) + } + className="ml-2 flex-grow" + /> +
+ setScanGenerateSprites(!scanGenerateSprites)} + /> + setScanGeneratePhashes(!scanGeneratePhashes)} + /> + setScanGenerateThumbnails(!scanGenerateThumbnails)} + /> + + + + + + {intl.formatMessage({ id: "config.tasks.scan_for_content_desc" })} + + -
+ +
+ +
+ + + + + -
{intl.formatMessage({ id: "config.tasks.auto_tagging" })}
+ +
{intl.formatMessage({ id: "config.tasks.auto_tagging" })}
- - setAutoTagPerformers(!autoTagPerformers)} - /> - setAutoTagStudios(!autoTagStudios)} - /> - setAutoTagTags(!autoTagTags)} - /> - - - - - - {intl.formatMessage({ - id: "config.tasks.auto_tag_based_on_filenames", - })} - + + setAutoTagPerformers(!autoTagPerformers)} + /> + setAutoTagStudios(!autoTagStudios)} + /> + setAutoTagTags(!autoTagTags)} + /> + + + + + + {intl.formatMessage({ + id: "config.tasks.auto_tag_based_on_filenames", + })} + + +
@@ -503,7 +545,7 @@ export const SettingsTasksPanel: React.FC = () => { @@ -533,7 +575,7 @@ export const SettingsTasksPanel: React.FC = () => { @@ -546,7 +588,7 @@ export const SettingsTasksPanel: React.FC = () => { diff --git a/ui/v2.5/src/components/Shared/FolderSelect/FolderSelect.tsx b/ui/v2.5/src/components/Shared/FolderSelect/FolderSelect.tsx index b0e43541a..6ceccb7b2 100644 --- a/ui/v2.5/src/components/Shared/FolderSelect/FolderSelect.tsx +++ b/ui/v2.5/src/components/Shared/FolderSelect/FolderSelect.tsx @@ -1,6 +1,7 @@ -import React, { useEffect } from "react"; +import React, { useEffect, useState, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Button, InputGroup, Form } from "react-bootstrap"; +import { debounce } from "lodash"; import { LoadingIndicator } from "src/components/Shared"; import { useDirectory } from "src/core/StashService"; @@ -17,22 +18,43 @@ export const FolderSelect: React.FC = ({ defaultDirectories, appendButton, }) => { - const { data, error, loading } = useDirectory(currentDirectory); + const [debouncedDirectory, setDebouncedDirectory] = useState( + currentDirectory + ); + const { data, error, loading } = useDirectory(debouncedDirectory); const selectableDirectories: string[] = currentDirectory ? data?.directory.directories ?? defaultDirectories ?? [] : defaultDirectories ?? []; + const debouncedSetDirectory = useMemo( + () => + debounce((input: string) => { + setDebouncedDirectory(input); + }, 250), + [] + ); + useEffect(() => { if (currentDirectory === "" && !defaultDirectories && data?.directory.path) setCurrentDirectory(data.directory.path); }, [currentDirectory, setCurrentDirectory, data, defaultDirectories]); + function setInstant(value: string) { + setCurrentDirectory(value); + setDebouncedDirectory(value); + } + + function setDebounced(value: string) { + setCurrentDirectory(value); + debouncedSetDirectory(value); + } + function goUp() { if (defaultDirectories?.includes(currentDirectory)) { - setCurrentDirectory(""); + setInstant(""); } else if (data?.directory.parent) { - setCurrentDirectory(data.directory.parent); + setInstant(data.directory.parent); } } @@ -51,9 +73,9 @@ export const FolderSelect: React.FC = ({ ) => - setCurrentDirectory(e.currentTarget.value) - } + onChange={(e: React.ChangeEvent) => { + setDebounced(e.currentTarget.value); + }} value={currentDirectory} spellCheck={false} /> @@ -71,7 +93,7 @@ export const FolderSelect: React.FC = ({ {selectableDirectories.map((path) => { return (
  • -
    • diff --git a/ui/v2.5/src/components/Shared/IndeterminateCheckbox.tsx b/ui/v2.5/src/components/Shared/IndeterminateCheckbox.tsx new file mode 100644 index 000000000..d38bb31f4 --- /dev/null +++ b/ui/v2.5/src/components/Shared/IndeterminateCheckbox.tsx @@ -0,0 +1,55 @@ +import React, { useEffect } from "react"; +import { Form, FormCheckProps } from "react-bootstrap"; + +const useIndeterminate = ( + ref: React.RefObject, + value: boolean | undefined +) => { + useEffect(() => { + if (ref.current) { + // eslint-disable-next-line no-param-reassign + ref.current.indeterminate = value === undefined; + } + }, [ref, value]); +}; + +interface IIndeterminateCheckbox extends FormCheckProps { + setChecked: (v: boolean | undefined) => void; + allowIndeterminate?: boolean; + indeterminateClassname?: string; +} + +export const IndeterminateCheckbox: React.FC = ({ + checked, + setChecked, + allowIndeterminate, + indeterminateClassname, + ...props +}) => { + const ref = React.createRef(); + + useIndeterminate(ref, checked); + + function cycleState() { + const undefAllowed = allowIndeterminate ?? true; + if (undefAllowed && checked) { + return undefined; + } + if ((!undefAllowed && checked) || checked === undefined) { + return false; + } + return true; + } + + return ( + setChecked(cycleState())} + /> + ); +}; diff --git a/ui/v2.5/src/components/Shared/Modal.tsx b/ui/v2.5/src/components/Shared/Modal.tsx index 5b032ac61..1ff9a1a50 100644 --- a/ui/v2.5/src/components/Shared/Modal.tsx +++ b/ui/v2.5/src/components/Shared/Modal.tsx @@ -21,6 +21,8 @@ interface IModal { disabled?: boolean; modalProps?: ModalProps; dialogClassName?: string; + footerButtons?: React.ReactNode; + leftFooterButtons?: React.ReactNode; } const defaultOnHide = () => {}; @@ -37,8 +39,11 @@ const ModalComponent: React.FC = ({ disabled, modalProps, dialogClassName, + footerButtons, + leftFooterButtons, }) => ( = ({ {header ?? ""} {children} - + +
    {leftFooterButtons}
    + {footerButtons} {cancel ? ( + {label} + + ); +}; diff --git a/ui/v2.5/src/components/Shared/index.ts b/ui/v2.5/src/components/Shared/index.ts index ae124793d..84a943751 100644 --- a/ui/v2.5/src/components/Shared/index.ts +++ b/ui/v2.5/src/components/Shared/index.ts @@ -17,5 +17,6 @@ export { GridCard } from "./GridCard"; export { RatingStars } from "./RatingStars"; export { ExportDialog } from "./ExportDialog"; export { default as DeleteEntityDialog } from "./DeleteEntityDialog"; +export { IndeterminateCheckbox } from "./IndeterminateCheckbox"; export { OperationButton } from "./OperationButton"; export const TITLE_SUFFIX = " | Stash"; diff --git a/ui/v2.5/src/components/Shared/styles.scss b/ui/v2.5/src/components/Shared/styles.scss index f814a7458..14021325e 100644 --- a/ui/v2.5/src/components/Shared/styles.scss +++ b/ui/v2.5/src/components/Shared/styles.scss @@ -209,9 +209,48 @@ button.collapse-button.btn-primary:not(:disabled):not(.disabled):active { } } +.three-state-checkbox { + align-items: center; + display: flex; + + button.btn { + font-size: 12.67px; + margin-left: -0.2em; + margin-right: 0.25rem; + padding: 0; + + &:not(:disabled):active, + &:not(:disabled):active:focus, + &:not(:disabled):hover, + &:not(:disabled):not(:hover) { + background-color: initial; + box-shadow: none; + } + } + + &.unset { + .label { + color: #bfccd6; + text-decoration: line-through; + } + } + + &.checked svg { + color: #0f9960; + } + + &.not-checked svg { + color: #db3737; + } +} + .input-group-prepend { .btn { border-bottom-right-radius: 0; border-top-right-radius: 0; } } + +.ModalComponent .modal-footer { + justify-content: space-between; +} diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 97f9d94d3..21714b936 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -757,6 +757,12 @@ export const useGenerateAPIKey = () => update: deleteCache([GQL.ConfigurationDocument]), }); +export const useConfigureDefaults = () => + GQL.useConfigureDefaultsMutation({ + refetchQueries: getQueryNames([GQL.ConfigurationDocument]), + update: deleteCache([GQL.ConfigurationDocument]), + }); + export const useJobsSubscribe = () => GQL.useJobsSubscribeSubscription(); export const useConfigureDLNA = () => @@ -1001,6 +1007,12 @@ export const mutateMetadataClean = (input: GQL.CleanMetadataInput) => variables: { input }, }); +export const mutateMetadataIdentify = (input: GQL.IdentifyMetadataInput) => + client.mutate({ + mutation: GQL.MetadataIdentifyDocument, + variables: { input }, + }); + export const mutateMigrateHashNaming = () => client.mutate({ mutation: GQL.MigrateHashNamingDocument, diff --git a/ui/v2.5/src/docs/en/Identify.md b/ui/v2.5/src/docs/en/Identify.md new file mode 100644 index 000000000..dceb8c1dc --- /dev/null +++ b/ui/v2.5/src/docs/en/Identify.md @@ -0,0 +1,31 @@ +# Identify + +This task iterates through your Scenes and attempts to identify the scene using a selection of scraping sources. + +This task accepts one or more scraper sources. Valid scraper sources for the Identify task are stash-box instances, and scene scrapers which support scraping via Scene Fragment. The order of the sources may be rearranged. + +For each Scene, the Identify task iterates through the scraper sources, in the order provided, and tries to identify the scene using each source. If a result is found in a source, then the Scene is updated, and no further sources are checked for that scene. + +## Options + +The following options can be set: + +| Option | Description | +|--------|-------------| +| Include male performers | If false, then male performers will not be created or set on scenes. | +| Set cover images | If false, then scene cover images will not be modified. | +| Set organised flag | If true, the organised flag is set to true when a scene is organised. | + +Field specific options may be set as well. Each field may have a Strategy. The behaviour for each strategy value is as follows: + +| Strategy | Description | +|----------|-------------| +| Ignore | Not set. | +| Overwrite | Overwrite existing value. | +| Merge (*default*) | For multi-value fields, adds to existing values. For single-value fields, only sets if not already set. | + +For Studio, Performers and Tags, an option is also available to Create Missing objects. This is false by default. When true, if a Studio/Performer/Tag is included during the identification process and does not exist in the system, then it will be created. + +Default Options are applied to all sources unless overridden in specific source options. + +The result of the identification process for each scene is output to the log. diff --git a/ui/v2.5/src/index.scss b/ui/v2.5/src/index.scss index 220228c2d..4d0f61808 100755 --- a/ui/v2.5/src/index.scss +++ b/ui/v2.5/src/index.scss @@ -20,6 +20,7 @@ @import "../node_modules/flag-icon-css/css/flag-icon.min.css"; @import "src/components/Tagger/styles.scss"; @import "src/hooks/Lightbox/lightbox.scss"; +@import "src/components/Dialogs/IdentifyDialog/styles.scss"; /* stylelint-disable */ #root { diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 6066f4de6..6b654d528 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -41,6 +41,7 @@ "generate_thumb_from_current": "Generate thumbnail from current", "hash_migration": "hash migration", "hide": "Hide", + "identify": "Identify", "import": "Import…", "import_from_file": "Import from file", "merge": "Merge", @@ -96,6 +97,7 @@ "actions_name": "Actions", "age": "Age", "aliases": "Aliases", + "all": "all", "also_known_as": "Also known as", "ascending": "Ascending", "average_resolution": "Average Resolution", @@ -271,6 +273,7 @@ "entity_scrapers": "{entityType} scrapers", "excluded_tag_patterns_desc": "Regexps of tag names to exclude from scraping results", "excluded_tag_patterns_head": "Excluded Tag Patterns", + "scraper": "Scraper", "scrapers": "Scrapers", "search_by_name": "Search by name", "supported_types": "Supported types", @@ -302,6 +305,29 @@ "generate_video_previews_during_scan": "Generate previews during scan (video previews which play when hovering over a scene)", "generate_thumbnails_during_scan": "Generate thumbnails for images during scan.", "generated_content": "Generated Content", + "identify": { + "and_create_missing": "and create missing", + "create_missing": "Create missing", + "heading": "Identify", + "description": "Automatically set scene metadata using stash-box and scraper sources.", + "default_options": "Default Options", + "explicit_set_description": "The following options will be used where not overridden in the source-specific options.", + "field_behaviour": "{strategy} {field}", + "field_options": "Field Options", + "field_strategies": { + "ignore": "Ignore", + "merge": "Merge", + "overwrite": "Overwrite" + }, + "identifying_scenes": "Identifying {num} {scene}", + "identifying_from_paths": "Identifying scenes from the following paths", + "include_male_performers": "Include male performers", + "set_cover_images": "Set cover images", + "set_organized": "Set organised flag", + "source_options": "{source} Options", + "source": "Source", + "sources": "Sources" + }, "import_from_exported_json": "Import from exported JSON in the metadata directory. Wipes the existing database.", "incremental_import": "Incremental import from a supplied export zip file.", "job_queue": "Job Queue", @@ -572,6 +598,7 @@ "ethnicity": "Ethnicity", "eye_color": "Eye Colour", "fake_tits": "Fake Tits", + "false": "False", "favourite": "Favourite", "file_info": "File Info", "file_mod_time": "File Modification Time", @@ -668,6 +695,7 @@ "settings": "Settings", "sub_tag_of": "Sub-tag of {parent}", "stash_id": "Stash ID", + "stash_ids": "Stash IDs", "status": "Status: {statusText}", "studio": "Studio", "studio_depth": "Levels (empty for all)", @@ -693,10 +721,12 @@ "updated_entity": "Updated {entity}" }, "total": "Total", + "true": "True", "twitter": "Twitter", "up-dir": "Up a directory", "updated_at": "Updated At", "url": "URL", + "use_default": "Use default", "weight": "Weight", "years_old": "years old", "stats": { diff --git a/ui/v2.5/src/utils/data.ts b/ui/v2.5/src/utils/data.ts index b6b12ca0e..19ba59092 100644 --- a/ui/v2.5/src/utils/data.ts +++ b/ui/v2.5/src/utils/data.ts @@ -1,2 +1,11 @@ export const filterData = (data?: (T | null | undefined)[] | null) => data ? (data.filter((item) => item) as T[]) : []; + +interface ITypename { + __typename?: string; +} + +export function withoutTypename(o: T) { + const { __typename, ...ret } = o; + return ret; +}