From 86abe7b24c79fd82fe7eb4543e11e1cace8a5182 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 24 Feb 2026 07:41:40 +1100 Subject: [PATCH] Backend support for image custom fields (#6598) * Initialise maps in bulk get custom fields to fix graphql validation error --- graphql/schema/types/filters.graphql | 2 + graphql/schema/types/image.graphql | 3 + internal/api/loaders/dataloaders.go | 18 + internal/api/resolver_model_image.go | 9 + internal/api/resolver_mutation_image.go | 14 + internal/autotag/integration_test.go | 5 +- internal/manager/task_export.go | 8 +- pkg/image/export.go | 15 +- pkg/image/export_test.go | 26 +- pkg/image/import.go | 13 +- pkg/image/import_test.go | 4 +- pkg/image/scan.go | 7 +- pkg/models/image.go | 39 +- pkg/models/jsonschema/image.go | 25 +- pkg/models/mocks/ImageReaderWriter.go | 70 ++- pkg/models/model_image.go | 8 + pkg/models/repository_image.go | 4 +- pkg/sqlite/custom_fields.go | 4 + pkg/sqlite/custom_fields_test.go | 6 + pkg/sqlite/database.go | 2 +- pkg/sqlite/image.go | 26 +- pkg/sqlite/image_filter.go | 7 + pkg/sqlite/image_test.go | 437 +++++++++++++++--- .../migrations/83_image_custom_fields.up.sql | 9 + pkg/sqlite/setup_test.go | 18 +- pkg/sqlite/tables.go | 1 + 26 files changed, 669 insertions(+), 111 deletions(-) create mode 100644 pkg/sqlite/migrations/83_image_custom_fields.up.sql diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index 4162f0af3..907e597f4 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -765,6 +765,8 @@ input ImageFilterType { tags_filter: TagFilterType "Filter by related files that meet this criteria" files_filter: FileFilterType + "Filter by custom fields" + custom_fields: [CustomFieldCriterionInput!] } input FileFilterType { diff --git a/graphql/schema/types/image.graphql b/graphql/schema/types/image.graphql index b7ec1a9f5..ccc414542 100644 --- a/graphql/schema/types/image.graphql +++ b/graphql/schema/types/image.graphql @@ -21,6 +21,7 @@ type Image { studio: Studio tags: [Tag!]! performers: [Performer!]! + custom_fields: Map! } type ImageFileType { @@ -56,6 +57,7 @@ input ImageUpdateInput { gallery_ids: [ID!] primary_file_id: ID + custom_fields: CustomFieldsInput } input BulkImageUpdateInput { @@ -76,6 +78,7 @@ input BulkImageUpdateInput { performer_ids: BulkUpdateIds tag_ids: BulkUpdateIds gallery_ids: BulkUpdateIds + custom_fields: CustomFieldsInput } input ImageDestroyInput { diff --git a/internal/api/loaders/dataloaders.go b/internal/api/loaders/dataloaders.go index ff8a87ab0..dac8ba6b8 100644 --- a/internal/api/loaders/dataloaders.go +++ b/internal/api/loaders/dataloaders.go @@ -57,6 +57,7 @@ type Loaders struct { GalleryByID *GalleryLoader GalleryCustomFields *CustomFieldsLoader ImageByID *ImageLoader + ImageCustomFields *CustomFieldsLoader PerformerByID *PerformerLoader PerformerCustomFields *CustomFieldsLoader @@ -100,6 +101,11 @@ func (m Middleware) Middleware(next http.Handler) http.Handler { maxBatch: maxBatch, fetch: m.fetchImages(ctx), }, + ImageCustomFields: &CustomFieldsLoader{ + wait: wait, + maxBatch: maxBatch, + fetch: m.fetchImageCustomFields(ctx), + }, PerformerByID: &PerformerLoader{ wait: wait, maxBatch: maxBatch, @@ -249,6 +255,18 @@ func (m Middleware) fetchImages(ctx context.Context) func(keys []int) ([]*models } } +func (m Middleware) fetchImageCustomFields(ctx context.Context) func(keys []int) ([]models.CustomFieldMap, []error) { + return func(keys []int) (ret []models.CustomFieldMap, errs []error) { + err := m.Repository.WithDB(ctx, func(ctx context.Context) error { + var err error + ret, err = m.Repository.Image.GetCustomFieldsBulk(ctx, keys) + return err + }) + + return ret, toErrorSlice(err) + } +} + func (m Middleware) fetchGalleries(ctx context.Context) func(keys []int) ([]*models.Gallery, []error) { return func(keys []int) (ret []*models.Gallery, errs []error) { err := m.Repository.WithDB(ctx, func(ctx context.Context) error { diff --git a/internal/api/resolver_model_image.go b/internal/api/resolver_model_image.go index 0886bea40..4a95ae1f4 100644 --- a/internal/api/resolver_model_image.go +++ b/internal/api/resolver_model_image.go @@ -161,3 +161,12 @@ func (r *imageResolver) Urls(ctx context.Context, obj *models.Image) ([]string, return obj.URLs.List(), nil } + +func (r *imageResolver) CustomFields(ctx context.Context, obj *models.Image) (map[string]interface{}, error) { + customFields, err := loaders.From(ctx).ImageCustomFields.Load(obj.ID) + if err != nil { + return nil, err + } + + return customFields, nil +} diff --git a/internal/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go index 230d48358..cc03c5286 100644 --- a/internal/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -177,6 +177,13 @@ func (r *mutationResolver) imageUpdate(ctx context.Context, input models.ImageUp return nil, fmt.Errorf("converting tag ids: %w", err) } + if input.CustomFields != nil { + updatedImage.CustomFields = *input.CustomFields + // convert json.Numbers to int/float + updatedImage.CustomFields.Full = convertMapJSONNumbers(updatedImage.CustomFields.Full) + updatedImage.CustomFields.Partial = convertMapJSONNumbers(updatedImage.CustomFields.Partial) + } + qb := r.repository.Image image, err := qb.UpdatePartial(ctx, imageID, updatedImage) if err != nil { @@ -237,6 +244,13 @@ func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input BulkImageU return nil, fmt.Errorf("converting tag ids: %w", err) } + if input.CustomFields != nil { + updatedImage.CustomFields = *input.CustomFields + // convert json.Numbers to int/float + updatedImage.CustomFields.Full = convertMapJSONNumbers(updatedImage.CustomFields.Full) + updatedImage.CustomFields.Partial = convertMapJSONNumbers(updatedImage.CustomFields.Partial) + } + // Start the transaction and save the images if err := r.withTxn(ctx, func(ctx context.Context) error { var updatedGalleryIDs []int diff --git a/internal/autotag/integration_test.go b/internal/autotag/integration_test.go index 9745d623e..f537ecfe7 100644 --- a/internal/autotag/integration_test.go +++ b/internal/autotag/integration_test.go @@ -365,7 +365,10 @@ func makeImage(expectedResult bool) *models.Image { } func createImage(ctx context.Context, w models.ImageWriter, o *models.Image, f *models.ImageFile) error { - err := w.Create(ctx, o, []models.FileID{f.ID}) + err := w.Create(ctx, &models.CreateImageInput{ + Image: o, + FileIDs: []models.FileID{f.ID}, + }) if err != nil { return fmt.Errorf("Failed to create image with path '%s': %s", f.Path, err.Error()) diff --git a/internal/manager/task_export.go b/internal/manager/task_export.go index 30adf626b..01bab9430 100644 --- a/internal/manager/task_export.go +++ b/internal/manager/task_export.go @@ -651,6 +651,7 @@ func (t *ExportTask) exportImage(ctx context.Context, wg *sync.WaitGroup, jobCha galleryReader := r.Gallery performerReader := r.Performer tagReader := r.Tag + imageReader := r.Image for s := range jobChan { imageHash := s.Checksum @@ -665,14 +666,17 @@ func (t *ExportTask) exportImage(ctx context.Context, wg *sync.WaitGroup, jobCha continue } - newImageJSON := image.ToBasicJSON(s) + newImageJSON, err := image.ToBasicJSON(ctx, imageReader, s) + if err != nil { + logger.Errorf("[images] <%s> error converting image to JSON: %v", imageHash, err) + continue + } // export files for _, f := range s.Files.List() { t.exportFile(f) } - var err error newImageJSON.Studio, err = image.GetStudioName(ctx, studioReader, s) if err != nil { logger.Errorf("[images] <%s> error getting image studio name: %v", imageHash, err) diff --git a/pkg/image/export.go b/pkg/image/export.go index fdba6165c..eb5d5da27 100644 --- a/pkg/image/export.go +++ b/pkg/image/export.go @@ -2,16 +2,21 @@ package image import ( "context" + "fmt" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/json" "github.com/stashapp/stash/pkg/models/jsonschema" ) +type ExportReader interface { + models.CustomFieldsReader +} + // ToBasicJSON converts a image object into its JSON object equivalent. It // does not convert the relationships to other objects, with the exception // of cover image. -func ToBasicJSON(image *models.Image) *jsonschema.Image { +func ToBasicJSON(ctx context.Context, reader ExportReader, image *models.Image) (*jsonschema.Image, error) { newImageJSON := jsonschema.Image{ Title: image.Title, Code: image.Code, @@ -33,11 +38,17 @@ func ToBasicJSON(image *models.Image) *jsonschema.Image { newImageJSON.Organized = image.Organized newImageJSON.OCounter = image.OCounter + var err error + newImageJSON.CustomFields, err = reader.GetCustomFields(ctx, image.ID) + if err != nil { + return nil, fmt.Errorf("getting image custom fields: %v", err) + } + for _, f := range image.Files.List() { newImageJSON.Files = append(newImageJSON.Files, f.Base().Path) } - return &newImageJSON + return &newImageJSON, nil } // GetStudioName returns the name of the provided image's studio. It returns an diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index 6adaf1d33..d0d36afbb 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -29,6 +29,10 @@ var ( dateObj, _ = models.ParseDate(date) organized = true ocounter = 2 + + customFields = map[string]interface{}{ + "customField1": "customValue1", + } ) const ( @@ -60,7 +64,7 @@ func createFullImage(id int) models.Image { } } -func createFullJSONImage() *jsonschema.Image { +func createFullJSONImage(customFields map[string]interface{}) *jsonschema.Image { return &jsonschema.Image{ Title: title, OCounter: ocounter, @@ -75,28 +79,40 @@ func createFullJSONImage() *jsonschema.Image { UpdatedAt: json.JSONTime{ Time: updateTime, }, + CustomFields: customFields, } } type basicTestScenario struct { - input models.Image - expected *jsonschema.Image + input models.Image + customFields map[string]interface{} + expected *jsonschema.Image } var scenarios = []basicTestScenario{ { createFullImage(imageID), - createFullJSONImage(), + customFields, + createFullJSONImage(customFields), }, } func TestToJSON(t *testing.T) { + db := mocks.NewDatabase() + db.Image.On("GetCustomFields", testCtx, imageID).Return(customFields, nil).Once() + for i, s := range scenarios { image := s.input - json := ToBasicJSON(&image) + json, err := ToBasicJSON(testCtx, db.Image, &image) + if err != nil { + t.Errorf("[%d] unexpected error: %s", i, err.Error()) + continue + } assert.Equal(t, s.expected, json, "[%d]", i) } + + db.AssertExpectations(t) } func createStudioImage(studioID int) models.Image { diff --git a/pkg/image/import.go b/pkg/image/import.go index c7ef7f00c..d8dfa987f 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -31,8 +31,9 @@ type Importer struct { Input jsonschema.Image MissingRefBehaviour models.ImportMissingRefEnum - ID int - image models.Image + ID int + image models.Image + customFields map[string]interface{} } func (i *Importer) PreImport(ctx context.Context) error { @@ -58,6 +59,8 @@ func (i *Importer) PreImport(ctx context.Context) error { return err } + i.customFields = i.Input.CustomFields + return nil } @@ -344,7 +347,11 @@ func (i *Importer) Create(ctx context.Context) (*int, error) { fileIDs = append(fileIDs, f.Base().ID) } - err := i.ReaderWriter.Create(ctx, &i.image, fileIDs) + err := i.ReaderWriter.Create(ctx, &models.CreateImageInput{ + Image: &i.image, + FileIDs: fileIDs, + CustomFields: i.customFields, + }) if err != nil { return nil, fmt.Errorf("error creating image: %v", err) } diff --git a/pkg/image/import_test.go b/pkg/image/import_test.go index 5d01d4b97..a693c4568 100644 --- a/pkg/image/import_test.go +++ b/pkg/image/import_test.go @@ -45,7 +45,8 @@ func TestImporterPreImportWithStudio(t *testing.T) { i := Importer{ StudioWriter: db.Studio, Input: jsonschema.Image{ - Studio: existingStudioName, + Studio: existingStudioName, + CustomFields: customFields, }, } @@ -57,6 +58,7 @@ func TestImporterPreImportWithStudio(t *testing.T) { err := i.PreImport(testCtx) assert.Nil(t, err) assert.Equal(t, existingStudioID, *i.image.StudioID) + assert.Equal(t, customFields, i.customFields) i.Input.Studio = existingStudioErr err = i.PreImport(testCtx) diff --git a/pkg/image/scan.go b/pkg/image/scan.go index 67f4b334c..317e3605f 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -27,7 +27,7 @@ type ScanCreatorUpdater interface { GetFiles(ctx context.Context, relatedID int) ([]models.File, error) GetGalleryIDs(ctx context.Context, relatedID int) ([]int, error) - Create(ctx context.Context, newImage *models.Image, fileIDs []models.FileID) error + Create(ctx context.Context, newImage *models.CreateImageInput) error UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error) AddFileID(ctx context.Context, id int, fileID models.FileID) error } @@ -124,7 +124,10 @@ func (h *ScanHandler) Handle(ctx context.Context, f models.File, oldFile models. logger.Infof("Adding %s to gallery %s", f.Base().Path, g.Path) } - if err := h.CreatorUpdater.Create(ctx, &newImage, []models.FileID{imageFile.ID}); err != nil { + if err := h.CreatorUpdater.Create(ctx, &models.CreateImageInput{ + Image: &newImage, + FileIDs: []models.FileID{imageFile.ID}, + }); err != nil { return fmt.Errorf("creating new image: %w", err) } diff --git a/pkg/models/image.go b/pkg/models/image.go index 84be79360..b99267e8c 100644 --- a/pkg/models/image.go +++ b/pkg/models/image.go @@ -1,6 +1,8 @@ package models -import "context" +import ( + "context" +) type ImageFilterType struct { OperatorFilter[ImageFilterType] @@ -65,25 +67,28 @@ type ImageFilterType struct { CreatedAt *TimestampCriterionInput `json:"created_at"` // Filter by updated at UpdatedAt *TimestampCriterionInput `json:"updated_at"` + // Filter by custom fields + CustomFields []CustomFieldCriterionInput `json:"custom_fields"` } type ImageUpdateInput struct { - ClientMutationID *string `json:"clientMutationId"` - ID string `json:"id"` - Title *string `json:"title"` - Code *string `json:"code"` - Urls []string `json:"urls"` - Date *string `json:"date"` - Details *string `json:"details"` - Photographer *string `json:"photographer"` - Rating100 *int `json:"rating100"` - Organized *bool `json:"organized"` - SceneIds []string `json:"scene_ids"` - StudioID *string `json:"studio_id"` - TagIds []string `json:"tag_ids"` - PerformerIds []string `json:"performer_ids"` - GalleryIds []string `json:"gallery_ids"` - PrimaryFileID *string `json:"primary_file_id"` + ClientMutationID *string `json:"clientMutationId"` + ID string `json:"id"` + Title *string `json:"title"` + Code *string `json:"code"` + Urls []string `json:"urls"` + Date *string `json:"date"` + Details *string `json:"details"` + Photographer *string `json:"photographer"` + Rating100 *int `json:"rating100"` + Organized *bool `json:"organized"` + SceneIds []string `json:"scene_ids"` + StudioID *string `json:"studio_id"` + TagIds []string `json:"tag_ids"` + PerformerIds []string `json:"performer_ids"` + GalleryIds []string `json:"gallery_ids"` + PrimaryFileID *string `json:"primary_file_id"` + CustomFields *CustomFieldsInput `json:"custom_fields"` // deprecated URL *string `json:"url"` diff --git a/pkg/models/jsonschema/image.go b/pkg/models/jsonschema/image.go index 1bdac8770..168ea9eec 100644 --- a/pkg/models/jsonschema/image.go +++ b/pkg/models/jsonschema/image.go @@ -18,18 +18,19 @@ type Image struct { // deprecated - for import only URL string `json:"url,omitempty"` - URLs []string `json:"urls,omitempty"` - Date string `json:"date,omitempty"` - Details string `json:"details,omitempty"` - Photographer string `json:"photographer,omitempty"` - Organized bool `json:"organized,omitempty"` - OCounter int `json:"o_counter,omitempty"` - Galleries []GalleryRef `json:"galleries,omitempty"` - Performers []string `json:"performers,omitempty"` - Tags []string `json:"tags,omitempty"` - Files []string `json:"files,omitempty"` - CreatedAt json.JSONTime `json:"created_at,omitempty"` - UpdatedAt json.JSONTime `json:"updated_at,omitempty"` + URLs []string `json:"urls,omitempty"` + Date string `json:"date,omitempty"` + Details string `json:"details,omitempty"` + Photographer string `json:"photographer,omitempty"` + Organized bool `json:"organized,omitempty"` + OCounter int `json:"o_counter,omitempty"` + Galleries []GalleryRef `json:"galleries,omitempty"` + Performers []string `json:"performers,omitempty"` + Tags []string `json:"tags,omitempty"` + Files []string `json:"files,omitempty"` + CreatedAt json.JSONTime `json:"created_at,omitempty"` + UpdatedAt json.JSONTime `json:"updated_at,omitempty"` + CustomFields map[string]interface{} `json:"custom_fields,omitempty"` } func (s Image) Filename(basename string, hash string) string { diff --git a/pkg/models/mocks/ImageReaderWriter.go b/pkg/models/mocks/ImageReaderWriter.go index afc5efdb7..f2c9934be 100644 --- a/pkg/models/mocks/ImageReaderWriter.go +++ b/pkg/models/mocks/ImageReaderWriter.go @@ -137,13 +137,13 @@ func (_m *ImageReaderWriter) CoverByGalleryID(ctx context.Context, galleryId int return r0, r1 } -// Create provides a mock function with given fields: ctx, newImage, fileIDs -func (_m *ImageReaderWriter) Create(ctx context.Context, newImage *models.Image, fileIDs []models.FileID) error { - ret := _m.Called(ctx, newImage, fileIDs) +// Create provides a mock function with given fields: ctx, newImage +func (_m *ImageReaderWriter) Create(ctx context.Context, newImage *models.CreateImageInput) error { + ret := _m.Called(ctx, newImage) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *models.Image, []models.FileID) error); ok { - r0 = rf(ctx, newImage, fileIDs) + if rf, ok := ret.Get(0).(func(context.Context, *models.CreateImageInput) error); ok { + r0 = rf(ctx, newImage) } else { r0 = ret.Error(0) } @@ -393,6 +393,52 @@ func (_m *ImageReaderWriter) FindMany(ctx context.Context, ids []int) ([]*models return r0, r1 } +// GetCustomFields provides a mock function with given fields: ctx, id +func (_m *ImageReaderWriter) GetCustomFields(ctx context.Context, id int) (map[string]interface{}, error) { + ret := _m.Called(ctx, id) + + var r0 map[string]interface{} + if rf, ok := ret.Get(0).(func(context.Context, int) map[string]interface{}); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]interface{}) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCustomFieldsBulk provides a mock function with given fields: ctx, ids +func (_m *ImageReaderWriter) GetCustomFieldsBulk(ctx context.Context, ids []int) ([]models.CustomFieldMap, error) { + ret := _m.Called(ctx, ids) + + var r0 []models.CustomFieldMap + if rf, ok := ret.Get(0).(func(context.Context, []int) []models.CustomFieldMap); ok { + r0 = rf(ctx, ids) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]models.CustomFieldMap) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []int) error); ok { + r1 = rf(ctx, ids) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetFiles provides a mock function with given fields: ctx, relatedID func (_m *ImageReaderWriter) GetFiles(ctx context.Context, relatedID int) ([]models.File, error) { ret := _m.Called(ctx, relatedID) @@ -694,6 +740,20 @@ func (_m *ImageReaderWriter) ResetOCounter(ctx context.Context, id int) (int, er return r0, r1 } +// SetCustomFields provides a mock function with given fields: ctx, id, fields +func (_m *ImageReaderWriter) SetCustomFields(ctx context.Context, id int, fields models.CustomFieldsInput) error { + ret := _m.Called(ctx, id, fields) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int, models.CustomFieldsInput) error); ok { + r0 = rf(ctx, id, fields) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Size provides a mock function with given fields: ctx func (_m *ImageReaderWriter) Size(ctx context.Context) (float64, error) { ret := _m.Called(ctx) diff --git a/pkg/models/model_image.go b/pkg/models/model_image.go index 1d0993536..72ca61826 100644 --- a/pkg/models/model_image.go +++ b/pkg/models/model_image.go @@ -47,6 +47,13 @@ func NewImage() Image { } } +type CreateImageInput struct { + *Image + + FileIDs []FileID + CustomFields map[string]interface{} `json:"custom_fields"` +} + type ImagePartial struct { Title OptionalString Code OptionalString @@ -66,6 +73,7 @@ type ImagePartial struct { TagIDs *UpdateIDs PerformerIDs *UpdateIDs PrimaryFileID *FileID + CustomFields CustomFieldsInput } func NewImagePartial() ImagePartial { diff --git a/pkg/models/repository_image.go b/pkg/models/repository_image.go index 672ecd063..99dab3479 100644 --- a/pkg/models/repository_image.go +++ b/pkg/models/repository_image.go @@ -43,7 +43,7 @@ type ImageCounter interface { // ImageCreator provides methods to create images. type ImageCreator interface { - Create(ctx context.Context, newImage *Image, fileIDs []FileID) error + Create(ctx context.Context, newImage *CreateImageInput) error } // ImageUpdater provides methods to update images. @@ -78,6 +78,7 @@ type ImageReader interface { FileLoader GalleryCoverFinder + CustomFieldsReader All(ctx context.Context) ([]*Image, error) Size(ctx context.Context) (float64, error) @@ -88,6 +89,7 @@ type ImageWriter interface { ImageCreator ImageUpdater ImageDestroyer + CustomFieldsWriter AddFileID(ctx context.Context, id int, fileID FileID) error RemoveFileID(ctx context.Context, id int, fileID FileID) error diff --git a/pkg/sqlite/custom_fields.go b/pkg/sqlite/custom_fields.go index 63f85b250..d78e3f9ab 100644 --- a/pkg/sqlite/custom_fields.go +++ b/pkg/sqlite/custom_fields.go @@ -192,6 +192,10 @@ func (s *customFieldsStore) GetCustomFieldsBulk(ctx context.Context, ids []int) const single = false ret := make([]models.CustomFieldMap, len(ids)) + // initialise ret with empty maps for each id + for i := range ret { + ret[i] = make(map[string]interface{}) + } idi := make(map[int]int, len(ids)) for i, id := range ids { diff --git a/pkg/sqlite/custom_fields_test.go b/pkg/sqlite/custom_fields_test.go index 2f7ecd7dc..5d5545210 100644 --- a/pkg/sqlite/custom_fields_test.go +++ b/pkg/sqlite/custom_fields_test.go @@ -247,6 +247,12 @@ func TestGallerySetCustomFields(t *testing.T) { testSetCustomFields(t, "Gallery", db.Gallery, galleryIDs[galleryIdx], getGalleryCustomFields(galleryIdx)) } +func TestImageSetCustomFields(t *testing.T) { + imageIdx := imageIdx2WithGallery + + testSetCustomFields(t, "Image", db.Image, imageIDs[imageIdx], getImageCustomFields(imageIdx)) +} + func TestGroupSetCustomFields(t *testing.T) { groupIdx := groupIdxWithScene diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 000b91c4d..003c6eebc 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -34,7 +34,7 @@ const ( cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE" ) -var appSchemaVersion uint = 82 +var appSchemaVersion uint = 83 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index bcaf3f42f..da1c67a10 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -185,6 +185,8 @@ var ( ) type ImageStore struct { + customFieldsStore + tableMgr *table oCounterManager @@ -193,6 +195,10 @@ type ImageStore struct { func NewImageStore(r *storeRepository) *ImageStore { return &ImageStore{ + customFieldsStore: customFieldsStore{ + table: imagesCustomFieldsTable, + fk: imagesCustomFieldsTable.Col(imageIDColumn), + }, tableMgr: imageTableMgr, oCounterManager: oCounterManager{imageTableMgr}, repo: r, @@ -236,18 +242,18 @@ func (qb *ImageStore) selectDataset() *goqu.SelectDataset { ) } -func (qb *ImageStore) Create(ctx context.Context, newObject *models.Image, fileIDs []models.FileID) error { +func (qb *ImageStore) Create(ctx context.Context, newObject *models.CreateImageInput) error { var r imageRow - r.fromImage(*newObject) + r.fromImage(*newObject.Image) id, err := qb.tableMgr.insertID(ctx, r) if err != nil { return err } - if len(fileIDs) > 0 { + if len(newObject.FileIDs) > 0 { const firstPrimary = true - if err := imagesFilesTableMgr.insertJoins(ctx, id, firstPrimary, fileIDs); err != nil { + if err := imagesFilesTableMgr.insertJoins(ctx, id, firstPrimary, newObject.FileIDs); err != nil { return err } } @@ -276,12 +282,18 @@ func (qb *ImageStore) Create(ctx context.Context, newObject *models.Image, fileI } } + if err := qb.SetCustomFields(ctx, id, models.CustomFieldsInput{ + Full: newObject.CustomFields, + }); err != nil { + return err + } + updated, err := qb.find(ctx, id) if err != nil { return fmt.Errorf("finding after create: %w", err) } - *newObject = *updated + *newObject.Image = *updated return nil } @@ -329,6 +341,10 @@ func (qb *ImageStore) UpdatePartial(ctx context.Context, id int, partial models. } } + if err := qb.SetCustomFields(ctx, id, partial.CustomFields); err != nil { + return nil, err + } + return qb.find(ctx, id) } diff --git a/pkg/sqlite/image_filter.go b/pkg/sqlite/image_filter.go index b56ade26d..aafd2aa40 100644 --- a/pkg/sqlite/image_filter.go +++ b/pkg/sqlite/image_filter.go @@ -100,6 +100,13 @@ func (qb *imageFilterHandler) criterionHandler() criterionHandler { ×tampCriterionHandler{imageFilter.CreatedAt, "images.created_at", nil}, ×tampCriterionHandler{imageFilter.UpdatedAt, "images.updated_at", nil}, + &customFieldsFilterHandler{ + table: imagesCustomFieldsTable.GetTable(), + fkCol: imageIDColumn, + c: imageFilter.CustomFields, + idCol: "images.id", + }, + &relatedFilterHandler{ relatedIDCol: "galleries_images.gallery_id", relatedRepo: galleryRepository.repository, diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index aa4ed3b99..3bad40b3b 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -73,81 +73,94 @@ func Test_imageQueryBuilder_Create(t *testing.T) { tests := []struct { name string - newObject models.Image + newObject models.CreateImageInput wantErr bool }{ { "full", - models.Image{ - Title: title, - Code: code, - Rating: &rating, - Date: &date, - Details: details, - Photographer: photographer, - URLs: models.NewRelatedStrings([]string{url}), - Organized: true, - OCounter: ocounter, - StudioID: &studioIDs[studioIdxWithImage], - CreatedAt: createdAt, - UpdatedAt: updatedAt, - GalleryIDs: models.NewRelatedIDs([]int{galleryIDs[galleryIdxWithImage]}), - TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithDupName], tagIDs[tagIdx1WithImage]}), - PerformerIDs: models.NewRelatedIDs([]int{performerIDs[performerIdx1WithImage], performerIDs[performerIdx1WithDupName]}), + models.CreateImageInput{ + Image: &models.Image{ + Title: title, + Code: code, + Rating: &rating, + Date: &date, + Details: details, + Photographer: photographer, + URLs: models.NewRelatedStrings([]string{url}), + Organized: true, + OCounter: ocounter, + StudioID: &studioIDs[studioIdxWithImage], + CreatedAt: createdAt, + UpdatedAt: updatedAt, + GalleryIDs: models.NewRelatedIDs([]int{galleryIDs[galleryIdxWithImage]}), + TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithDupName], tagIDs[tagIdx1WithImage]}), + PerformerIDs: models.NewRelatedIDs([]int{performerIDs[performerIdx1WithImage], performerIDs[performerIdx1WithDupName]}), + }, + CustomFields: testCustomFields, }, false, }, { "with file", - models.Image{ - Title: title, - Code: code, - Rating: &rating, - Date: &date, - Details: details, - Photographer: photographer, - URLs: models.NewRelatedStrings([]string{url}), - Organized: true, - OCounter: ocounter, - StudioID: &studioIDs[studioIdxWithImage], - Files: models.NewRelatedFiles([]models.File{ - imageFile.(*models.ImageFile), - }), - PrimaryFileID: &imageFile.Base().ID, - Path: imageFile.Base().Path, - CreatedAt: createdAt, - UpdatedAt: updatedAt, - GalleryIDs: models.NewRelatedIDs([]int{galleryIDs[galleryIdxWithImage]}), - TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithDupName], tagIDs[tagIdx1WithImage]}), - PerformerIDs: models.NewRelatedIDs([]int{performerIDs[performerIdx1WithImage], performerIDs[performerIdx1WithDupName]}), + models.CreateImageInput{ + Image: &models.Image{ + Title: title, + Code: code, + Rating: &rating, + Date: &date, + Details: details, + Photographer: photographer, + URLs: models.NewRelatedStrings([]string{url}), + Organized: true, + OCounter: ocounter, + StudioID: &studioIDs[studioIdxWithImage], + Files: models.NewRelatedFiles([]models.File{ + imageFile.(*models.ImageFile), + }), + PrimaryFileID: &imageFile.Base().ID, + Path: imageFile.Base().Path, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + GalleryIDs: models.NewRelatedIDs([]int{galleryIDs[galleryIdxWithImage]}), + TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithDupName], tagIDs[tagIdx1WithImage]}), + PerformerIDs: models.NewRelatedIDs([]int{performerIDs[performerIdx1WithImage], performerIDs[performerIdx1WithDupName]}), + }, }, false, }, { "invalid studio id", - models.Image{ - StudioID: &invalidID, + models.CreateImageInput{ + Image: &models.Image{ + StudioID: &invalidID, + }, }, true, }, { "invalid gallery id", - models.Image{ - GalleryIDs: models.NewRelatedIDs([]int{invalidID}), + models.CreateImageInput{ + Image: &models.Image{ + GalleryIDs: models.NewRelatedIDs([]int{invalidID}), + }, }, true, }, { "invalid tag id", - models.Image{ - TagIDs: models.NewRelatedIDs([]int{invalidID}), + models.CreateImageInput{ + Image: &models.Image{ + TagIDs: models.NewRelatedIDs([]int{invalidID}), + }, }, true, }, { "invalid performer id", - models.Image{ - PerformerIDs: models.NewRelatedIDs([]int{invalidID}), + models.CreateImageInput{ + Image: &models.Image{ + PerformerIDs: models.NewRelatedIDs([]int{invalidID}), + }, }, true, }, @@ -165,8 +178,11 @@ func Test_imageQueryBuilder_Create(t *testing.T) { fileIDs = append(fileIDs, f.Base().ID) } } - s := tt.newObject - if err := qb.Create(ctx, &s, fileIDs); (err != nil) != tt.wantErr { + s := *tt.newObject.Image + if err := qb.Create(ctx, &models.CreateImageInput{ + Image: &s, + FileIDs: fileIDs, + }); (err != nil) != tt.wantErr { t.Errorf("imageQueryBuilder.Create() error = %v, wantErr = %v", err, tt.wantErr) } @@ -177,7 +193,7 @@ func Test_imageQueryBuilder_Create(t *testing.T) { assert.NotZero(s.ID) - copy := tt.newObject + copy := *tt.newObject.Image copy.ID = s.ID // load relationships @@ -201,8 +217,6 @@ func Test_imageQueryBuilder_Create(t *testing.T) { } assert.Equal(copy, *found) - - return }) } } @@ -387,8 +401,6 @@ func Test_imageQueryBuilder_Update(t *testing.T) { } assert.Equal(copy, *s) - - return }) } } @@ -832,6 +844,79 @@ func Test_imageQueryBuilder_UpdatePartialRelationships(t *testing.T) { } } +func Test_ImageStore_UpdatePartialCustomFields(t *testing.T) { + tests := []struct { + name string + id int + partial models.ImagePartial + expected map[string]interface{} // nil to use the partial + }{ + { + "set custom fields", + imageIDs[imageIdx1WithGallery], + models.ImagePartial{ + CustomFields: models.CustomFieldsInput{ + Full: testCustomFields, + }, + }, + nil, + }, + { + "clear custom fields", + imageIDs[imageIdx1WithGallery], + models.ImagePartial{ + CustomFields: models.CustomFieldsInput{ + Full: map[string]interface{}{}, + }, + }, + nil, + }, + { + "partial custom fields", + imageIDs[imageIdxWithStudio], + models.ImagePartial{ + CustomFields: models.CustomFieldsInput{ + Partial: map[string]interface{}{ + "string": "bbb", + "new_field": "new", + }, + }, + }, + map[string]interface{}{ + "int": int64(2), + "real": 1.2, + "string": "bbb", + "new_field": "new", + }, + }, + } + for _, tt := range tests { + qb := db.Image + + runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { + assert := assert.New(t) + + _, err := qb.UpdatePartial(ctx, tt.id, tt.partial) + if err != nil { + t.Errorf("ImageStore.UpdatePartial() error = %v", err) + return + } + + // ensure custom fields are correct + cf, err := qb.GetCustomFields(ctx, tt.id) + if err != nil { + t.Errorf("ImageStore.GetCustomFields() error = %v", err) + return + } + if tt.expected == nil { + assert.Equal(tt.partial.CustomFields.Full, cf) + } else { + assert.Equal(tt.expected, cf) + } + }) + } +} + func Test_imageQueryBuilder_IncrementOCounter(t *testing.T) { tests := []struct { name string @@ -3018,6 +3103,252 @@ func TestImageQueryPagination(t *testing.T) { }) } +func TestImageQueryCustomFields(t *testing.T) { + tests := []struct { + name string + filter *models.ImageFilterType + includeIdxs []int + excludeIdxs []int + wantErr bool + }{ + { + "equals", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierEquals, + Value: []any{getImageStringValue(imageIdx1WithGallery, "custom")}, + }, + }, + }, + []int{imageIdx1WithGallery}, + nil, + false, + }, + { + "not equals", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdx1WithGallery, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierNotEquals, + Value: []any{getImageStringValue(imageIdx1WithGallery, "custom")}, + }, + }, + }, + nil, + []int{imageIdx1WithGallery}, + false, + }, + { + "includes", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierIncludes, + Value: []any{getImageStringValue(imageIdx1WithGallery, "custom")[9:]}, + }, + }, + }, + []int{imageIdx1WithGallery}, + nil, + false, + }, + { + "excludes", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdx1WithGallery, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierExcludes, + Value: []any{getImageStringValue(imageIdx1WithGallery, "custom")[9:]}, + }, + }, + }, + nil, + []int{imageIdx1WithGallery}, + false, + }, + { + "regex", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierMatchesRegex, + Value: []any{".*17_custom"}, + }, + }, + }, + []int{imageIdxWithPerformerTag}, + nil, + false, + }, + { + "invalid regex", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierMatchesRegex, + Value: []any{"["}, + }, + }, + }, + nil, + nil, + true, + }, + { + "not matches regex", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdxWithPerformerTag, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierNotMatchesRegex, + Value: []any{".*17_custom"}, + }, + }, + }, + nil, + []int{imageIdxWithPerformerTag}, + false, + }, + { + "invalid not matches regex", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierNotMatchesRegex, + Value: []any{"["}, + }, + }, + }, + nil, + nil, + true, + }, + { + "null", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdx1WithGallery, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "not existing", + Modifier: models.CriterionModifierIsNull, + }, + }, + }, + []int{imageIdx1WithGallery}, + nil, + false, + }, + { + "not null", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdx1WithGallery, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "string", + Modifier: models.CriterionModifierNotNull, + }, + }, + }, + []int{imageIdx1WithGallery}, + nil, + false, + }, + { + "between", + &models.ImageFilterType{ + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "real", + Modifier: models.CriterionModifierBetween, + Value: []any{0.15, 0.25}, + }, + }, + }, + []int{imageIdx2WithGallery}, + nil, + false, + }, + { + "not between", + &models.ImageFilterType{ + Title: &models.StringCriterionInput{ + Value: getImageStringValue(imageIdx2WithGallery, titleField), + Modifier: models.CriterionModifierEquals, + }, + CustomFields: []models.CustomFieldCriterionInput{ + { + Field: "real", + Modifier: models.CriterionModifierNotBetween, + Value: []any{0.15, 0.25}, + }, + }, + }, + nil, + []int{imageIdx2WithGallery}, + false, + }, + } + + for _, tt := range tests { + runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { + assert := assert.New(t) + + result, err := db.Image.Query(ctx, models.ImageQueryOptions{ + ImageFilter: tt.filter, + }) + if (err != nil) != tt.wantErr { + t.Errorf("ImageStore.Query() error = %v, wantErr %v", err, tt.wantErr) + } + + if err != nil { + return + } + + images, err := result.Resolve(ctx) + if err != nil { + t.Errorf("ImageStore.Query().Resolve() error = %v", err) + } + + ids := imagesToIDs(images) + include := indexesToIDs(imageIDs, tt.includeIdxs) + exclude := indexesToIDs(imageIDs, tt.excludeIdxs) + + for _, i := range include { + assert.Contains(ids, i) + } + for _, e := range exclude { + assert.NotContains(ids, e) + } + }) + } +} + // TODO Count // TODO SizeCount // TODO All diff --git a/pkg/sqlite/migrations/83_image_custom_fields.up.sql b/pkg/sqlite/migrations/83_image_custom_fields.up.sql new file mode 100644 index 000000000..0aa3aa4d7 --- /dev/null +++ b/pkg/sqlite/migrations/83_image_custom_fields.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE `image_custom_fields` ( + `image_id` integer NOT NULL, + `field` varchar(64) NOT NULL, + `value` BLOB NOT NULL, + PRIMARY KEY (`image_id`, `field`), + foreign key(`image_id`) references `images`(`id`) on delete CASCADE +); + +CREATE INDEX `index_image_custom_fields_field_value` ON `image_custom_fields` (`field`, `value`); diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 0078f1a67..2848a0a14 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1247,6 +1247,18 @@ func getImageBasename(index int) string { return getImageStringValue(index, pathField) } +func getImageCustomFields(index int) map[string]interface{} { + if index%5 == 0 { + return nil + } + + return map[string]interface{}{ + "string": getImageStringValue(index, "custom"), + "int": int64(index % 5), + "real": float64(index) / 10, + } +} + func makeImageFile(i int) *models.ImageFile { return &models.ImageFile{ BaseFile: &models.BaseFile{ @@ -1309,7 +1321,11 @@ func createImages(ctx context.Context, n int) error { image := makeImage(i) - err := qb.Create(ctx, image, []models.FileID{f.ID}) + err := qb.Create(ctx, &models.CreateImageInput{ + Image: image, + FileIDs: []models.FileID{f.ID}, + CustomFields: getImageCustomFields(i), + }) if err != nil { return fmt.Errorf("Error creating image %v+: %s", image, err.Error()) diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index 6c898048d..4c09113f0 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -14,6 +14,7 @@ var ( performersImagesJoinTable = goqu.T(performersImagesTable) imagesFilesJoinTable = goqu.T(imagesFilesTable) imagesURLsJoinTable = goqu.T(imagesURLsTable) + imagesCustomFieldsTable = goqu.T("image_custom_fields") galleriesFilesJoinTable = goqu.T(galleriesFilesTable) galleriesTagsJoinTable = goqu.T(galleriesTagsTable)