Fix panic when scraping with unknown field (#6220)

* Fix URL in group scraper causing panic
* Return error instead of panicking on unknown field

pkg/models/model_scraped_item.go

@@ -462,6 +462,7 @@ type ScrapedGroup struct {
 	Date     *string        `json:"date"`
 	Rating   *string        `json:"rating"`
 	Director *string        `json:"director"`
+	URL      *string        `json:"url"` // included for backward compatibility
 	URLs     []string       `json:"urls"`
 	Synopsis *string        `json:"synopsis"`
 	Studio   *ScrapedStudio `json:"studio"`

pkg/scraper/mapped.go

@@ -873,50 +873,55 @@ func (r mappedResult) apply(dest interface{}) {
 
 func mapFieldValue(destVal reflect.Value, key string, value interface{}) error {
 	field := destVal.FieldByName(key)
+
+	if !field.IsValid() {
+		return fmt.Errorf("field %s does not exist on %s", key, destVal.Type().Name())
+	}
+
+	if !field.CanSet() {
+		return fmt.Errorf("field %s cannot be set on %s", key, destVal.Type().Name())
+	}
+
 	fieldType := field.Type()
 
-	if field.IsValid() && field.CanSet() {
+	switch v := value.(type) {
-		switch v := value.(type) {
+	case string:
-		case string:
+		// if the field is a pointer to a string, then we need to convert the string to a pointer
-			// if the field is a pointer to a string, then we need to convert the string to a pointer
+		// if the field is a string slice, then we need to convert the string to a slice
-			// if the field is a string slice, then we need to convert the string to a slice
+		switch {
-			switch {
+		case fieldType.Kind() == reflect.String:
-			case fieldType.Kind() == reflect.String:
+			field.SetString(v)
-				field.SetString(v)
+		case fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.String:
-			case fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.String:
+			ptr := reflect.New(fieldType.Elem())
-				ptr := reflect.New(fieldType.Elem())
+			ptr.Elem().SetString(v)
-				ptr.Elem().SetString(v)
+			field.Set(ptr)
-				field.Set(ptr)
+		case fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String:
-			case fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String:
+			field.Set(reflect.ValueOf([]string{v}))
-				field.Set(reflect.ValueOf([]string{v}))
-			default:
-				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
-			}
-		case []string:
-			// expect the field to be a string slice
-			if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String {
-				field.Set(reflect.ValueOf(v))
-			} else {
-				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
-			}
 		default:
-			// fallback to reflection
+			return fmt.Errorf("cannot convert %T to %s", value, fieldType)
-			reflectValue := reflect.ValueOf(value)
+		}
-			reflectValueType := reflectValue.Type()
+	case []string:
-
+		// expect the field to be a string slice
-			switch {
+		if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String {
-			case reflectValueType.ConvertibleTo(fieldType):
+			field.Set(reflect.ValueOf(v))
-				field.Set(reflectValue.Convert(fieldType))
+		} else {
-			case fieldType.Kind() == reflect.Pointer && reflectValueType.ConvertibleTo(fieldType.Elem()):
+			return fmt.Errorf("cannot convert %T to %s", value, fieldType)
-				ptr := reflect.New(fieldType.Elem())
+		}
-				ptr.Elem().Set(reflectValue.Convert(fieldType.Elem()))
+	default:
-				field.Set(ptr)
+		// fallback to reflection
-			default:
+		reflectValue := reflect.ValueOf(value)
-				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
+		reflectValueType := reflectValue.Type()
-			}
+
+		switch {
+		case reflectValueType.ConvertibleTo(fieldType):
+			field.Set(reflectValue.Convert(fieldType))
+		case fieldType.Kind() == reflect.Pointer && reflectValueType.ConvertibleTo(fieldType.Elem()):
+			ptr := reflect.New(fieldType.Elem())
+			ptr.Elem().Set(reflectValue.Convert(fieldType.Elem()))
+			field.Set(ptr)
+		default:
+			return fmt.Errorf("cannot convert %T to %s", value, fieldType)
 		}
-	} else {
-		return fmt.Errorf("field does not exist or cannot be set")
 	}
 
 	return nil

pkg/scraper/postprocessing.go

@@ -143,6 +143,21 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, exclu
 		return nil, nil, err
 	}
 
+	// populate URL/URLs
+	// if URLs are provided, only use those
+	if len(m.URLs) > 0 {
+		m.URL = &m.URLs[0]
+	} else {
+		urls := []string{}
+		if m.URL != nil {
+			urls = append(urls, *m.URL)
+		}
+
+		if len(urls) > 0 {
+			m.URLs = urls
+		}
+	}
+
 	// post-process - set the image if applicable
 	if err := setMovieFrontImage(ctx, c.client, &m, c.globalConfig); err != nil {
 		logger.Warnf("could not set front image using URL %s: %v", *m.FrontImage, err)
@@ -175,6 +190,21 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, exclu
 		return nil, nil, err
 	}
 
+	// populate URL/URLs
+	// if URLs are provided, only use those
+	if len(m.URLs) > 0 {
+		m.URL = &m.URLs[0]
+	} else {
+		urls := []string{}
+		if m.URL != nil {
+			urls = append(urls, *m.URL)
+		}
+
+		if len(urls) > 0 {
+			m.URLs = urls
+		}
+	}
+
 	// post-process - set the image if applicable
 	if err := setGroupFrontImage(ctx, c.client, &m, c.globalConfig); err != nil {
 		logger.Warnf("could not set front image using URL %s: %v", *m.FrontImage, err)