Include blobs in backup (#6586)

* Optionally backup blobs into zip
* Add backup dialog

graphql/schema/types/metadata.graphql

@@ -325,6 +325,8 @@ input ImportObjectsInput {
 
 input BackupDatabaseInput {
   download: Boolean
+  "If true, blob files will be included in the backup. This can significantly increase the size of the backup and the time it takes to create it, but allows for a complete backup of the system that can be restored without needing access to the original media files."
+  includeBlobs: Boolean
 }
 
 input AnonymiseDatabaseInput {

internal/api/resolver_mutation_metadata.go

@@ -122,9 +122,10 @@ func (r *mutationResolver) MigrateHashNaming(ctx context.Context) (string, error
 func (r *mutationResolver) BackupDatabase(ctx context.Context, input BackupDatabaseInput) (*string, error) {
 	// if download is true, then backup to temporary file and return a link
 	download := input.Download != nil && *input.Download
+	includeBlobs := input.IncludeBlobs != nil && *input.IncludeBlobs
 	mgr := manager.GetInstance()
 
-	backupPath, backupName, err := mgr.BackupDatabase(download)
+	backupPath, backupName, err := mgr.BackupDatabase(download, includeBlobs)
 	if err != nil {
 		logger.Errorf("Error backing up database: %v", err)
 		return nil, err

internal/manager/backup.go

@@ -0,0 +1,185 @@
+package manager
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/stashapp/stash/internal/manager/config"
+	"github.com/stashapp/stash/pkg/fsutil"
+	"github.com/stashapp/stash/pkg/logger"
+)
+
+type databaseBackupZip struct {
+	*zip.Writer
+}
+
+func (z *databaseBackupZip) zipFileRename(fn, outDir, outFn string) error {
+	p := filepath.Join(outDir, outFn)
+	p = filepath.ToSlash(p)
+
+	f, err := z.Create(p)
+	if err != nil {
+		return fmt.Errorf("error creating zip entry for %s: %v", fn, err)
+	}
+
+	i, err := os.Open(fn)
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", fn, err)
+	}
+
+	defer i.Close()
+
+	if _, err := io.Copy(f, i); err != nil {
+		return fmt.Errorf("error writing %s to zip: %v", fn, err)
+	}
+
+	return nil
+}
+
+func (z *databaseBackupZip) zipFile(fn, outDir string) error {
+	return z.zipFileRename(fn, outDir, filepath.Base(fn))
+}
+
+func (s *Manager) BackupDatabase(download bool, includeBlobs bool) (string, string, error) {
+	var backupPath string
+	var backupName string
+
+	// if we include blobs, then the output is a zip file
+	// if not, using the same backup logic as before, which creates a sqlite file
+	if !includeBlobs || s.Config.GetBlobsStorage() != config.BlobStorageTypeFilesystem {
+		return s.backupDatabaseOnly(download)
+	}
+
+	// use tmp directory for the backup
+	backupDir := s.Paths.Generated.Tmp
+	if err := fsutil.EnsureDir(backupDir); err != nil {
+		return "", "", fmt.Errorf("could not create backup directory %v: %w", backupDir, err)
+	}
+	f, err := os.CreateTemp(backupDir, "backup*.sqlite")
+	if err != nil {
+		return "", "", err
+	}
+
+	backupPath = f.Name()
+	backupName = s.Database.DatabaseBackupPath("")
+	f.Close()
+
+	// delete the temp file so that the backup operation can create it
+	if err := os.Remove(backupPath); err != nil {
+		return "", "", fmt.Errorf("could not remove temporary backup file %v: %w", backupPath, err)
+	}
+
+	if err := s.Database.Backup(backupPath); err != nil {
+		return "", "", err
+	}
+
+	// create a zip file
+	zipFileDir := s.Paths.Generated.Downloads
+	if !download {
+		zipFileDir = s.Config.GetBackupDirectoryPathOrDefault()
+		if zipFileDir != "" {
+			if err := fsutil.EnsureDir(zipFileDir); err != nil {
+				return "", "", fmt.Errorf("could not create backup directory %v: %w", zipFileDir, err)
+			}
+		}
+	}
+
+	zipFileName := backupName + ".zip"
+	zipFilePath := filepath.Join(zipFileDir, zipFileName)
+
+	logger.Debugf("Preparing zip file for database backup at %v", zipFilePath)
+
+	zf, err := os.Create(zipFilePath)
+	if err != nil {
+		return "", "", fmt.Errorf("could not create zip file %v: %w", zipFilePath, err)
+	}
+	defer zf.Close()
+
+	z := databaseBackupZip{
+		Writer: zip.NewWriter(zf),
+	}
+
+	defer z.Close()
+
+	// move the database file into the zip
+	dbFn := filepath.Base(s.Config.GetDatabasePath())
+	if err := z.zipFileRename(backupPath, "", dbFn); err != nil {
+		return "", "", fmt.Errorf("could not add database backup to zip file: %w", err)
+	}
+
+	if err := os.Remove(backupPath); err != nil {
+		return "", "", fmt.Errorf("could not remove temporary backup file %v: %w", backupPath, err)
+	}
+
+	// walk the blobs directory and add files to the zip
+	blobsDir := s.Config.GetBlobsPath()
+	err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		// calculate out dir by removing the blobsDir prefix from the path
+		outDir := filepath.Join("blobs", strings.TrimPrefix(filepath.Dir(path), blobsDir))
+		if err := z.zipFile(path, outDir); err != nil {
+			return fmt.Errorf("could not add blob %v to zip file: %w", path, err)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return "", "", fmt.Errorf("error walking blobs directory: %w", err)
+	}
+
+	return zipFilePath, zipFileName, nil
+}
+
+func (s *Manager) backupDatabaseOnly(download bool) (string, string, error) {
+	var backupPath string
+	var backupName string
+
+	if download {
+		backupDir := s.Paths.Generated.Downloads
+		if err := fsutil.EnsureDir(backupDir); err != nil {
+			return "", "", fmt.Errorf("could not create backup directory %v: %w", backupDir, err)
+		}
+		f, err := os.CreateTemp(backupDir, "backup*.sqlite")
+		if err != nil {
+			return "", "", err
+		}
+
+		backupPath = f.Name()
+		backupName = s.Database.DatabaseBackupPath("")
+		f.Close()
+
+		// delete the temp file so that the backup operation can create it
+		if err := os.Remove(backupPath); err != nil {
+			return "", "", fmt.Errorf("could not remove temporary backup file %v: %w", backupPath, err)
+		}
+	} else {
+		backupDir := s.Config.GetBackupDirectoryPathOrDefault()
+		if backupDir != "" {
+			if err := fsutil.EnsureDir(backupDir); err != nil {
+				return "", "", fmt.Errorf("could not create backup directory %v: %w", backupDir, err)
+			}
+		}
+		backupPath = s.Database.DatabaseBackupPath(backupDir)
+		backupName = filepath.Base(backupPath)
+	}
+
+	err := s.Database.Backup(backupPath)
+	if err != nil {
+		return "", "", err
+	}
+
+	return backupPath, backupName, nil
+}

internal/manager/manager.go

@@ -313,46 +313,6 @@ func (s *Manager) validateFFmpeg() error {
 	return nil
 }
 
-func (s *Manager) BackupDatabase(download bool) (string, string, error) {
-	var backupPath string
-	var backupName string
-	if download {
-		backupDir := s.Paths.Generated.Downloads
-		if err := fsutil.EnsureDir(backupDir); err != nil {
-			return "", "", fmt.Errorf("could not create backup directory %v: %w", backupDir, err)
-		}
-		f, err := os.CreateTemp(backupDir, "backup*.sqlite")
-		if err != nil {
-			return "", "", err
-		}
-
-		backupPath = f.Name()
-		backupName = s.Database.DatabaseBackupPath("")
-		f.Close()
-
-		// delete the temp file so that the backup operation can create it
-		if err := os.Remove(backupPath); err != nil {
-			return "", "", fmt.Errorf("could not remove temporary backup file %v: %w", backupPath, err)
-		}
-	} else {
-		backupDir := s.Config.GetBackupDirectoryPathOrDefault()
-		if backupDir != "" {
-			if err := fsutil.EnsureDir(backupDir); err != nil {
-				return "", "", fmt.Errorf("could not create backup directory %v: %w", backupDir, err)
-			}
-		}
-		backupPath = s.Database.DatabaseBackupPath(backupDir)
-		backupName = filepath.Base(backupPath)
-	}
-
-	err := s.Database.Backup(backupPath)
-	if err != nil {
-		return "", "", err
-	}
-
-	return backupPath, backupName, nil
-}
-
 func (s *Manager) AnonymiseDatabase(download bool) (string, string, error) {
 	var outPath string
 	var outName string

ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx

@@ -25,6 +25,8 @@ import { Icon } from "src/components/Shared/Icon";
 import { useConfigurationContext } from "src/hooks/Config";
 import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect";
 import {
+  faBoxArchive,
+  faExclamationTriangle,
   faMinus,
   faPlus,
   faQuestionCircle,
@@ -153,6 +155,125 @@ const CleanOptions: React.FC<ICleanOptions> = ({
   );
 };
 
+const BackupDialog: React.FC<{
+  onClose: (
+    confirmed?: boolean,
+    download?: boolean,
+    includeBlobs?: boolean
+  ) => void;
+}> = ({ onClose }) => {
+  const intl = useIntl();
+  const { configuration } = useConfigurationContext();
+
+  const includeBlobsDefault =
+    configuration?.general.blobsStorage === GQL.BlobsStorageType.Filesystem;
+  const backupDir =
+    configuration.general.backupDirectoryPath ||
+    `<${intl.formatMessage({
+      id: "config.general.backup_directory_path.heading",
+    })}>`;
+
+  const [download, setDownload] = useState(false);
+  const [includeBlobs, setIncludeBlobs] = useState(includeBlobsDefault);
+
+  let msg;
+  if (!includeBlobs) {
+    msg = intl.formatMessage(
+      { id: "config.tasks.backup_database.sqlite" },
+      {
+        filename_format: (
+          <code>[origFilename].sqlite.[schemaVersion].[YYYYMMDD_HHMMSS]</code>
+        ),
+      }
+    );
+  } else {
+    msg = intl.formatMessage(
+      { id: "config.tasks.backup_database.zip" },
+      {
+        filename_format: (
+          <code>
+            [origFilename].sqlite.[schemaVersion].[YYYYMMDD_HHMMSS].zip
+          </code>
+        ),
+      }
+    );
+  }
+
+  const warning =
+    includeBlobs !== includeBlobsDefault ? (
+      <p className="lead">
+        <Icon icon={faExclamationTriangle} className="text-warning" />
+        <FormattedMessage id="config.tasks.backup_database.warning_blobs" />
+      </p>
+    ) : null;
+
+  const acceptID = download
+    ? "config.tasks.backup_database.download"
+    : "actions.backup";
+
+  return (
+    <ModalComponent
+      show
+      icon={faBoxArchive}
+      accept={{
+        text: intl.formatMessage({ id: acceptID }),
+        onClick: () => onClose(true, download, includeBlobs),
+      }}
+      cancel={{
+        onClick: () => onClose(),
+        variant: "secondary",
+      }}
+    >
+      <div className="dialog-container">
+        <Form.Group>
+          <h5>
+            <FormattedMessage id="config.tasks.backup_database.destination" />
+          </h5>
+          <Form.Check
+            type="radio"
+            id="backup-directory"
+            checked={!download}
+            onChange={() => setDownload(false)}
+            label={intl.formatMessage(
+              {
+                id: "config.tasks.backup_database.to_directory",
+              },
+              {
+                directory: <code>{backupDir}</code>,
+              }
+            )}
+          />
+
+          <Form.Check
+            type="radio"
+            id="backup-download"
+            checked={download}
+            onChange={() => setDownload(true)}
+            label={intl.formatMessage({
+              id: "config.tasks.backup_database.download",
+            })}
+          />
+        </Form.Group>
+
+        <SettingSection>
+          <BooleanSetting
+            id="backup-include-blobs"
+            // if includeBlobsDefault is false, then blobs are in the database, so we check the box and disable it
+            checked={includeBlobs || !includeBlobsDefault}
+            headingID="config.tasks.backup_database.include_blobs"
+            onChange={(v) => setIncludeBlobs(v)}
+            // if includeBlobsDefault is false, then blobs are in the database
+            disabled={!includeBlobsDefault}
+          />
+        </SettingSection>
+
+        <p>{msg}</p>
+        {warning}
+      </div>
+    </ModalComponent>
+  );
+};
+
 interface IDataManagementTasks {
   setIsBackupRunning: (v: boolean) => void;
   setIsAnonymiseRunning: (v: boolean) => void;
@@ -167,6 +288,7 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
   const [dialogOpen, setDialogOpenState] = useState({
     importAlert: false,
     import: false,
+    backup: false,
     clean: false,
     cleanAlert: false,
     cleanGenerated: false,
@@ -344,11 +466,12 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     }
   }
 
-  async function onBackup(download?: boolean) {
+  async function onBackup(download?: boolean, includeBlobs?: boolean) {
     try {
       setIsBackupRunning(true);
       const ret = await mutateBackupDatabase({
         download,
+        includeBlobs,
       });
 
       // download the result
@@ -439,6 +562,17 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
           }}
         />
       )}
+      {dialogOpen.backup && (
+        <BackupDialog
+          onClose={(confirmed, download, includeBlobs) => {
+            if (confirmed) {
+              onBackup(download, includeBlobs);
+            }
+
+            setDialogOpen({ backup: false });
+          }}
+        />
+      )}
 
       <SettingSection headingID="config.tasks.maintenance">
         <div className="setting-group">
@@ -555,39 +689,25 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
 
       <SettingSection headingID="actions.backup">
         <Setting
-          headingID="actions.backup"
-          subHeading={intl.formatMessage(
-            { id: "config.tasks.backup_database" },
-            {
-              filename_format: (
-                <code>
-                  [origFilename].sqlite.[schemaVersion].[YYYYMMDD_HHMMSS]
-                </code>
-              ),
-            }
-          )}
+          heading={
+            <>
+              <FormattedMessage id="actions.backup" />
+              <ManualLink tab="Tasks">
+                <Icon icon={faQuestionCircle} />
+              </ManualLink>
+            </>
+          }
+          subHeading={intl.formatMessage({
+            id: "config.tasks.backup_database.description",
+          })}
         >
           <Button
             id="backup"
             variant="secondary"
             type="submit"
-            onClick={() => onBackup()}
+            onClick={() => setDialogOpen({ backup: true })}
           >
-            <FormattedMessage id="actions.backup" />
-          </Button>
-        </Setting>
-
-        <Setting
-          headingID="actions.download_backup"
-          subHeadingID="config.tasks.backup_and_download"
-        >
-          <Button
-            id="backupDownload"
-            variant="secondary"
-            type="submit"
-            onClick={() => onBackup(true)}
-          >
-            <FormattedMessage id="actions.download_backup" />
+            <FormattedMessage id="actions.backup" />…
           </Button>
         </Setting>
       </SettingSection>

ui/v2.5/src/docs/en/Manual/Tasks.md

@@ -85,3 +85,19 @@ The import and export tasks read and write JSON files to the configured metadata
 > **⚠️ Note:** The full import task wipes the current database completely before importing.
 
 See the [JSON Specification](/help/JSONSpec.md) page for details on the exported JSON format.
+
+## Backing up
+
+The backup task creates a backup of the stash database and (optionally) blob files. The backup can either be downloaded or output into the backup directory (under `Settings > Paths`) or the database directory if the backup directory is not configured.
+
+For a full backup, the database file and all blob files must be copied. The backup is stored as a zip file, with the database file at the root of the zip and the blob files in a `blobs` directory.
+
+> **⚠️ Note:** generated files are not included in the backup, so these will need to be regenerated when restoring with an empty system from backup.
+
+For database-only backups, only the database file is copied into the destination. This is useful for quick backups before performing risky operations, or for users who do not use filesystem blob storage.
+
+## Restoring from backup
+
+Restoring from backup is currently a manual process. The database backup zip file must be unzipped, and the database file and blob files (if applicable) copied into the database and blob directories respectively. Stash should then be restarted to load the restored database.
+
+> **⚠️ Note:** the filename for a database-only backup is not the same as the original database file, so the database file from the backup must be renamed to match the original database filename before copying it into the database directory. The original database filename can be found in `Settings > Paths > Database path`.

ui/v2.5/src/locales/en-GB.json

@@ -499,7 +499,17 @@
       "auto_tagging": "Auto tagging",
       "backing_up_database": "Backing up database",
       "backup_and_download": "Performs a backup of the database and downloads the resulting file.",
-      "backup_database": "Performs a backup of the database to the backups directory, with the filename format {filename_format}.",
+      "backup_database": {
+        "description": "Performs a backup of the database and blob files.",
+        "destination": "Destination",
+        "download": "Download backup",
+        "include_blobs": "Include blobs in backup",
+        "include_blobs_desc": "Disable to only backup the SQLite database file.",
+        "sqlite": "Backup file will be a copy of the SQLite database file, with the filename {filename_format}",
+        "to_directory": "To {directory}",
+        "warning_blobs": "Blob files will not be included in the backup. This means that to succesfully restore from the backup, the blob files must be present in the blob storage location.",
+        "zip": "SQLite database file and blob files will be zipped into a single file, with the filename {filename_format}"
+      },
       "cleanup_desc": "Check for missing files and remove them from the database. This is a destructive action.",
       "clean_generated": {
         "blob_files": "Blob files",