diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 15ca0ee3af..bcbdebf16a 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -482,7 +482,11 @@ type Mutation { "Start auto-tagging. Returns the job ID" metadataAutoTag(input: AutoTagMetadataInput!): ID! "Clean metadata. Returns the job ID" - metadataClean(input: CleanMetadataInput!): ID! + metadataClean(input: CleanMetadataInput!): ID! @deprecated(reason: "Use verifyPaths and purgeMissing instead") + "Verifies the existence of files and folders in library, marking any missing items. Returns the job ID" + verifyPaths(input: VerifyPathsInput!): ID! + "Cleans up any missing files and folders from the library. Returns the job ID" + purgeMissing(input: PurgeMissingInput!): ID! "Clean generated files. Returns the job ID" metadataCleanGenerated(input: CleanGeneratedInput!): ID! "Identifies scenes using scrapers. Returns the job ID" diff --git a/graphql/schema/types/file.graphql b/graphql/schema/types/file.graphql index e9f01144e8..aa221a7c54 100644 --- a/graphql/schema/types/file.graphql +++ b/graphql/schema/types/file.graphql @@ -19,6 +19,9 @@ type Folder { "Returns direct sub-folders" sub_folders: [Folder!]! + "If not null, indicates when the folder was detected as missing. Set by the clean task" + missing_since: Time + mod_time: Time! created_at: Time! @@ -36,6 +39,9 @@ interface BaseFile { parent_folder: Folder! zip_file: BasicFile + "If not null, indicates when the folder was detected as missing. Set by the verifyPaths task" + missing_since: Time + mod_time: Time! size: Int64! @@ -57,6 +63,9 @@ type BasicFile implements BaseFile { parent_folder: Folder! zip_file: BasicFile + "If not null, indicates when the folder was detected as missing. Set by the verifyPaths task" + missing_since: Time + mod_time: Time! size: Int64! @@ -78,6 +87,9 @@ type VideoFile implements BaseFile { parent_folder: Folder! zip_file: BasicFile + "If not null, indicates when the folder was detected as missing. Set by the verifyPaths task" + missing_since: Time + mod_time: Time! size: Int64! @@ -110,6 +122,9 @@ type ImageFile implements BaseFile { parent_folder: Folder! zip_file: BasicFile + "If not null, indicates when the folder was detected as missing. Set by the verifyPaths task" + missing_since: Time + mod_time: Time! size: Int64! @@ -139,6 +154,9 @@ type GalleryFile implements BaseFile { parent_folder: Folder! zip_file: BasicFile + "If not null, indicates when the folder was detected as missing. Set by the verifyPaths task" + missing_since: Time + mod_time: Time! size: Int64! diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index ec6c57ace5..37e5467eeb 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -789,6 +789,9 @@ input FileFilterType { parent_folder: HierarchicalMultiCriterionInput zip_file: MultiCriterionInput + "Filter by missing since time" + missing_since: TimestampCriterionInput + "Filter by modification time" mod_time: TimestampCriterionInput @@ -829,6 +832,9 @@ input FolderFilterType { parent_folder: HierarchicalMultiCriterionInput zip_file: MultiCriterionInput + "Filter by missing since time" + missing_since: TimestampCriterionInput + "Filter by modification time" mod_time: TimestampCriterionInput diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 6ad620dbeb..f480bebb96 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -128,6 +128,34 @@ type ScanMetadataOptions { scanGenerateClipPreviews: Boolean! } +input VerifyPathsInput { + paths: [String!] + + """ + Check zip file contents when checking the existence of files. + This can significantly slow down the process, but will ensure removed files within zip files are detected. + Only necessary where users modify zip files contents. + Defaults to false. + """ + checkZipFileContents: Boolean + + "If true, only logs missing files to the log file and does not make any changes" + dryRun: Boolean + + "If true, removes any missing files from the database" + purgeMissing: Boolean +} + +input PurgeMissingInput { + paths: [String!] + + "If provided, only purge files that have been missing since before this time" + missingSinceBefore: Timestamp + + "If true, only logs actions that will be taken to the log file and does not make any changes" + dryRun: Boolean +} + input CleanMetadataInput { paths: [String!] diff --git a/internal/api/resolver_mutation_metadata.go b/internal/api/resolver_mutation_metadata.go index ea6496800a..d3c24665e1 100644 --- a/internal/api/resolver_mutation_metadata.go +++ b/internal/api/resolver_mutation_metadata.go @@ -11,7 +11,9 @@ import ( "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/internal/manager/task" + "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/utils" ) func (r *mutationResolver) MetadataScan(ctx context.Context, input manager.ScanMetadataInput) (string, error) { @@ -99,6 +101,33 @@ func (r *mutationResolver) MetadataClean(ctx context.Context, input manager.Clea return strconv.Itoa(jobID), nil } +func (r *mutationResolver) VerifyPaths(ctx context.Context, input VerifyPathsInput) (string, error) { + dryRun := utils.IsTrue(input.DryRun) + + options := file.VerifyOptions{ + Paths: input.Paths, + IgnoreZipFileContents: !utils.IsTrue(input.CheckZipFileContents), + DryRun: dryRun, + PurgeMissing: !dryRun && utils.IsTrue(input.PurgeMissing), + } + + jobID := manager.GetInstance().VerifyPaths(ctx, options) + return strconv.Itoa(jobID), nil +} + +func (r *mutationResolver) PurgeMissing(ctx context.Context, input PurgeMissingInput) (string, error) { + dryRun := utils.IsTrue(input.DryRun) + + options := file.PurgeMissingOptions{ + Paths: input.Paths, + DryRun: dryRun, + MissingSinceBefore: input.MissingSinceBefore, + } + + jobID := manager.GetInstance().PurgeMissing(ctx, options) + return strconv.Itoa(jobID), nil +} + func (r *mutationResolver) MetadataCleanGenerated(ctx context.Context, input task.CleanGeneratedOptions) (string, error) { mgr := manager.GetInstance() t := &task.CleanGeneratedJob{ diff --git a/internal/manager/manager_tasks.go b/internal/manager/manager_tasks.go index 518b872f6d..d73ad991fc 100644 --- a/internal/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -319,25 +319,56 @@ type CleanMetadataInput struct { } func (s *Manager) Clean(ctx context.Context, input CleanMetadataInput) int { - cleaner := &file.Cleaner{ + // deprecated - run verify paths task with purge missing turned on + return s.VerifyPaths(ctx, file.VerifyOptions{ + Paths: input.Paths, + IgnoreZipFileContents: input.IgnoreZipFileContents, + DryRun: input.DryRun, + PurgeMissing: !input.DryRun, + }) +} + +func (s *Manager) VerifyPaths(ctx context.Context, options file.VerifyOptions) int { + verifier := &file.Verifier{ FS: &file.OsFS{}, Repository: file.NewRepository(s.Repository), - Handlers: []file.CleanHandler{ - &cleanHandler{}, + PurgeHandlers: []file.PurgeHandler{ + &purgeHandler{}, + }, + TrashPath: s.Config.GetDeleteTrashPath(), + } + + j := verifyJob{ + verifier: verifier, + repository: s.Repository, + sceneService: s.SceneService, + imageService: s.ImageService, + options: options, + scanSubs: s.scanSubs, + } + + return s.JobManager.Add(ctx, "Verifying paths...", &j) +} + +func (s *Manager) PurgeMissing(ctx context.Context, options file.PurgeMissingOptions) int { + purger := &file.MissingPurger{ + Repository: file.NewRepository(s.Repository), + PurgeHandlers: []file.PurgeHandler{ + &purgeHandler{}, }, TrashPath: s.Config.GetDeleteTrashPath(), } - j := cleanJob{ - cleaner: cleaner, + j := purgeMissingJob{ + purger: purger, repository: s.Repository, sceneService: s.SceneService, imageService: s.ImageService, - input: input, + options: options, scanSubs: s.scanSubs, } - return s.JobManager.Add(ctx, "Cleaning...", &j) + return s.JobManager.Add(ctx, "Purging missing files and folders...", &j) } func (s *Manager) OptimiseDatabase(ctx context.Context) int { diff --git a/internal/manager/task_clean.go b/internal/manager/task_purge_missing.go similarity index 59% rename from internal/manager/task_clean.go rename to internal/manager/task_purge_missing.go index 0852092cd3..ad32c0e65d 100644 --- a/internal/manager/task_clean.go +++ b/internal/manager/task_purge_missing.go @@ -3,11 +3,8 @@ package manager import ( "context" "fmt" - "io/fs" - "path/filepath" "time" - "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/image" @@ -19,47 +16,42 @@ import ( "github.com/stashapp/stash/pkg/scene" ) -type cleaner interface { - Clean(ctx context.Context, options file.CleanOptions, progress *job.Progress) -} +type purgeMissingJob struct { + purger *file.MissingPurger -type cleanJob struct { - cleaner cleaner + options file.PurgeMissingOptions repository models.Repository - input CleanMetadataInput sceneService SceneService imageService ImageService scanSubs *subscriptionManager } -func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) error { - logger.Infof("Starting cleaning of tracked files") +func (j *purgeMissingJob) Execute(ctx context.Context, progress *job.Progress) error { + logger.Infof("Starting purging of missing files and folders") start := time.Now() - if j.input.DryRun { + if j.options.DryRun { logger.Infof("Running in Dry Mode") } - j.cleaner.Clean(ctx, file.CleanOptions{ - Paths: j.input.Paths, - DryRun: j.input.DryRun, - IgnoreZipFileContents: j.input.IgnoreZipFileContents, - PathFilter: newCleanFilter(instance.Config), - }, progress) + j.purger.PurgeMissing(ctx, j.options, progress) if job.IsCancelled(ctx) { logger.Info("Stopping due to user request") return nil } - j.cleanEmptyGalleries(ctx) + // only clean empty galleries if not in dry run mode + if !j.options.DryRun { + j.cleanEmptyGalleries(ctx) + } j.scanSubs.notify() elapsed := time.Since(start) - logger.Info(fmt.Sprintf("Finished Cleaning (%s)", elapsed)) + logger.Info(fmt.Sprintf("Finished purging missing files and folders (%s)", elapsed)) return nil } -func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) { +func (j *purgeMissingJob) cleanEmptyGalleries(ctx context.Context) { const batchSize = 1000 var toClean []int findFilter := models.BatchFindFilter(batchSize) @@ -85,7 +77,7 @@ func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) { continue } - if len(j.input.Paths) > 0 && !fsutil.IsPathInDirs(j.input.Paths, g.Path) { + if len(j.options.Paths) > 0 && !fsutil.IsPathInDirs(j.options.Paths, g.Path) { continue } @@ -102,14 +94,14 @@ func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) { return } - if !j.input.DryRun { + if !j.options.DryRun { for _, id := range toClean { j.deleteGallery(ctx, id) } } } -func (j *cleanJob) deleteGallery(ctx context.Context, id int) { +func (j *purgeMissingJob) deleteGallery(ctx context.Context, id int) { pluginCache := GetInstance().PluginCache r := j.repository @@ -143,130 +135,9 @@ func (j *cleanJob) deleteGallery(ctx context.Context, id int) { } } -type cleanFilter struct { - scanFilter -} - -func newCleanFilter(c *config.Config) *cleanFilter { - return &cleanFilter{ - scanFilter: scanFilter{ - extensionConfig: newExtensionConfig(c), - stashPaths: c.GetStashPaths(), - generatedPath: c.GetGeneratedPath(), - videoExcludeRegex: generateRegexps(c.GetExcludes()), - imageExcludeRegex: generateRegexps(c.GetImageExcludes()), - stashIgnoreFilter: file.NewStashIgnoreFilter(), - }, - } -} - -func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool { - // #1102 - clean anything in generated path - generatedPath := f.generatedPath - - var stash *config.StashConfig - fileOrFolder := "File" - - if info.IsDir() { - fileOrFolder = "Folder" - stash = f.stashPaths.GetStashFromDirPath(path) - } else { - stash = f.stashPaths.GetStashFromPath(path) - } - - if stash == nil { - logger.Infof("%s not in any stash library directories. Marking to clean: %q", fileOrFolder, path) - return false - } - - if fsutil.IsPathInDir(generatedPath, path) { - logger.Infof("%s is in generated path. Marking to clean: %q", fileOrFolder, path) - return false - } - - // Check .stashignore files, bounded to the library root. - if !f.stashIgnoreFilter.Accept(ctx, path, info, f.stashPaths.GetStashRootFromDirPath(path), zipFilePath) { - logger.Infof("%s is excluded due to .stashignore. Marking to clean: %q", fileOrFolder, path) - return false - } - - if info.IsDir() { - return !f.shouldCleanFolder(path, stash) - } - - return !f.shouldCleanFile(path, info, stash) -} - -func (f *cleanFilter) shouldCleanFolder(path string, s *config.StashConfig) bool { - // only delete folders where it is excluded from everything - pathExcludeTest := path + string(filepath.Separator) - if (s.ExcludeVideo || matchFileRegex(pathExcludeTest, f.videoExcludeRegex)) && (s.ExcludeImage || matchFileRegex(pathExcludeTest, f.imageExcludeRegex)) { - logger.Infof("Folder is excluded from both video and image. Marking to clean: \"%s\"", path) - return true - } - - return false -} - -func (f *cleanFilter) shouldCleanFile(path string, info fs.FileInfo, stash *config.StashConfig) bool { - switch { - case info.IsDir() || fsutil.MatchExtension(path, f.zipExt): - return f.shouldCleanGallery(path, stash) - case useAsVideo(path): - return f.shouldCleanVideoFile(path, stash) - case useAsImage(path): - return f.shouldCleanImage(path, stash) - default: - logger.Infof("File extension does not match any media extensions. Marking to clean: \"%s\"", path) - return true - } -} - -func (f *cleanFilter) shouldCleanVideoFile(path string, stash *config.StashConfig) bool { - if stash.ExcludeVideo { - logger.Infof("File in stash library that excludes video. Marking to clean: \"%s\"", path) - return true - } - - if matchFileRegex(path, f.videoExcludeRegex) { - logger.Infof("File matched regex. Marking to clean: \"%s\"", path) - return true - } - - return false -} - -func (f *cleanFilter) shouldCleanGallery(path string, stash *config.StashConfig) bool { - if stash.ExcludeImage { - logger.Infof("File in stash library that excludes images. Marking to clean: \"%s\"", path) - return true - } - - if matchFileRegex(path, f.imageExcludeRegex) { - logger.Infof("File matched regex. Marking to clean: \"%s\"", path) - return true - } - - return false -} - -func (f *cleanFilter) shouldCleanImage(path string, stash *config.StashConfig) bool { - if stash.ExcludeImage { - logger.Infof("File in stash library that excludes images. Marking to clean: \"%s\"", path) - return true - } - - if matchFileRegex(path, f.imageExcludeRegex) { - logger.Infof("File matched regex. Marking to clean: \"%s\"", path) - return true - } - - return false -} - -type cleanHandler struct{} +type purgeHandler struct{} -func (h *cleanHandler) HandleFile(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { +func (h *purgeHandler) HandleFile(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { if err := h.handleRelatedScenes(ctx, fileDeleter, fileID); err != nil { return err } @@ -280,11 +151,11 @@ func (h *cleanHandler) HandleFile(ctx context.Context, fileDeleter *file.Deleter return nil } -func (h *cleanHandler) HandleFolder(ctx context.Context, fileDeleter *file.Deleter, folderID models.FolderID) error { +func (h *purgeHandler) HandleFolder(ctx context.Context, fileDeleter *file.Deleter, folderID models.FolderID) error { return h.deleteRelatedFolderGalleries(ctx, folderID) } -func (h *cleanHandler) handleRelatedScenes(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { +func (h *purgeHandler) handleRelatedScenes(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { mgr := GetInstance() sceneQB := mgr.Repository.Scene scenes, err := sceneQB.FindByFileID(ctx, fileID) @@ -342,7 +213,7 @@ func (h *cleanHandler) handleRelatedScenes(ctx context.Context, fileDeleter *fil return nil } -func (h *cleanHandler) handleRelatedGalleries(ctx context.Context, fileID models.FileID) error { +func (h *purgeHandler) handleRelatedGalleries(ctx context.Context, fileID models.FileID) error { mgr := GetInstance() qb := mgr.Repository.Gallery galleries, err := qb.FindByFileID(ctx, fileID) @@ -388,7 +259,7 @@ func (h *cleanHandler) handleRelatedGalleries(ctx context.Context, fileID models return nil } -func (h *cleanHandler) deleteRelatedFolderGalleries(ctx context.Context, folderID models.FolderID) error { +func (h *purgeHandler) deleteRelatedFolderGalleries(ctx context.Context, folderID models.FolderID) error { mgr := GetInstance() qb := mgr.Repository.Gallery galleries, err := qb.FindByFolderID(ctx, folderID) @@ -412,7 +283,7 @@ func (h *cleanHandler) deleteRelatedFolderGalleries(ctx context.Context, folderI return nil } -func (h *cleanHandler) handleRelatedImages(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { +func (h *purgeHandler) handleRelatedImages(ctx context.Context, fileDeleter *file.Deleter, fileID models.FileID) error { mgr := GetInstance() imageQB := mgr.Repository.Image images, err := imageQB.FindByFileID(ctx, fileID) diff --git a/internal/manager/task_stashignore_test.go b/internal/manager/task_stashignore_test.go index 9807333fd8..3f48f3ef9e 100644 --- a/internal/manager/task_stashignore_test.go +++ b/internal/manager/task_stashignore_test.go @@ -52,7 +52,7 @@ func TestStashIgnoreUsesTopmostLibraryRootWithNestedLibraries(t *testing.T) { t.Fatalf("expected scan filter to reject file due to parent .stashignore") } - cleanFilter := &cleanFilter{ + verifyFilter := &verifyFilter{ scanFilter: scanFilter{ stashPaths: stashPaths, generatedPath: filepath.Join(root, "generated"), @@ -60,7 +60,7 @@ func TestStashIgnoreUsesTopmostLibraryRootWithNestedLibraries(t *testing.T) { }, } - if cleanFilter.Accept(context.Background(), ignoredFile, info, "") { + if verifyFilter.Accept(context.Background(), ignoredFile, info, "") { t.Fatalf("expected clean filter to reject file due to parent .stashignore") } } diff --git a/internal/manager/task_verify.go b/internal/manager/task_verify.go new file mode 100644 index 0000000000..53ac37e175 --- /dev/null +++ b/internal/manager/task_verify.go @@ -0,0 +1,185 @@ +package manager + +import ( + "context" + "fmt" + "io/fs" + "path/filepath" + "time" + + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" +) + +// TODO: ideally this would be in the task package, but deferring for now + +type verifier interface { + Verify(ctx context.Context, options file.VerifyOptions, progress *job.Progress) +} + +type verifyJob struct { + verifier verifier + repository models.Repository + options file.VerifyOptions + sceneService SceneService + imageService ImageService + scanSubs *subscriptionManager +} + +func (j *verifyJob) Execute(ctx context.Context, progress *job.Progress) error { + logger.Infof("Starting verification of tracked files") + start := time.Now() + if j.options.DryRun { + logger.Infof("Running in Dry Mode") + } + + options := j.options + options.PathFilter = newVerifyFilter(instance.Config) + + j.verifier.Verify(ctx, options, progress) + + if job.IsCancelled(ctx) { + logger.Info("Stopping due to user request") + return nil + } + + // only clean empty galleries if purging + if !j.options.DryRun && j.options.PurgeMissing { + // HACK - use purge job to clean empty galleries + pj := &purgeMissingJob{ + // only need to provide repository + repository: j.repository, + } + pj.cleanEmptyGalleries(ctx) + } + + j.scanSubs.notify() + elapsed := time.Since(start) + logger.Info(fmt.Sprintf("Finished verifying (%s)", elapsed)) + return nil +} + +type verifyFilter struct { + scanFilter +} + +func newVerifyFilter(c *config.Config) *verifyFilter { + return &verifyFilter{ + scanFilter: scanFilter{ + extensionConfig: newExtensionConfig(c), + stashPaths: c.GetStashPaths(), + generatedPath: c.GetGeneratedPath(), + videoExcludeRegex: generateRegexps(c.GetExcludes()), + imageExcludeRegex: generateRegexps(c.GetImageExcludes()), + stashIgnoreFilter: file.NewStashIgnoreFilter(), + }, + } +} + +func (f *verifyFilter) Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool { + // #1102 - clean anything in generated path + generatedPath := f.generatedPath + + var stash *config.StashConfig + fileOrFolder := "File" + + if info.IsDir() { + fileOrFolder = "Folder" + stash = f.stashPaths.GetStashFromDirPath(path) + } else { + stash = f.stashPaths.GetStashFromPath(path) + } + + if stash == nil { + logger.Infof("%s not in any stash library directories. Marking as missing: %q", fileOrFolder, path) + return false + } + + if fsutil.IsPathInDir(generatedPath, path) { + logger.Infof("%s is in generated path. Marking as missing: %q", fileOrFolder, path) + return false + } + + // Check .stashignore files, bounded to the library root. + if !f.stashIgnoreFilter.Accept(ctx, path, info, f.stashPaths.GetStashRootFromDirPath(path), zipFilePath) { + logger.Infof("%s is excluded due to .stashignore. Marking as missing: %q", fileOrFolder, path) + return false + } + + if info.IsDir() { + return !f.shouldCleanFolder(path, stash) + } + + return !f.shouldCleanFile(path, info, stash) +} + +func (f *verifyFilter) shouldCleanFolder(path string, s *config.StashConfig) bool { + // only delete folders where it is excluded from everything + pathExcludeTest := path + string(filepath.Separator) + if (s.ExcludeVideo || matchFileRegex(pathExcludeTest, f.videoExcludeRegex)) && (s.ExcludeImage || matchFileRegex(pathExcludeTest, f.imageExcludeRegex)) { + logger.Infof("Folder is excluded from both video and image. Marking as missing: \"%s\"", path) + return true + } + + return false +} + +func (f *verifyFilter) shouldCleanFile(path string, info fs.FileInfo, stash *config.StashConfig) bool { + switch { + case info.IsDir() || fsutil.MatchExtension(path, f.zipExt): + return f.shouldCleanGallery(path, stash) + case useAsVideo(path): + return f.shouldCleanVideoFile(path, stash) + case useAsImage(path): + return f.shouldCleanImage(path, stash) + default: + logger.Infof("File extension does not match any media extensions. Marking as missing: \"%s\"", path) + return true + } +} + +func (f *verifyFilter) shouldCleanVideoFile(path string, stash *config.StashConfig) bool { + if stash.ExcludeVideo { + logger.Infof("File in stash library that excludes video. Marking as missing: \"%s\"", path) + return true + } + + if matchFileRegex(path, f.videoExcludeRegex) { + logger.Infof("File matched regex. Marking as missing: \"%s\"", path) + return true + } + + return false +} + +func (f *verifyFilter) shouldCleanGallery(path string, stash *config.StashConfig) bool { + if stash.ExcludeImage { + logger.Infof("File in stash library that excludes images. Marking as missing: \"%s\"", path) + return true + } + + if matchFileRegex(path, f.imageExcludeRegex) { + logger.Infof("File matched regex. Marking as missing: \"%s\"", path) + return true + } + + return false +} + +func (f *verifyFilter) shouldCleanImage(path string, stash *config.StashConfig) bool { + if stash.ExcludeImage { + logger.Infof("File in stash library that excludes images. Marking as missing: \"%s\"", path) + return true + } + + if matchFileRegex(path, f.imageExcludeRegex) { + logger.Infof("File matched regex. Marking as missing: \"%s\"", path) + return true + } + + return false +} diff --git a/pkg/file/clean.go b/pkg/file/clean.go deleted file mode 100644 index 369600f4ce..0000000000 --- a/pkg/file/clean.go +++ /dev/null @@ -1,470 +0,0 @@ -package file - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - - "github.com/stashapp/stash/pkg/job" - "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/models" -) - -// Cleaner scans through stored file and folder instances and removes those that are no longer present on disk. -type Cleaner struct { - FS models.FS - Repository Repository - - Handlers []CleanHandler - TrashPath string -} - -type cleanJob struct { - *Cleaner - - progress *job.Progress - options CleanOptions -} - -// CleanOptions provides options for scanning files. -type CleanOptions struct { - Paths []string - - // IgnoreZipFileContents will skip checking the contents of zip files when determining whether to clean a file. - // This can significantly speed up the clean process, but will potentially miss removed files within zip files. - // Where users do not modify zip files contents directly, this should be safe to use. - IgnoreZipFileContents bool - - // Do a dry run. Don't delete any files - DryRun bool - - // PathFilter are used to determine if a file should be included. - // Excluded files are marked for cleaning. - PathFilter PathFilter -} - -// Clean starts the clean process. -func (s *Cleaner) Clean(ctx context.Context, options CleanOptions, progress *job.Progress) { - j := &cleanJob{ - Cleaner: s, - progress: progress, - options: options, - } - - if err := j.execute(ctx); err != nil { - logger.Errorf("error cleaning files: %v", err) - return - } -} - -type fileOrFolder struct { - fileID models.FileID - folderID models.FolderID -} - -type deleteSet struct { - orderedList []fileOrFolder - fileIDSet map[models.FileID]string - - folderIDSet map[models.FolderID]string -} - -func newDeleteSet() deleteSet { - return deleteSet{ - fileIDSet: make(map[models.FileID]string), - folderIDSet: make(map[models.FolderID]string), - } -} - -func (s *deleteSet) add(id models.FileID, path string) { - if _, ok := s.fileIDSet[id]; !ok { - s.orderedList = append(s.orderedList, fileOrFolder{fileID: id}) - s.fileIDSet[id] = path - } -} - -func (s *deleteSet) has(id models.FileID) bool { - _, ok := s.fileIDSet[id] - return ok -} - -func (s *deleteSet) addFolder(id models.FolderID, path string) { - if _, ok := s.folderIDSet[id]; !ok { - s.orderedList = append(s.orderedList, fileOrFolder{folderID: id}) - s.folderIDSet[id] = path - } -} - -func (s *deleteSet) hasFolder(id models.FolderID) bool { - _, ok := s.folderIDSet[id] - return ok -} - -func (s *deleteSet) len() int { - return len(s.orderedList) -} - -func (j *cleanJob) execute(ctx context.Context) error { - progress := j.progress - - toDelete := newDeleteSet() - - var ( - fileCount int - folderCount int - ) - - r := j.Repository - if err := r.WithReadTxn(ctx, func(ctx context.Context) error { - var err error - fileCount, err = r.File.CountAllInPaths(ctx, j.options.Paths) - if err != nil { - return err - } - - folderCount, err = r.Folder.CountAllInPaths(ctx, j.options.Paths) - if err != nil { - return err - } - - return nil - }); err != nil { - return err - } - - progress.AddTotal(fileCount + folderCount) - progress.Definite() - - if err := j.assessFiles(ctx, &toDelete); err != nil { - return err - } - - if err := j.assessFolders(ctx, &toDelete); err != nil { - return err - } - - if j.options.DryRun && toDelete.len() > 0 { - // add progress for files that would've been deleted - progress.AddProcessed(toDelete.len()) - return nil - } - - progress.ExecuteTask(fmt.Sprintf("Cleaning %d files and folders", toDelete.len()), func() { - for _, ff := range toDelete.orderedList { - if job.IsCancelled(ctx) { - return - } - - if ff.fileID != 0 { - j.deleteFile(ctx, ff.fileID, toDelete.fileIDSet[ff.fileID]) - } - if ff.folderID != 0 { - j.deleteFolder(ctx, ff.folderID, toDelete.folderIDSet[ff.folderID]) - } - - progress.Increment() - } - }) - - return nil -} - -func (j *cleanJob) assessFiles(ctx context.Context, toDelete *deleteSet) error { - const batchSize = 1000 - offset := 0 - progress := j.progress - - more := true - r := j.Repository - - includeZipContents := !j.options.IgnoreZipFileContents - - if err := r.WithReadTxn(ctx, func(ctx context.Context) error { - for more { - if job.IsCancelled(ctx) { - return nil - } - - files, err := r.File.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset) - if err != nil { - return fmt.Errorf("error querying for files: %w", err) - } - - for _, f := range files { - path := f.Base().Path - err = nil - fileID := f.Base().ID - - // short-cut, don't assess if already added - if toDelete.has(fileID) { - continue - } - - progress.ExecuteTask(fmt.Sprintf("Assessing file %s for clean", path), func() { - if j.shouldClean(ctx, f) { - err = j.flagFileForDelete(ctx, toDelete, f) - } else { - // increment progress, no further processing - progress.Increment() - } - }) - if err != nil { - return err - } - } - - if len(files) != batchSize { - more = false - } else { - offset += batchSize - } - } - - return nil - }); err != nil { - return err - } - - return nil -} - -// flagFolderForDelete adds folders to the toDelete set, with the leaf folders added first -func (j *cleanJob) flagFileForDelete(ctx context.Context, toDelete *deleteSet, f models.File) error { - r := j.Repository - // add contained files first - containedFiles, err := r.File.FindByZipFileID(ctx, f.Base().ID) - if err != nil { - return fmt.Errorf("error finding contained files for %q: %w", f.Base().Path, err) - } - - for _, cf := range containedFiles { - logger.Infof("Marking contained file %q to clean", cf.Base().Path) - toDelete.add(cf.Base().ID, cf.Base().Path) - } - - // add contained folders as well - containedFolders, err := r.Folder.FindByZipFileID(ctx, f.Base().ID) - if err != nil { - return fmt.Errorf("error finding contained folders for %q: %w", f.Base().Path, err) - } - - for _, cf := range containedFolders { - logger.Infof("Marking contained folder %q to clean", cf.Path) - toDelete.addFolder(cf.ID, cf.Path) - } - - toDelete.add(f.Base().ID, f.Base().Path) - - return nil -} - -func (j *cleanJob) assessFolders(ctx context.Context, toDelete *deleteSet) error { - const batchSize = 1000 - offset := 0 - progress := j.progress - - includeZipContents := !j.options.IgnoreZipFileContents - - more := true - r := j.Repository - if err := r.WithReadTxn(ctx, func(ctx context.Context) error { - for more { - if job.IsCancelled(ctx) { - return nil - } - - folders, err := r.Folder.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset) - if err != nil { - return fmt.Errorf("error querying for folders: %w", err) - } - - for _, f := range folders { - path := f.Path - folderID := f.ID - - // short-cut, don't assess if already added - if toDelete.hasFolder(folderID) { - continue - } - - err = nil - progress.ExecuteTask(fmt.Sprintf("Assessing folder %s for clean", path), func() { - if j.shouldCleanFolder(ctx, f) { - if err = j.flagFolderForDelete(ctx, toDelete, f); err != nil { - return - } - } else { - // increment progress, no further processing - progress.Increment() - } - }) - if err != nil { - return err - } - } - - if len(folders) != batchSize { - more = false - } else { - offset += batchSize - } - } - - return nil - }); err != nil { - return err - } - - return nil -} - -func (j *cleanJob) flagFolderForDelete(ctx context.Context, toDelete *deleteSet, folder *models.Folder) error { - // it is possible that child folders may be included while parent folders are not - // so we need to check child folders separately - toDelete.addFolder(folder.ID, folder.Path) - - return nil -} - -func isNotFound(err error) bool { - // ErrInvalid can occur in zip files where the zip file path changed - // and the underlying folder did not - // #3877 - fs.PathError can occur if the network share no longer exists - var pathErr *fs.PathError - return err != nil && - (errors.Is(err, fs.ErrNotExist) || - errors.Is(err, fs.ErrInvalid) || - errors.As(err, &pathErr)) -} - -func (j *cleanJob) shouldClean(ctx context.Context, f models.File) bool { - path := f.Base().Path - - info, err := f.Base().Info(j.FS) - if err != nil && !isNotFound(err) { - logger.Errorf("error getting file info for %q, not cleaning: %v", path, err) - return false - } - - if info == nil { - // info is nil - file not exist - logger.Infof("File not found. Marking to clean: \"%s\"", path) - return true - } - - // run through path filter, if returns false then the file should be cleaned - filter := j.options.PathFilter - - // need to get the zip file path if present - zipFilePath := "" - if f.Base().ZipFile != nil { - zipFilePath = f.Base().ZipFile.Base().Path - } - - // don't log anything - assume filter will have logged the reason - return !filter.Accept(ctx, path, info, zipFilePath) -} - -func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *models.Folder) bool { - path := f.Path - - info, err := f.Info(j.FS) - - if err != nil && !isNotFound(err) { - logger.Errorf("error getting folder info for %q, not cleaning: %v", path, err) - return false - } - - if info == nil { - // info is nil - file not exist - logger.Infof("Folder not found. Marking to clean: \"%s\"", path) - return true - } - - // #3261 - handle symlinks - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - finalPath, err := filepath.EvalSymlinks(path) - if err != nil { - // don't bail out if symlink is invalid - logger.Infof("Invalid symlink. Marking to clean: \"%s\"", path) - return true - } - - info, err = j.FS.Lstat(finalPath) - if err != nil && !isNotFound(err) { - logger.Errorf("error getting file info for %q (-> %s), not cleaning: %v", path, finalPath, err) - return false - } - } - - // run through path filter, if returns false then the file should be cleaned - filter := j.options.PathFilter - - // need to get the zip file path if present - zipFilePath := "" - if f.ZipFile != nil { - zipFilePath = f.ZipFile.Base().Path - } - - // don't log anything - assume filter will have logged the reason - return !filter.Accept(ctx, path, info, zipFilePath) -} - -func (j *cleanJob) deleteFile(ctx context.Context, fileID models.FileID, fn string) { - // delete associated objects - fileDeleter := NewDeleterWithTrash(j.TrashPath) - r := j.Repository - if err := r.WithTxn(ctx, func(ctx context.Context) error { - fileDeleter.RegisterHooks(ctx) - - if err := j.fireHandlers(ctx, fileDeleter, fileID); err != nil { - return err - } - - return r.File.Destroy(ctx, fileID) - }); err != nil { - logger.Errorf("Error deleting file %q from database: %s", fn, err.Error()) - return - } -} - -func (j *cleanJob) deleteFolder(ctx context.Context, folderID models.FolderID, fn string) { - // delete associated objects - fileDeleter := NewDeleterWithTrash(j.TrashPath) - r := j.Repository - if err := r.WithTxn(ctx, func(ctx context.Context) error { - fileDeleter.RegisterHooks(ctx) - - if err := j.fireFolderHandlers(ctx, fileDeleter, folderID); err != nil { - return err - } - - return r.Folder.Destroy(ctx, folderID) - }); err != nil { - logger.Errorf("Error deleting folder %q from database: %s", fn, err.Error()) - return - } -} - -func (j *cleanJob) fireHandlers(ctx context.Context, fileDeleter *Deleter, fileID models.FileID) error { - for _, h := range j.Handlers { - if err := h.HandleFile(ctx, fileDeleter, fileID); err != nil { - return err - } - } - - return nil -} - -func (j *cleanJob) fireFolderHandlers(ctx context.Context, fileDeleter *Deleter, folderID models.FolderID) error { - for _, h := range j.Handlers { - if err := h.HandleFolder(ctx, fileDeleter, folderID); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/file/handler.go b/pkg/file/handler.go index b4056f1958..c7f6301c1a 100644 --- a/pkg/file/handler.go +++ b/pkg/file/handler.go @@ -48,8 +48,8 @@ func (h *FilteredHandler) Handle(ctx context.Context, f models.File, oldFile mod return nil } -// CleanHandler provides a handler for cleaning Files and Folders. -type CleanHandler interface { +// PurgeHandler provides a handler for cleaning Files and Folders. +type PurgeHandler interface { HandleFile(ctx context.Context, fileDeleter *Deleter, fileID models.FileID) error HandleFolder(ctx context.Context, fileDeleter *Deleter, folderID models.FolderID) error } diff --git a/pkg/file/purge_missing.go b/pkg/file/purge_missing.go new file mode 100644 index 0000000000..9e85eebaa5 --- /dev/null +++ b/pkg/file/purge_missing.go @@ -0,0 +1,243 @@ +package file + +import ( + "context" + "fmt" + "time" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" +) + +// MissingPurger purges files and folders marked as missing and their associated objects from the database. +type MissingPurger struct { + Repository Repository + + PurgeHandlers []PurgeHandler + TrashPath string +} + +type purgeMissingJob struct { + *MissingPurger + + progress *job.Progress + options PurgeMissingOptions +} + +// PurgeMissingOptions provides options for purging missing files. +type PurgeMissingOptions struct { + Paths []string + + // DryRun indicates if this is a dry run. A dry run will not make any changes and will only log what would have been done. + DryRun bool + + // MissingSinceBefore is an optional filter to only purge files and folders that have been marked as missing since before the specified time. + MissingSinceBefore *time.Time +} + +// PurgeMissing starts the purge missing process. +func (s *MissingPurger) PurgeMissing(ctx context.Context, options PurgeMissingOptions, progress *job.Progress) { + j := &purgeMissingJob{ + MissingPurger: s, + progress: progress, + options: options, + } + + if err := j.execute(ctx); err != nil { + logger.Errorf("error purging missing files: %v", err) + return + } +} + +func (j *purgeMissingJob) execute(ctx context.Context) error { + progress := j.progress + + var ( + fileCount int + folderCount int + ) + + r := j.Repository + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + var err error + fileCount, err = r.File.CountMissingInPaths(ctx, j.options.Paths, j.options.MissingSinceBefore) + if err != nil { + return err + } + + folderCount, err = r.Folder.CountMissingInPaths(ctx, j.options.Paths, j.options.MissingSinceBefore) + if err != nil { + return err + } + + return nil + }); err != nil { + return err + } + + progress.AddTotal(fileCount + folderCount) + progress.Definite() + + if err := j.purgeMissingFiles(ctx, progress); err != nil { + return err + } + + if err := j.purgeMissingFolders(ctx); err != nil { + return err + } + + return nil +} + +func (j *purgeMissingJob) purgeMissingFiles(ctx context.Context, progress *job.Progress) error { + const batchSize = 1000 + + offset := 0 + more := true + r := j.Repository + + for more { + var files []models.File + + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + if job.IsCancelled(ctx) { + return nil + } + + var err error + files, err = r.File.FindMissingInPaths(ctx, j.options.Paths, j.options.MissingSinceBefore, batchSize, offset) + if err != nil { + return fmt.Errorf("error querying for files: %w", err) + } + + return nil + }); err != nil { + return err + } + + for _, f := range files { + if j.options.DryRun { + logger.Infof("Would delete file %q from database", f.Base().Path) + } else { + logger.Infof("Deleting file %q from database", f.Base().Path) + j.deleteFile(ctx, f.Base().ID, f.Base().Path) + } + progress.Increment() + } + + if len(files) != batchSize { + more = false + } else if j.options.DryRun { + // when not in dry run, we should be continuing until there's none left + // in dry run, we can just increment the offset and continue to the next batch + offset += batchSize + } + } + + return nil +} + +func (j *purgeMissingJob) purgeMissingFolders(ctx context.Context) error { + const batchSize = 1000 + offset := 0 + progress := j.progress + + more := true + r := j.Repository + for more { + if job.IsCancelled(ctx) { + return nil + } + + var folders []*models.Folder + + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + var err error + folders, err = r.Folder.FindMissingInPaths(ctx, j.options.Paths, j.options.MissingSinceBefore, batchSize, offset) + if err != nil { + return fmt.Errorf("error querying for folders: %w", err) + } + + return nil + }); err != nil { + return err + } + + for _, f := range folders { + if j.options.DryRun { + logger.Infof("Would delete folder %q from database", f.Path) + } else { + logger.Infof("Deleting folder %q from database", f.Path) + j.deleteFolder(ctx, f.ID, f.Path) + } + progress.Increment() + } + + if len(folders) != batchSize { + more = false + } else if j.options.DryRun { + // when not in dry run, we should be continuing until there's none left + // in dry run, we can just increment the offset and continue to the next batch + offset += batchSize + } + } + + return nil +} + +func (j *purgeMissingJob) deleteFile(ctx context.Context, fileID models.FileID, fn string) { + // delete associated objects + fileDeleter := NewDeleterWithTrash(j.TrashPath) + r := j.Repository + if err := r.WithTxn(ctx, func(ctx context.Context) error { + fileDeleter.RegisterHooks(ctx) + + if err := j.fireHandlers(ctx, fileDeleter, fileID); err != nil { + return err + } + + return r.File.Destroy(ctx, fileID) + }); err != nil { + logger.Errorf("Error deleting file %q from database: %s", fn, err.Error()) + return + } +} + +func (j *purgeMissingJob) deleteFolder(ctx context.Context, folderID models.FolderID, fn string) { + // delete associated objects + fileDeleter := NewDeleterWithTrash(j.TrashPath) + r := j.Repository + if err := r.WithTxn(ctx, func(ctx context.Context) error { + fileDeleter.RegisterHooks(ctx) + + if err := j.fireFolderHandlers(ctx, fileDeleter, folderID); err != nil { + return err + } + + return r.Folder.Destroy(ctx, folderID) + }); err != nil { + logger.Errorf("Error deleting folder %q from database: %s", fn, err.Error()) + return + } +} + +func (j *purgeMissingJob) fireHandlers(ctx context.Context, fileDeleter *Deleter, fileID models.FileID) error { + for _, h := range j.PurgeHandlers { + if err := h.HandleFile(ctx, fileDeleter, fileID); err != nil { + return err + } + } + + return nil +} + +func (j *purgeMissingJob) fireFolderHandlers(ctx context.Context, fileDeleter *Deleter, folderID models.FolderID) error { + for _, h := range j.PurgeHandlers { + if err := h.HandleFolder(ctx, fileDeleter, folderID); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/file/scan.go b/pkg/file/scan.go index c9a5ca52d8..dee2b2e43b 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -267,6 +267,9 @@ func (s *Scanner) handleFolderRename(ctx context.Context, file ScannedFile) (*mo logger.Infof("%s moved to %s. Updating path...", renamedFrom.Path, file.Path) renamedFrom.Path = file.Path + // clear any missing flag, since the folder is no longer missing + renamedFrom.MissingSince = nil + // update the parent folder ID // find the parent folder parentFolderID, err := s.getFolderID(ctx, filepath.Dir(file.Path)) @@ -291,6 +294,13 @@ func (s *Scanner) handleFolderRename(ctx context.Context, file ScannedFile) (*mo func (s *Scanner) onExistingFolder(ctx context.Context, f ScannedFile, existing *models.Folder) (*models.Folder, error) { update := false + // update if missing + if existing.MissingSince != nil { + logger.Infof("Marking folder %q as no longer missing.", existing.Path) + existing.MissingSince = nil + update = true + } + // update if mod time is changed entryModTime := f.ModTime if !entryModTime.Equal(existing.ModTime) { @@ -650,6 +660,8 @@ func (s *Scanner) handleRename(ctx context.Context, f models.File, fp []models.F fBaseCopy.ID = updatedBase.ID fBaseCopy.CreatedAt = updatedBase.CreatedAt fBaseCopy.Fingerprints = updatedBase.Fingerprints + // clear missing since flag, since the file is no longer missing + fBaseCopy.MissingSince = nil *updatedBase = fBaseCopy zipMover := zipHierarchyMover{ @@ -790,6 +802,9 @@ func (s *Scanner) onExistingFile(ctx context.Context, f ScannedFile, existing mo base.Size = f.Size base.UpdatedAt = time.Now() + // clear any missing flag, since the file is no longer missing + base.MissingSince = nil + // calculate and update fingerprints for the file const useExisting = false fp, err := s.calculateFingerprints(f.FS, base, path, useExisting) @@ -867,6 +882,18 @@ func (s *Scanner) removeOutdatedFingerprints(existing models.File, fp models.Fin func (s *Scanner) onUnchangedFile(ctx context.Context, f ScannedFile, existing models.File) (*ScanFileResult, error) { var err error + if existing.Base().MissingSince != nil { + logger.Infof("Marking file %q as no longer missing.", existing.Base().Path) + if err := s.Repository.WithTxn(ctx, func(ctx context.Context) error { + if err := s.Repository.File.SetMissing(ctx, existing.Base().ID, nil); err != nil { + return fmt.Errorf("updating file %q: %w", existing.Base().Path, err) + } + return nil + }); err != nil { + return nil, err + } + } + isMissingMetdata := s.isMissingMetadata(ctx, f, existing) // set missing information if isMissingMetdata { diff --git a/pkg/file/verify.go b/pkg/file/verify.go new file mode 100644 index 0000000000..a613d8132b --- /dev/null +++ b/pkg/file/verify.go @@ -0,0 +1,617 @@ +package file + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" +) + +// Verifier scans through stored file and folder instances and marks missing those that are no longer present on disk. +type Verifier struct { + FS models.FS + Repository Repository + + PurgeHandlers []PurgeHandler + TrashPath string +} + +type missingFileFn func(ctx context.Context, f models.File) (processed bool, err error) +type missingFolderFn func(ctx context.Context, f *models.Folder) (processed bool, err error) + +type verifyJob struct { + *Verifier + + progress *job.Progress + options VerifyOptions + + toDelete deleteSet + + missingFileHandler missingFileFn + missingFolderHandler missingFolderFn +} + +// VerifyOptions provides options for verifying files. +type VerifyOptions struct { + Paths []string + + // IgnoreZipFileContents will skip checking the contents of zip files when determining whether to verify a file. + // This can significantly speed up the verify process, but will potentially miss removed files within zip files. + // Where users do not modify zip files contents directly, this should be safe to use. + IgnoreZipFileContents bool + + // DryRun indicates if this is a dry run. A dry run will not make any changes and will only log what would have been done. + DryRun bool + + // PurgeMissing indicates if missing files should be purged. If true, missing files will be deleted from the database and any associated objects will be deleted as well. + // No effect if DryRun is true. + PurgeMissing bool + + // PathFilter are used to determine if a file should be included. + // Excluded files are marked for cleaning. + PathFilter PathFilter +} + +// Verify starts the verify process. +func (s *Verifier) Verify(ctx context.Context, options VerifyOptions, progress *job.Progress) { + j := &verifyJob{ + Verifier: s, + progress: progress, + options: options, + } + + if err := j.execute(ctx); err != nil { + logger.Errorf("error verifying files: %v", err) + return + } +} + +type fileOrFolder struct { + fileID models.FileID + folderID models.FolderID +} + +type deleteSet struct { + orderedList []fileOrFolder + fileIDSet map[models.FileID]string + + folderIDSet map[models.FolderID]string +} + +func newDeleteSet() deleteSet { + return deleteSet{ + fileIDSet: make(map[models.FileID]string), + folderIDSet: make(map[models.FolderID]string), + } +} + +func (s *deleteSet) add(id models.FileID, path string) { + if _, ok := s.fileIDSet[id]; !ok { + s.orderedList = append(s.orderedList, fileOrFolder{fileID: id}) + s.fileIDSet[id] = path + } +} + +func (s *deleteSet) has(id models.FileID) bool { + _, ok := s.fileIDSet[id] + return ok +} + +func (s *deleteSet) addFolder(id models.FolderID, path string) { + if _, ok := s.folderIDSet[id]; !ok { + s.orderedList = append(s.orderedList, fileOrFolder{folderID: id}) + s.folderIDSet[id] = path + } +} + +func (s *deleteSet) hasFolder(id models.FolderID) bool { + _, ok := s.folderIDSet[id] + return ok +} + +func (s *deleteSet) len() int { + return len(s.orderedList) +} + +func (j *verifyJob) init() { + j.toDelete = newDeleteSet() + + switch { + case j.options.DryRun: + j.missingFileHandler = j.noopMissingFile + j.missingFolderHandler = j.noopMissingFolder + case j.options.PurgeMissing: + j.missingFileHandler = j.markToDeleteMissingFile + j.missingFolderHandler = j.markToDeleteMissingFolder + default: + j.missingFileHandler = j.markMissingFile + j.missingFolderHandler = j.markMissingFolder + } +} + +func (j *verifyJob) noopMissingFile(ctx context.Context, f models.File) (processed bool, err error) { + return true, nil +} + +func (j *verifyJob) markMissingFile(ctx context.Context, f models.File) (processed bool, err error) { + missingSince := time.Now() + if err := j.Repository.File.SetMissing(ctx, f.Base().ID, &missingSince); err != nil { + return false, err + } + return true, nil +} + +func (j *verifyJob) markToDeleteMissingFile(ctx context.Context, f models.File) (processed bool, err error) { + j.toDelete.add(f.Base().ID, f.Base().Path) + // needs to be deleted, mark as not processed + return false, nil +} + +func (j *verifyJob) noopMissingFolder(ctx context.Context, f *models.Folder) (processed bool, err error) { + return true, nil +} + +func (j *verifyJob) markMissingFolder(ctx context.Context, f *models.Folder) (processed bool, err error) { + missingSince := time.Now() + if err := j.Repository.Folder.SetMissing(ctx, f.ID, &missingSince); err != nil { + return false, err + } + return true, nil +} + +func (j *verifyJob) markToDeleteMissingFolder(ctx context.Context, f *models.Folder) (processed bool, err error) { + j.toDelete.addFolder(f.ID, f.Path) + // needs to be deleted, mark as not processed + return false, nil +} + +func (j *verifyJob) execute(ctx context.Context) error { + j.init() + progress := j.progress + + var ( + fileCount int + folderCount int + ) + + r := j.Repository + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + var err error + fileCount, err = r.File.CountAllInPaths(ctx, j.options.Paths) + if err != nil { + return err + } + + folderCount, err = r.Folder.CountAllInPaths(ctx, j.options.Paths) + if err != nil { + return err + } + + return nil + }); err != nil { + return err + } + + progress.AddTotal(fileCount + folderCount) + progress.Definite() + + if err := j.assessFolders(ctx); err != nil { + return err + } + + if err := j.assessFiles(ctx); err != nil { + return err + } + + if j.options.DryRun || !j.options.PurgeMissing { + // nothing further to do + return nil + } + + progress.ExecuteTask(fmt.Sprintf("Purging %d files and folders", j.toDelete.len()), func() { + for _, ff := range j.toDelete.orderedList { + if job.IsCancelled(ctx) { + return + } + + if ff.fileID != 0 { + j.deleteFile(ctx, ff.fileID, j.toDelete.fileIDSet[ff.fileID]) + } + if ff.folderID != 0 { + j.deleteFolder(ctx, ff.folderID, j.toDelete.folderIDSet[ff.folderID]) + } + + progress.Increment() + } + }) + + return nil +} + +func (j *verifyJob) assessFiles(ctx context.Context) error { + const batchSize = 1000 + offset := 0 + progress := j.progress + + more := true + r := j.Repository + + includeZipContents := !j.options.IgnoreZipFileContents + + for more { + if job.IsCancelled(ctx) { + return nil + } + + var files []models.File + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + var err error + files, err = r.File.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset) + if err != nil { + return fmt.Errorf("error querying for files: %w", err) + } + + return nil + }); err != nil { + return err + } + + for _, f := range files { + path := f.Base().Path + fileID := f.Base().ID + + // short-cut, don't assess if already missing + if f.Base().MissingSince != nil { + // increment progress, no further processing + progress.Increment() + continue + } + + // skip if already added to delete set + // don't increment progress here, as it will be incremented when the file is processed + if j.toDelete.has(fileID) { + continue + } + + var err error + progress.ExecuteTask(fmt.Sprintf("Verifying file %s", path), func() { + if j.shouldClean(ctx, f) { + err = j.handleMissingFile(ctx, f) + } else { + // increment progress, no further processing + progress.Increment() + } + }) + if err != nil { + return err + } + } + + if len(files) != batchSize { + more = false + } else { + offset += batchSize + } + } + + return nil +} + +// flagFolderForDelete adds folders to the toDelete set, with the leaf folders added first +func (j *verifyJob) handleMissingFile(ctx context.Context, f models.File) error { + r := j.Repository + + // do all this in a transaction so that all contained files are marked in a single transaction + if err := r.WithTxn(ctx, func(ctx context.Context) error { + // add contained files first + containedFiles, err := r.File.FindByZipFileID(ctx, f.Base().ID) + if err != nil { + return fmt.Errorf("error finding contained files for %q: %w", f.Base().Path, err) + } + + for _, cf := range containedFiles { + logger.Infof("Marking contained file %q to clean", cf.Base().Path) + processed, err := j.missingFileHandler(ctx, cf) + if err != nil { + return err + } + + if processed { + j.progress.Increment() + } + } + + // add contained folders as well + containedFolders, err := r.Folder.FindByZipFileID(ctx, f.Base().ID) + if err != nil { + return fmt.Errorf("error finding contained folders for %q: %w", f.Base().Path, err) + } + + for _, cf := range containedFolders { + logger.Infof("Marking contained folder %q to clean", cf.Path) + processed, err := j.missingFolderHandler(ctx, cf) + if err != nil { + return err + } + + if processed { + j.progress.Increment() + } + } + + processed, err := j.missingFileHandler(ctx, f) + if err != nil { + return err + } + + if processed { + j.progress.Increment() + } + + return nil + }); err != nil { + return err + } + + return nil +} + +func (j *verifyJob) assessFolders(ctx context.Context) error { + const batchSize = 1000 + offset := 0 + progress := j.progress + + includeZipContents := !j.options.IgnoreZipFileContents + + more := true + r := j.Repository + + for more { + if job.IsCancelled(ctx) { + return nil + } + + var folders []*models.Folder + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + var err error + folders, err = r.Folder.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset) + if err != nil { + return fmt.Errorf("error querying for folders: %w", err) + } + + return nil + }); err != nil { + return err + } + + for _, f := range folders { + path := f.Path + folderID := f.ID + + // don't assess if already missing + if f.MissingSince != nil { + // increment progress, no further processing + progress.Increment() + continue + } + + // skip if already added to delete set + // don't increment progress here, as it will be incremented when the folder is processed + if j.toDelete.hasFolder(folderID) { + continue + } + + var err error + progress.ExecuteTask(fmt.Sprintf("Verifying folder %s", path), func() { + if j.shouldCleanFolder(ctx, f) { + err = j.handleMissingFolder(ctx, f) + } else { + // increment progress, no further processing + progress.Increment() + } + }) + if err != nil { + return err + } + } + + if len(folders) != batchSize { + more = false + } else { + offset += batchSize + } + } + + return nil +} + +func (j *verifyJob) handleMissingFolder(ctx context.Context, folder *models.Folder) error { + r := j.Repository + + // do all this in a transaction so that all contained files are marked in a single transaction + if err := r.WithTxn(ctx, func(ctx context.Context) error { + // add contained files first + containedFiles, err := r.File.FindByFolderID(ctx, folder.ID) + if err != nil { + return fmt.Errorf("error finding contained files for %q: %w", folder.Path, err) + } + + for _, cf := range containedFiles { + logger.Infof("Marking contained file %q to clean", cf.Base().Path) + processed, err := j.missingFileHandler(ctx, cf) + if err != nil { + return err + } + + if processed { + j.progress.Increment() + } + } + + // it is possible that child folders may be included while parent folders are not + // so we need to check child folders separately + + // only use the processed return value for the top-level folder + processed, err := j.missingFolderHandler(ctx, folder) + if err != nil { + return err + } + + if processed { + j.progress.Increment() + } + + return nil + }); err != nil { + return err + } + + return nil +} + +func isNotFound(err error) bool { + // ErrInvalid can occur in zip files where the zip file path changed + // and the underlying folder did not + // #3877 - fs.PathError can occur if the network share no longer exists + var pathErr *fs.PathError + return err != nil && + (errors.Is(err, fs.ErrNotExist) || + errors.Is(err, fs.ErrInvalid) || + errors.As(err, &pathErr)) +} + +func (j *verifyJob) shouldClean(ctx context.Context, f models.File) bool { + path := f.Base().Path + + info, err := f.Base().Info(j.FS) + if err != nil && !isNotFound(err) { + logger.Errorf("error getting file info for %q, not cleaning: %v", path, err) + return false + } + + if info == nil { + // info is nil - file not exist + logger.Infof("File not found. Marking as missing: \"%s\"", path) + return true + } + + // run through path filter, if returns false then the file should be cleaned + filter := j.options.PathFilter + + // need to get the zip file path if present + zipFilePath := "" + if f.Base().ZipFile != nil { + zipFilePath = f.Base().ZipFile.Base().Path + } + + // don't log anything - assume filter will have logged the reason + return !filter.Accept(ctx, path, info, zipFilePath) +} + +func (j *verifyJob) shouldCleanFolder(ctx context.Context, f *models.Folder) bool { + path := f.Path + + info, err := f.Info(j.FS) + + if err != nil && !isNotFound(err) { + logger.Errorf("error getting folder info for %q, not cleaning: %v", path, err) + return false + } + + if info == nil { + // info is nil - file not exist + logger.Infof("Folder not found. Marking as missing: \"%s\"", path) + return true + } + + // #3261 - handle symlinks + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + finalPath, err := filepath.EvalSymlinks(path) + if err != nil { + // don't bail out if symlink is invalid + logger.Infof("Invalid symlink. Marking as missing: \"%s\"", path) + return true + } + + info, err = j.FS.Lstat(finalPath) + if err != nil && !isNotFound(err) { + logger.Errorf("error getting file info for %q (-> %s), not cleaning: %v", path, finalPath, err) + return false + } + } + + // run through path filter, if returns false then the file should be cleaned + filter := j.options.PathFilter + + // need to get the zip file path if present + zipFilePath := "" + if f.ZipFile != nil { + zipFilePath = f.ZipFile.Base().Path + } + + // don't log anything - assume filter will have logged the reason + return !filter.Accept(ctx, path, info, zipFilePath) +} + +func (j *verifyJob) deleteFile(ctx context.Context, fileID models.FileID, fn string) { + // delete associated objects + fileDeleter := NewDeleterWithTrash(j.TrashPath) + r := j.Repository + if err := r.WithTxn(ctx, func(ctx context.Context) error { + fileDeleter.RegisterHooks(ctx) + + if err := j.fireHandlers(ctx, fileDeleter, fileID); err != nil { + return err + } + + return r.File.Destroy(ctx, fileID) + }); err != nil { + logger.Errorf("Error deleting file %q from database: %s", fn, err.Error()) + return + } +} + +func (j *verifyJob) deleteFolder(ctx context.Context, folderID models.FolderID, fn string) { + // delete associated objects + fileDeleter := NewDeleterWithTrash(j.TrashPath) + r := j.Repository + if err := r.WithTxn(ctx, func(ctx context.Context) error { + fileDeleter.RegisterHooks(ctx) + + if err := j.fireFolderHandlers(ctx, fileDeleter, folderID); err != nil { + return err + } + + return r.Folder.Destroy(ctx, folderID) + }); err != nil { + logger.Errorf("Error deleting folder %q from database: %s", fn, err.Error()) + return + } +} + +func (j *verifyJob) fireHandlers(ctx context.Context, fileDeleter *Deleter, fileID models.FileID) error { + for _, h := range j.PurgeHandlers { + if err := h.HandleFile(ctx, fileDeleter, fileID); err != nil { + return err + } + } + + return nil +} + +func (j *verifyJob) fireFolderHandlers(ctx context.Context, fileDeleter *Deleter, folderID models.FolderID) error { + for _, h := range j.PurgeHandlers { + if err := h.HandleFolder(ctx, fileDeleter, folderID); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/models/file.go b/pkg/models/file.go index 32263319c7..7186c6e911 100644 --- a/pkg/models/file.go +++ b/pkg/models/file.go @@ -36,6 +36,7 @@ type FileFilterType struct { ScenesFilter *SceneFilterType `json:"scenes_filter"` ImagesFilter *ImageFilterType `json:"images_filter"` GalleriesFilter *GalleryFilterType `json:"galleries_filter"` + MissingSince *TimestampCriterionInput `json:"missing_since"` CreatedAt *TimestampCriterionInput `json:"created_at"` UpdatedAt *TimestampCriterionInput `json:"updated_at"` } diff --git a/pkg/models/folder.go b/pkg/models/folder.go index e9e9a3971e..ec2c42f135 100644 --- a/pkg/models/folder.go +++ b/pkg/models/folder.go @@ -22,6 +22,7 @@ type FolderFilterType struct { Basename *StringCriterionInput `json:"basename,omitempty"` ParentFolder *HierarchicalMultiCriterionInput `json:"parent_folder,omitempty"` ZipFile *MultiCriterionInput `json:"zip_file,omitempty"` + MissingSince *TimestampCriterionInput `json:"missing_since"` // Filter by modification time ModTime *TimestampCriterionInput `json:"mod_time,omitempty"` GalleryCount *IntCriterionInput `json:"gallery_count,omitempty"` diff --git a/pkg/models/mocks/FileReaderWriter.go b/pkg/models/mocks/FileReaderWriter.go index 4b370459e4..139fffbfc0 100644 --- a/pkg/models/mocks/FileReaderWriter.go +++ b/pkg/models/mocks/FileReaderWriter.go @@ -9,6 +9,8 @@ import ( mock "github.com/stretchr/testify/mock" models "github.com/stashapp/stash/pkg/models" + + time "time" ) // FileReaderWriter is an autogenerated mock type for the FileReaderWriter type @@ -58,6 +60,27 @@ func (_m *FileReaderWriter) CountByFolderID(ctx context.Context, folderID models return r0, r1 } +// CountMissingInPaths provides a mock function with given fields: ctx, p, missingSinceBefore +func (_m *FileReaderWriter) CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) { + ret := _m.Called(ctx, p, missingSinceBefore) + + var r0 int + if rf, ok := ret.Get(0).(func(context.Context, []string, *time.Time) int); ok { + r0 = rf(ctx, p, missingSinceBefore) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []string, *time.Time) error); ok { + r1 = rf(ctx, p, missingSinceBefore) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Create provides a mock function with given fields: ctx, f func (_m *FileReaderWriter) Create(ctx context.Context, f models.File) error { ret := _m.Called(ctx, f) @@ -222,6 +245,29 @@ func (_m *FileReaderWriter) FindByFingerprint(ctx context.Context, fp models.Fin return r0, r1 } +// FindByFolderID provides a mock function with given fields: ctx, folderID +func (_m *FileReaderWriter) FindByFolderID(ctx context.Context, folderID models.FolderID) ([]models.File, error) { + ret := _m.Called(ctx, folderID) + + var r0 []models.File + if rf, ok := ret.Get(0).(func(context.Context, models.FolderID) []models.File); ok { + r0 = rf(ctx, folderID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]models.File) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, models.FolderID) error); ok { + r1 = rf(ctx, folderID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindByPath provides a mock function with given fields: ctx, path, caseSensitive func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (models.File, error) { ret := _m.Called(ctx, path, caseSensitive) @@ -268,6 +314,29 @@ func (_m *FileReaderWriter) FindByZipFileID(ctx context.Context, zipFileID model return r0, r1 } +// FindMissingInPaths provides a mock function with given fields: ctx, p, missingSinceBefore, limit, offset +func (_m *FileReaderWriter) FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit int, offset int) ([]models.File, error) { + ret := _m.Called(ctx, p, missingSinceBefore, limit, offset) + + var r0 []models.File + if rf, ok := ret.Get(0).(func(context.Context, []string, *time.Time, int, int) []models.File); ok { + r0 = rf(ctx, p, missingSinceBefore, limit, offset) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]models.File) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []string, *time.Time, int, int) error); ok { + r1 = rf(ctx, p, missingSinceBefore, limit, offset) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetCaptions provides a mock function with given fields: ctx, fileID func (_m *FileReaderWriter) GetCaptions(ctx context.Context, fileID models.FileID) ([]*models.VideoCaption, error) { ret := _m.Called(ctx, fileID) @@ -349,6 +418,20 @@ func (_m *FileReaderWriter) Query(ctx context.Context, options models.FileQueryO return r0, r1 } +// SetMissing provides a mock function with given fields: ctx, id, missingSince +func (_m *FileReaderWriter) SetMissing(ctx context.Context, id models.FileID, missingSince *time.Time) error { + ret := _m.Called(ctx, id, missingSince) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.FileID, *time.Time) error); ok { + r0 = rf(ctx, id, missingSince) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Update provides a mock function with given fields: ctx, f func (_m *FileReaderWriter) Update(ctx context.Context, f models.File) error { ret := _m.Called(ctx, f) diff --git a/pkg/models/mocks/FolderReaderWriter.go b/pkg/models/mocks/FolderReaderWriter.go index d2230c645a..8032dcdfed 100644 --- a/pkg/models/mocks/FolderReaderWriter.go +++ b/pkg/models/mocks/FolderReaderWriter.go @@ -7,6 +7,8 @@ import ( models "github.com/stashapp/stash/pkg/models" mock "github.com/stretchr/testify/mock" + + time "time" ) // FolderReaderWriter is an autogenerated mock type for the FolderReaderWriter type @@ -35,6 +37,27 @@ func (_m *FolderReaderWriter) CountAllInPaths(ctx context.Context, p []string) ( return r0, r1 } +// CountMissingInPaths provides a mock function with given fields: ctx, p, missingSinceBefore +func (_m *FolderReaderWriter) CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) { + ret := _m.Called(ctx, p, missingSinceBefore) + + var r0 int + if rf, ok := ret.Get(0).(func(context.Context, []string, *time.Time) int); ok { + r0 = rf(ctx, p, missingSinceBefore) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []string, *time.Time) error); ok { + r1 = rf(ctx, p, missingSinceBefore) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Create provides a mock function with given fields: ctx, f func (_m *FolderReaderWriter) Create(ctx context.Context, f *models.Folder) error { ret := _m.Called(ctx, f) @@ -201,6 +224,29 @@ func (_m *FolderReaderWriter) FindMany(ctx context.Context, id []models.FolderID return r0, r1 } +// FindMissingInPaths provides a mock function with given fields: ctx, p, missingSinceBefore, limit, offset +func (_m *FolderReaderWriter) FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit int, offset int) ([]*models.Folder, error) { + ret := _m.Called(ctx, p, missingSinceBefore, limit, offset) + + var r0 []*models.Folder + if rf, ok := ret.Get(0).(func(context.Context, []string, *time.Time, int, int) []*models.Folder); ok { + r0 = rf(ctx, p, missingSinceBefore, limit, offset) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Folder) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []string, *time.Time, int, int) error); ok { + r1 = rf(ctx, p, missingSinceBefore, limit, offset) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetManyParentFolderIDs provides a mock function with given fields: ctx, folderIDs func (_m *FolderReaderWriter) GetManyParentFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) { ret := _m.Called(ctx, folderIDs) @@ -270,6 +316,20 @@ func (_m *FolderReaderWriter) Query(ctx context.Context, options models.FolderQu return r0, r1 } +// SetMissing provides a mock function with given fields: ctx, id, missingSince +func (_m *FolderReaderWriter) SetMissing(ctx context.Context, id models.FolderID, missingSince *time.Time) error { + ret := _m.Called(ctx, id, missingSince) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.FolderID, *time.Time) error); ok { + r0 = rf(ctx, id, missingSince) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Update provides a mock function with given fields: ctx, f func (_m *FolderReaderWriter) Update(ctx context.Context, f *models.Folder) error { ret := _m.Called(ctx, f) diff --git a/pkg/models/model_file.go b/pkg/models/model_file.go index f6b8bdc517..d37eca6a3a 100644 --- a/pkg/models/model_file.go +++ b/pkg/models/model_file.go @@ -95,7 +95,8 @@ type DirEntry struct { // only guaranteed to have id, path and basename set ZipFile File - ModTime time.Time `json:"mod_time"` + ModTime time.Time `json:"mod_time"` + MissingSince *time.Time `json:"missing_since"` } func (e *DirEntry) info(fs FS, path string) (fs.FileInfo, error) { diff --git a/pkg/models/repository_file.go b/pkg/models/repository_file.go index 83d5c9bc1b..712a4d5fc2 100644 --- a/pkg/models/repository_file.go +++ b/pkg/models/repository_file.go @@ -3,6 +3,7 @@ package models import ( "context" "io/fs" + "time" ) // FileGetter provides methods to get files by ID. @@ -15,8 +16,10 @@ type FileFinder interface { FileGetter FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]File, error) FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]File, error) + FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit, offset int) ([]File, error) FindByPath(ctx context.Context, path string, caseSensitive bool) (File, error) FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error) + FindByFolderID(ctx context.Context, folderID FolderID) ([]File, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]File, error) FindByFileInfo(ctx context.Context, info fs.FileInfo, size int64) ([]File, error) } @@ -29,6 +32,7 @@ type FileQueryer interface { // FileCounter provides methods to count files. type FileCounter interface { CountAllInPaths(ctx context.Context, p []string) (int, error) + CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) CountByFolderID(ctx context.Context, folderID FolderID) (int, error) } @@ -40,6 +44,7 @@ type FileCreator interface { // FileUpdater provides methods to update files. type FileUpdater interface { Update(ctx context.Context, f File) error + SetMissing(ctx context.Context, id FileID, missingSince *time.Time) error } // FileDestroyer provides methods to destroy files. diff --git a/pkg/models/repository_folder.go b/pkg/models/repository_folder.go index 1169e53aca..5206b0839a 100644 --- a/pkg/models/repository_folder.go +++ b/pkg/models/repository_folder.go @@ -1,6 +1,9 @@ package models -import "context" +import ( + "context" + "time" +) // FolderGetter provides methods to get folders by ID. type FolderGetter interface { @@ -12,6 +15,7 @@ type FolderGetter interface { type FolderFinder interface { FolderGetter FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]*Folder, error) + FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit, offset int) ([]*Folder, error) FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error) FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error) @@ -25,6 +29,7 @@ type FolderQueryer interface { type FolderCounter interface { CountAllInPaths(ctx context.Context, p []string) (int, error) + CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) } // FolderCreator provides methods to create folders. @@ -35,6 +40,7 @@ type FolderCreator interface { // FolderUpdater provides methods to update folders. type FolderUpdater interface { Update(ctx context.Context, f *Folder) error + SetMissing(ctx context.Context, id FolderID, missingSince *time.Time) error } type FolderDestroyer interface { diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 7c383dc4ca..026a18c081 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -34,7 +34,7 @@ const ( cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE" ) -var appSchemaVersion uint = 85 +var appSchemaVersion uint = 86 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/file.go b/pkg/sqlite/file.go index d8b1265ed4..e0b43443b4 100644 --- a/pkg/sqlite/file.go +++ b/pkg/sqlite/file.go @@ -33,6 +33,7 @@ type basicFileRow struct { Basename string `db:"basename"` ZipFileID null.Int `db:"zip_file_id"` ParentFolderID models.FolderID `db:"parent_folder_id"` + MissingSince NullTimestamp `db:"missing_since"` Size int64 `db:"size"` ModTime Timestamp `db:"mod_time"` CreatedAt Timestamp `db:"created_at"` @@ -45,6 +46,7 @@ func (r *basicFileRow) fromBasicFile(o models.BaseFile) { r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID) r.ParentFolderID = o.ParentFolderID r.Size = o.Size + r.MissingSince = NullTimestampFromTimePtr(o.MissingSince) r.ModTime = Timestamp{Timestamp: o.ModTime} r.CreatedAt = Timestamp{Timestamp: o.CreatedAt} r.UpdatedAt = Timestamp{Timestamp: o.UpdatedAt} @@ -170,6 +172,7 @@ type fileQueryRow struct { Basename null.String `db:"basename"` ZipFileID null.Int `db:"zip_file_id"` ParentFolderID null.Int `db:"parent_folder_id"` + MissingSince NullTimestamp `db:"missing_since"` Size null.Int `db:"size"` ModTime NullTimestamp `db:"mod_time"` CreatedAt NullTimestamp `db:"file_created_at"` @@ -189,8 +192,9 @@ func (r *fileQueryRow) resolve() models.File { basic := &models.BaseFile{ ID: models.FileID(r.FileID.Int64), DirEntry: models.DirEntry{ - ZipFileID: nullIntFileIDPtr(r.ZipFileID), - ModTime: r.ModTime.Timestamp, + ZipFileID: nullIntFileIDPtr(r.ZipFileID), + ModTime: r.ModTime.Timestamp, + MissingSince: r.MissingSince.TimePtr(), }, Path: filepath.Join(r.FolderPath.String, r.Basename.String), ParentFolderID: models.FolderID(r.ParentFolderID.Int64), @@ -399,6 +403,29 @@ func (qb *FileStore) Update(ctx context.Context, f models.File) error { return nil } +func (qb *FileStore) SetMissing(ctx context.Context, id models.FileID, missingSince *time.Time) error { + if err := qb.tableMgr.checkIDExists(ctx, int(id)); err != nil { + return err + } + + table := qb.tableMgr.table + + var timestampValue NullTimestamp + if missingSince != nil { + timestampValue = NullTimestamp{Timestamp: *missingSince, Valid: true} + } + + q := dialect.Update(table).Set(goqu.Record{ + "missing_since": timestampValue, + }).Where(qb.tableMgr.byID(id)) + + if _, err := exec(ctx, q); err != nil { + return fmt.Errorf("updating %s: %w", table.GetTable(), err) + } + + return nil +} + // ModifyFingerprints updates existing fingerprints and adds new ones. func (qb *FileStore) ModifyFingerprints(ctx context.Context, fileID models.FileID, fingerprints []models.Fingerprint) error { return FingerprintReaderWriter.upsertJoins(ctx, fileID, fingerprints) @@ -492,6 +519,7 @@ func (qb *FileStore) selectDataset() *goqu.SelectDataset { table.Col("parent_folder_id"), table.Col("size"), table.Col("mod_time"), + table.Col("missing_since"), table.Col("created_at").As("file_created_at"), table.Col("updated_at").As("file_updated_at"), folderTable.Col("path").As("parent_folder_path"), @@ -733,6 +761,52 @@ func (qb *FileStore) CountAllInPaths(ctx context.Context, p []string) (int, erro return count(ctx, q) } +func (qb *FileStore) FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit, offset int) ([]models.File, error) { + table := qb.table() + folderTable := folderTableMgr.table + + q := dialect.From(table).Prepared(true).InnerJoin( + folderTable, + goqu.On(table.Col("parent_folder_id").Eq(folderTable.Col(idColumn))), + ).Select(table.Col(idColumn)) + + q = qb.allInPaths(q, p) + + if missingSinceBefore != nil { + v := missingSinceBefore.Format(time.RFC3339) + q = q.Where(qb.table().Col("missing_since").Lt(v)) + } else { + q = q.Where(qb.table().Col("missing_since").IsNotNull()) + } + + if limit > -1 { + q = q.Limit(uint(limit)) + } + + q = q.Offset(uint(offset)) + + ret, err := qb.findBySubquery(ctx, q) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("getting files by path %s: %w", p, err) + } + + return ret, nil +} + +func (qb *FileStore) CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) { + q := qb.countDataset().Prepared(true) + q = qb.allInPaths(q, p) + + if missingSinceBefore != nil { + v := missingSinceBefore.Format(time.RFC3339) + q = q.Where(qb.table().Col("missing_since").Lt(v)) + } else { + q = q.Where(qb.table().Col("missing_since").IsNotNull()) + } + + return count(ctx, q) +} + func (qb *FileStore) findBySubquery(ctx context.Context, sq *goqu.SelectDataset) ([]models.File, error) { table := qb.table() @@ -758,6 +832,16 @@ func (qb *FileStore) FindByFingerprint(ctx context.Context, fp models.Fingerprin return qb.findBySubquery(ctx, sq) } +func (qb *FileStore) FindByFolderID(ctx context.Context, folderID models.FolderID) ([]models.File, error) { + table := qb.table() + + q := qb.selectDataset().Prepared(true).Where( + table.Col("parent_folder_id").Eq(folderID), + ) + + return qb.getMany(ctx, q) +} + func (qb *FileStore) FindByZipFileID(ctx context.Context, zipFileID models.FileID) ([]models.File, error) { table := qb.table() @@ -979,6 +1063,7 @@ var fileSortOptions = sortOptions{ "id", "path", "random", + "missing_since", "updated_at", } diff --git a/pkg/sqlite/file_filter.go b/pkg/sqlite/file_filter.go index 6c1005c8d4..a94def5cfa 100644 --- a/pkg/sqlite/file_filter.go +++ b/pkg/sqlite/file_filter.go @@ -83,6 +83,7 @@ func (qb *fileFilterHandler) criterionHandler() criterionHandler { qb.hashesCriterionHandler(fileFilter.Hashes), qb.duplicatedCriterionHandler(fileFilter.Duplicated), + ×tampCriterionHandler{fileFilter.MissingSince, "files.missing_since", nil}, ×tampCriterionHandler{fileFilter.CreatedAt, "files.created_at", nil}, ×tampCriterionHandler{fileFilter.UpdatedAt, "files.updated_at", nil}, diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index f12789c383..a986f6a683 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -7,6 +7,7 @@ import ( "fmt" "path/filepath" "slices" + "time" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -24,6 +25,7 @@ type folderRow struct { Path string `db:"path"` ZipFileID null.Int `db:"zip_file_id"` ParentFolderID null.Int `db:"parent_folder_id"` + MissingSince NullTimestamp `db:"missing_since"` ModTime Timestamp `db:"mod_time"` CreatedAt Timestamp `db:"created_at"` UpdatedAt Timestamp `db:"updated_at"` @@ -36,6 +38,7 @@ func (r *folderRow) fromFolder(o models.Folder) { r.Path = o.Path r.ZipFileID = nullIntFromFileIDPtr(o.ZipFileID) r.ParentFolderID = nullIntFromFolderIDPtr(o.ParentFolderID) + r.MissingSince = NullTimestampFromTimePtr(o.MissingSince) r.ModTime = Timestamp{Timestamp: o.ModTime} r.CreatedAt = Timestamp{Timestamp: o.CreatedAt} r.UpdatedAt = Timestamp{Timestamp: o.UpdatedAt} @@ -53,8 +56,9 @@ func (r *folderQueryRow) resolve() *models.Folder { ret := &models.Folder{ ID: r.ID, DirEntry: models.DirEntry{ - ZipFileID: nullIntFileIDPtr(r.ZipFileID), - ModTime: r.ModTime.Timestamp, + ZipFileID: nullIntFileIDPtr(r.ZipFileID), + ModTime: r.ModTime.Timestamp, + MissingSince: r.MissingSince.TimePtr(), }, Path: string(r.Path), ParentFolderID: nullIntFolderIDPtr(r.ParentFolderID), @@ -149,6 +153,29 @@ func (qb *FolderStore) Update(ctx context.Context, updatedObject *models.Folder) return nil } +func (qb *FolderStore) SetMissing(ctx context.Context, id models.FolderID, missingSince *time.Time) error { + if err := qb.tableMgr.checkIDExists(ctx, int(id)); err != nil { + return err + } + + table := qb.tableMgr.table + + var timestampValue NullTimestamp + if missingSince != nil { + timestampValue = NullTimestamp{Timestamp: *missingSince, Valid: true} + } + + q := dialect.Update(table).Set(goqu.Record{ + "missing_since": timestampValue, + }).Where(qb.tableMgr.byID(id)) + + if _, err := exec(ctx, q); err != nil { + return fmt.Errorf("updating %s: %w", table.GetTable(), err) + } + + return nil +} + func (qb *FolderStore) Destroy(ctx context.Context, id models.FolderID) error { return qb.tableMgr.destroyExisting(ctx, []int{int(id)}) } @@ -169,6 +196,7 @@ func (qb *FolderStore) selectDataset() *goqu.SelectDataset { table.Col("path"), table.Col("zip_file_id"), table.Col("parent_folder_id"), + table.Col("missing_since"), table.Col("mod_time"), table.Col("created_at"), table.Col("updated_at"), @@ -493,6 +521,45 @@ func (qb *FolderStore) CountAllInPaths(ctx context.Context, p []string) (int, er return count(ctx, q) } +func (qb *FolderStore) FindMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time, limit, offset int) ([]*models.Folder, error) { + q := qb.selectDataset().Prepared(true) + q = qb.allInPaths(q, p) + + if missingSinceBefore != nil { + v := missingSinceBefore.Format(time.RFC3339) + q = q.Where(qb.table().Col("missing_since").Lt(v)) + } else { + q = q.Where(qb.table().Col("missing_since").IsNotNull()) + } + + if limit > -1 { + q = q.Limit(uint(limit)) + } + + q = q.Offset(uint(offset)) + + ret, err := qb.getMany(ctx, q) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("getting folders in path %s: %w", p, err) + } + + return ret, nil +} + +func (qb *FolderStore) CountMissingInPaths(ctx context.Context, p []string, missingSinceBefore *time.Time) (int, error) { + q := qb.countDataset().Prepared(true) + q = qb.allInPaths(q, p) + + if missingSinceBefore != nil { + v := missingSinceBefore.Format(time.RFC3339) + q = q.Where(qb.table().Col("missing_since").Lt(v)) + } else { + q = q.Where(qb.table().Col("missing_since").IsNotNull()) + } + + return count(ctx, q) +} + // func (qb *FolderStore) findBySubquery(ctx context.Context, sq *goqu.SelectDataset) ([]*file.Folder, error) { // table := qb.table() @@ -638,6 +705,7 @@ var folderSortOptions = sortOptions{ "path", "basename", "random", + "missing_since", "updated_at", } diff --git a/pkg/sqlite/folder_filter.go b/pkg/sqlite/folder_filter.go index 9b99453cf8..e0855e9c29 100644 --- a/pkg/sqlite/folder_filter.go +++ b/pkg/sqlite/folder_filter.go @@ -73,6 +73,7 @@ func (qb *folderFilterHandler) criterionHandler() criterionHandler { qb.galleryCountCriterionHandler(folderFilter.GalleryCount), + ×tampCriterionHandler{folderFilter.MissingSince, qb.table.Col("missing_since"), nil}, ×tampCriterionHandler{folderFilter.CreatedAt, qb.table.Col("created_at"), nil}, ×tampCriterionHandler{folderFilter.UpdatedAt, qb.table.Col("updated_at"), nil}, diff --git a/pkg/sqlite/migrations/86_file_missing_since.up.sql b/pkg/sqlite/migrations/86_file_missing_since.up.sql new file mode 100644 index 0000000000..511972aa29 --- /dev/null +++ b/pkg/sqlite/migrations/86_file_missing_since.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE `files` ADD COLUMN `missing_since` datetime; +CREATE INDEX `files_missing_since_index` ON `files` (`missing_since`) WHERE `missing_since` IS NOT NULL; + +ALTER TABLE `folders` ADD COLUMN `missing_since` datetime; +CREATE INDEX `folders_missing_since_index` ON `folders` (`missing_since`) WHERE `missing_since` IS NOT NULL; \ No newline at end of file diff --git a/ui/v2.5/graphql/mutations/metadata.graphql b/ui/v2.5/graphql/mutations/metadata.graphql index eb89de0d7b..4fffea3734 100644 --- a/ui/v2.5/graphql/mutations/metadata.graphql +++ b/ui/v2.5/graphql/mutations/metadata.graphql @@ -34,6 +34,14 @@ mutation MetadataClean($input: CleanMetadataInput!) { metadataClean(input: $input) } +mutation VerifyPaths($input: VerifyPathsInput!) { + verifyPaths(input: $input) +} + +mutation PurgeMissing($input: PurgeMissingInput!) { + purgeMissing(input: $input) +} + mutation MetadataCleanGenerated($input: CleanGeneratedInput!) { metadataCleanGenerated(input: $input) } diff --git a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx index 23bfe5b162..e9b240f027 100644 --- a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Button, Col, Form, Row } from "react-bootstrap"; import { @@ -6,12 +6,13 @@ import { mutateMetadataExport, mutateBackupDatabase, mutateMetadataImport, - mutateMetadataClean, + mutateVerifyPaths, mutateAnonymiseDatabase, mutateMigrateSceneScreenshots, mutateMigrateBlobs, mutateOptimiseDatabase, mutateCleanGenerated, + mutatePurgeMissing, } from "src/core/StashService"; import { useToast } from "src/hooks/Toast"; import downloadFile from "src/utils/download"; @@ -33,16 +34,19 @@ import { faTrashAlt, } from "@fortawesome/free-solid-svg-icons"; import { CleanGeneratedDialog } from "./CleanGeneratedDialog"; +import { useSettings } from "../context"; -interface ICleanDialog { +interface IVerifyDialog { pathSelection?: boolean; dryRun: boolean; + purgeMissing: boolean; onClose: (paths?: string[]) => void; } -const CleanDialog: React.FC = ({ +const VerifyDialog: React.FC = ({ pathSelection = false, dryRun, + purgeMissing, onClose, }) => { const intl = useIntl(); @@ -68,19 +72,24 @@ const CleanDialog: React.FC = ({ msg = (

{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}

); - } else { + } else if (purgeMissing) { msg = ( -

{intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}

+

+ {intl.formatMessage({ + id: "actions.tasks.verify_purge_confirm_message", + })} +

); } return ( } icon={faTrashAlt} disabled={pathSelection && paths.length === 0} accept={{ - text: intl.formatMessage({ id: "actions.clean" }), + text: intl.formatMessage({ id: "actions.verify_files" }), variant: "danger", onClick: () => onClose(paths), }} @@ -130,38 +139,169 @@ const CleanDialog: React.FC = ({ ); }; -interface ICleanOptions { - options: GQL.CleanMetadataInput; - setOptions: (s: GQL.CleanMetadataInput) => void; +interface IVerifyOptions { + options: GQL.VerifyPathsInput; + setOptions: (s: GQL.VerifyPathsInput) => void; } -const CleanOptions: React.FC = ({ +const VerifyOptions: React.FC = ({ options, setOptions: setOptionsState, }) => { - function setOptions(input: Partial) { + function setOptions(input: Partial) { setOptionsState({ ...options, ...input }); } return ( <> setOptions({ ignoreZipFileContents: v })} + id="verify-ignore-zip-contents" + checked={options.checkZipFileContents ?? false} + headingID="config.tasks.verify_check_zip_contents" + subHeadingID="config.tasks.verify_check_zip_contents_desc" + onChange={(v) => setOptions({ checkZipFileContents: v })} /> setOptions({ dryRun: v })} /> + setOptions({ purgeMissing: v })} + /> ); }; +interface IPurgeMissingDialog { + pathSelection?: boolean; + dryRun: boolean; + onClose: (paths?: string[]) => void; +} + +const PurgeMissingDialog: React.FC = ({ + pathSelection = false, + dryRun, + onClose, +}) => { + const intl = useIntl(); + const { configuration } = useConfigurationContext(); + + const libraryPaths = configuration?.general.stashes.map((s) => s.path); + + const [paths, setPaths] = useState([]); + const [currentDirectory, setCurrentDirectory] = useState(""); + + function removePath(p: string) { + setPaths(paths.filter((path) => path !== p)); + } + + function addPath(p: string) { + if (p && !paths.includes(p)) { + setPaths(paths.concat(p)); + } + } + + let msg: React.ReactNode; + if (dryRun) { + msg = ( +

{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}

+ ); + } else { + msg = ( +

+ {intl.formatMessage({ + id: "actions.tasks.purge_missing_confirm_message", + })} +

+ ); + } + + return ( + } + disabled={pathSelection && paths.length === 0} + accept={{ + text: intl.formatMessage({ id: "config.tasks.purge_missing" }), + variant: "danger", + onClick: () => onClose(paths), + }} + cancel={{ onClick: () => onClose() }} + > +
+
+ {paths.map((p) => ( + + + {p} + + + + + + ))} + + {pathSelection ? ( + addPath(currentDirectory)} + > + + + } + /> + ) : undefined} +
+ + {msg} +
+ + ); +}; + +interface IPurgeMissingOptions { + options: GQL.PurgeMissingInput; + setOptions: (s: GQL.PurgeMissingInput) => void; +} + +const PurgeMissingOptions: React.FC = ({ + options, + setOptions: setOptionsState, +}) => { + function setOptions(input: Partial) { + setOptionsState({ ...options, ...input }); + } + + return ( + setOptions({ dryRun: v })} + /> + ); +}; + const BackupDialog: React.FC<{ onClose: ( confirmed?: boolean, @@ -296,15 +436,24 @@ export const DataManagementTasks: React.FC = ({ importAlert: false, import: false, backup: false, - clean: false, - cleanAlert: false, + verify: false, + verifyAlert: false, + purgeMissing: false, + purgeMissingAlert: false, cleanGenerated: false, }); - const [cleanOptions, setCleanOptions] = useState({ + const [verifyOptions, setVerifyOptions] = useState({ dryRun: false, + checkZipFileContents: false, + purgeMissing: false, }); + const [purgeMissingOptions, setPurgeMissingOptions] = + useState({ + dryRun: false, + }); + const [migrateBlobsOptions, setMigrateBlobsOptions] = useState({ deleteOld: true, @@ -316,6 +465,37 @@ export const DataManagementTasks: React.FC = ({ overwriteExisting: false, }); + const { ui, saveUI, loading } = useSettings(); + + const { taskDefaults } = ui; + + useEffect(() => { + if (loading) { + return; + } + + if (taskDefaults?.verify) { + setVerifyOptions(taskDefaults.verify); + } + if (taskDefaults?.purgeMissing) { + setPurgeMissingOptions(taskDefaults.purgeMissing); + } + }, [taskDefaults, loading]); + + function configureDefaults(partial: Record) { + saveUI({ taskDefaults: { ...partial } }); + } + + function onSetVerifyOptions(s: GQL.VerifyPathsInput) { + configureDefaults({ verify: s }); + setVerifyOptions(s); + } + + function onSetPurgeMissingOptions(s: GQL.PurgeMissingInput) { + configureDefaults({ purgeMissing: s }); + setPurgeMissingOptions(s); + } + type DialogOpenState = typeof dialogOpen; function setDialogOpen(s: Partial) { @@ -364,24 +544,66 @@ export const DataManagementTasks: React.FC = ({ return setDialogOpen({ import: false })} />; } - async function onClean(paths?: string[]) { + async function onVerify(paths?: string[]) { try { - await mutateMetadataClean({ - ...cleanOptions, + await mutateVerifyPaths({ + ...verifyOptions, paths, }); Toast.success( intl.formatMessage( { id: "config.tasks.added_job_to_queue" }, - { operation_name: intl.formatMessage({ id: "actions.clean" }) } + { operation_name: intl.formatMessage({ id: "actions.verify_files" }) } ) ); } catch (e) { Toast.error(e); } finally { - setDialogOpen({ clean: false }); + setDialogOpen({ verify: false }); + } + } + + function onVerifyClicked() { + if (verifyOptions.dryRun || !verifyOptions.purgeMissing) { + onVerify(); + return; } + + setDialogOpen({ verifyAlert: true }); + } + + async function onPurgeMissing(paths?: string[]) { + try { + await mutatePurgeMissing({ + ...purgeMissingOptions, + paths, + }); + + Toast.success( + intl.formatMessage( + { id: "config.tasks.added_job_to_queue" }, + { + operation_name: intl.formatMessage({ + id: "config.tasks.purge_missing", + }), + } + ) + ); + } catch (e) { + Toast.error(e); + } finally { + setDialogOpen({ purgeMissing: false }); + } + } + + function onPurgeMissingClicked() { + if (purgeMissingOptions.dryRun) { + onPurgeMissing(); + return; + } + + setDialogOpen({ purgeMissingAlert: true }); } async function onCleanGenerated(options: GQL.CleanGeneratedInput) { @@ -534,30 +756,51 @@ export const DataManagementTasks: React.FC = ({ {renderImportAlert()} {renderImportDialog()} - {dialogOpen.cleanAlert || dialogOpen.clean ? ( - { // undefined means cancelled if (p !== undefined) { - if (dialogOpen.cleanAlert) { + if (dialogOpen.verifyAlert) { // don't provide paths - onClean(); + onVerify(); } else { - onClean(p); + onVerify(p); } } setDialogOpen({ - clean: false, - cleanAlert: false, + verify: false, + verifyAlert: false, }); }} /> - ) : ( - dialogOpen.clean - )} + ) : null} + {dialogOpen.purgeMissingAlert || dialogOpen.purgeMissing ? ( + { + // undefined means cancelled + if (p !== undefined) { + if (dialogOpen.purgeMissingAlert) { + // don't provide paths + onPurgeMissing(); + } else { + onPurgeMissing(p); + } + } + + setDialogOpen({ + purgeMissing: false, + purgeMissingAlert: false, + }); + }} + /> + ) : null} {dialogOpen.cleanGenerated && ( { @@ -586,32 +829,65 @@ export const DataManagementTasks: React.FC = ({ - + + + + + + } + subHeadingID="config.tasks.verify_files_desc" + > + + + + onSetVerifyOptions(o)} + /> + + +
+ + } - subHeadingID="config.tasks.cleanup_desc" + subHeadingID="config.tasks.purge_missing_desc" > - setCleanOptions(o)} + onSetPurgeMissingOptions(o)} />
diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 319ccbc440..6e3bb5c29f 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -2939,9 +2939,15 @@ export const mutateMetadataGenerate = (input: GQL.GenerateMetadataInput) => variables: { input }, }); -export const mutateMetadataClean = (input: GQL.CleanMetadataInput) => - client.mutate({ - mutation: GQL.MetadataCleanDocument, +export const mutateVerifyPaths = (input: GQL.VerifyPathsInput) => + client.mutate({ + mutation: GQL.VerifyPathsDocument, + variables: { input }, + }); + +export const mutatePurgeMissing = (input: GQL.PurgeMissingInput) => + client.mutate({ + mutation: GQL.PurgeMissingDocument, variables: { input }, }); diff --git a/ui/v2.5/src/docs/en/Manual/Tasks.md b/ui/v2.5/src/docs/en/Manual/Tasks.md index 9075dc4189..a92b53e844 100644 --- a/ui/v2.5/src/docs/en/Manual/Tasks.md +++ b/ui/v2.5/src/docs/en/Manual/Tasks.md @@ -109,11 +109,15 @@ Stash has since implemented live transcoding, so transcodes are essentially unne These are generated when the gallery is first viewed, so generating them beforehand is not necessary. -## Cleaning +## Verify files -This task will walk through your configured media directories and remove any scene from the database that can no longer be found. It will also remove generated files for scenes that subsequently no longer exist. +This task will walk through your configured media directories and mark files and folders in the database that can no longer be found in the filesystem. It can optionally remove these missing files and their associated metadata objects (e.g., scenes, images, and galleries) and generated files from the database. -Care should be taken with this task, especially where the configured media directories may be inaccessible due to network issues. +Files marked as missing may be removed by the Purge missing task. Files and folders can have their missing status cleared by running the scan task, assuming the scan task finds the file or folder in the filesystem. + +## Purge missing + +This task will remove files and folders marked as missing from the database, along with their associated metadata objects (e.g., scenes, images, and galleries) and generated files. ## Exporting and importing @@ -137,4 +141,4 @@ For database-only backups, only the database file is copied into the destination Restoring from backup is currently a manual process. The database backup zip file must be unzipped, and the database file and blob files (if applicable) copied into the database and blob directories respectively. Stash should then be restarted to load the restored database. -> **⚠️ Note:** the filename for a database-only backup is not the same as the original database file, so the database file from the backup must be renamed to match the original database filename before copying it into the database directory. The original database filename can be found in `Settings > Paths > Database path`. \ No newline at end of file +> **⚠️ Note:** the filename for a database-only backup is not the same as the original database file, so the database file from the backup must be renamed to match the original database filename before copying it into the database directory. The original database filename can be found in `Settings > Paths > Database path`. diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 6dfb08b835..e28a7a63fb 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -19,7 +19,6 @@ "browse_for_image": "Browse for image…", "cancel": "Cancel", "choose_date": "Choose a date", - "clean": "Clean", "clean_generated": "Clean generated files", "clear": "Clear", "clear_back_image": "Clear back image", @@ -125,9 +124,9 @@ "select_none": "Select None", "invert_selection": "Invert Selection", "selective_auto_tag": "Selective auto tag", - "selective_clean": "Selective clean", "selective_generate": "Selective generate", "selective_scan": "Selective scan", + "selective_verify": "Selective verify", "set_as_default": "Set as default", "set_as_performer_image": "Set as performer image", "set_back_image": "Back image…", @@ -151,14 +150,16 @@ "submit_update": "Submit update", "swap": "Swap", "tasks": { - "clean_confirm_message": "Are you sure you want to Clean? This will delete database information and generated content for all scenes and galleries that are no longer found in the filesystem.", - "dry_mode_selected": "Dry Mode selected. No actual deleting will take place, only logging.", - "import_warning": "Are you sure you want to import? This will delete the database and re-import from your exported metadata." + "dry_mode_selected": "Dry Mode selected. No changes to the database will be made, only logging.", + "import_warning": "Are you sure you want to import? This will delete the database and re-import from your exported metadata.", + "purge_missing_confirm_message": "Are you sure you want to purge files and folders marked missing? This will delete database information and generated content for all files and folders that are marked as missing in the database.", + "verify_purge_confirm_message": "Are you sure you want to purge files and folders found missing? This will delete database information and generated content for all files and folders that are no longer found in the filesystem." }, "temp_disable": "Disable temporarily…", "temp_enable": "Enable temporarily…", "unset": "Unset", "use_default": "Use default", + "verify_files": "Verify files", "view_history": "View history", "view_random": "View Random" }, @@ -554,6 +555,7 @@ "data_management": "Data management", "defaults_set": "Defaults have been set and will be used when clicking the {action} button on the Tasks page.", "dont_include_file_extension_as_part_of_the_title": "Don't include file extension as part of the title", + "dry_run": "Dry run only. Don't make any changes to the database.", "empty_queue": "No tasks are currently running.", "export_to_json": "Exports the database content into JSON format in the metadata directory.", "generate": { @@ -624,6 +626,8 @@ "optimise_database": "Attempt to improve performance by analysing and then rebuilding the entire database file.", "optimise_database_warning": "Warning: while this task is running, any operations that modify the database will fail, and depending on your database size, it could take several minutes to complete. It also requires at the very minimum as much free disk space as your database is large, but 1.5x is recommended.", "plugin_tasks": "Plugin Tasks", + "purge_missing": "Purge missing", + "purge_missing_desc": "Permanently remove files and folders marked missing from the database. Removes associated objects and generated files. This is a destructive action.", "rescan": "Rescan files", "rescan_tooltip": "Rescan every file in the path. Used to force update file metadata and rescan zip files.", "scan": { @@ -631,7 +635,12 @@ "scanning_paths": "Scanning the following paths" }, "scan_for_content_desc": "Scan for new content and add it to the database.", - "set_name_date_details_from_metadata_if_present": "Set name, date, details from embedded file metadata" + "selective_purge_missing": "Selective purge", + "set_name_date_details_from_metadata_if_present": "Set name, date, details from embedded file metadata", + "verify_check_zip_contents": "Verify zip file contents", + "verify_check_zip_contents_desc": "Include files inside zip files when verifying. Only required if files within zip files have been deleted in place.", + "verify_files_desc": "Verify files in the database against file system, marking any missing files.", + "verify_purge_missing": "Permanently remove missing files from the database." }, "tools": { "graphql_playground": "GraphQL playground",