Add missing_since file/folder field and add verify and purge missing tasks

graphql/schema/schema.graphql

@@ -482,7 +482,11 @@ type Mutation {
   "Start auto-tagging. Returns the job ID"
   metadataAutoTag(input: AutoTagMetadataInput!): ID!
   "Clean metadata. Returns the job ID"
-  metadataClean(input: CleanMetadataInput!): ID!
+  metadataClean(input: CleanMetadataInput!): ID! @deprecated(reason: "Use verifyPaths and purgeMissing instead")
+  "Verifies the existence of files and folders in library, marking any missing items. Returns the job ID"
+  verifyPaths(input: VerifyPathsInput!): ID!
+  "Cleans up any missing files and folders from the library. Returns the job ID"
+  purgeMissing(input: PurgeMissingInput!): ID!
   "Clean generated files. Returns the job ID"
   metadataCleanGenerated(input: CleanGeneratedInput!): ID!
   "Identifies scenes using scrapers. Returns the job ID"

graphql/schema/types/file.graphql

@@ -19,6 +19,9 @@ type Folder {
   "Returns direct sub-folders"
   sub_folders: [Folder!]!
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
 
   created_at: Time!
@@ -36,6 +39,9 @@ interface BaseFile {
   parent_folder: Folder!
   zip_file: BasicFile
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
   size: Int64!
 
@@ -57,6 +63,9 @@ type BasicFile implements BaseFile {
   parent_folder: Folder!
   zip_file: BasicFile
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
   size: Int64!
 
@@ -78,6 +87,9 @@ type VideoFile implements BaseFile {
   parent_folder: Folder!
   zip_file: BasicFile
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
   size: Int64!
 
@@ -110,6 +122,9 @@ type ImageFile implements BaseFile {
   parent_folder: Folder!
   zip_file: BasicFile
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
   size: Int64!
 
@@ -139,6 +154,9 @@ type GalleryFile implements BaseFile {
   parent_folder: Folder!
   zip_file: BasicFile
 
+  "If not null, indicates when the folder was detected as missing. Set by the clean task"
+  missing_since: Time
+
   mod_time: Time!
   size: Int64!
 

graphql/schema/types/filters.graphql

@@ -789,6 +789,9 @@ input FileFilterType {
   parent_folder: HierarchicalMultiCriterionInput
   zip_file: MultiCriterionInput
 
+  "Filter by missing since time"
+  missing_since: TimestampCriterionInput
+
   "Filter by modification time"
   mod_time: TimestampCriterionInput
 
@@ -829,6 +832,9 @@ input FolderFilterType {
   parent_folder: HierarchicalMultiCriterionInput
   zip_file: MultiCriterionInput
 
+  "Filter by missing since time"
+  missing_since: TimestampCriterionInput
+
   "Filter by modification time"
   mod_time: TimestampCriterionInput
 

graphql/schema/types/metadata.graphql

@@ -128,6 +128,34 @@ type ScanMetadataOptions {
   scanGenerateClipPreviews: Boolean!
 }
 
+input VerifyPathsInput {
+  paths: [String!]
+
+  """
+  Check zip file contents when checking the existence of files.
+  This can significantly slow down the process, but will ensure removed files within zip files are detected.
+  Only necessary where users modify zip files contents.
+  Defaults to false.
+  """
+  checkZipFileContents: Boolean
+
+  "If true, only logs missing files to the log file and does not make any changes"
+  dryRun: Boolean
+
+  "If true, removes any missing files from the database"
+  purgeMissing: Boolean
+}
+
+input PurgeMissingInput {
+  paths: [String!]
+
+  "If provided, only purge files that have been missing since before this time"
+  missingSinceBefore: Timestamp
+
+  "If true, only logs actions that will be taken to the log file and does not make any changes"
+  dryRun: Boolean
+}
+
 input CleanMetadataInput {
   paths: [String!]
 

internal/api/resolver_mutation_metadata.go

@@ -11,7 +11,9 @@ import (
 	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/internal/manager/task"
+	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/utils"
 )
 
 func (r *mutationResolver) MetadataScan(ctx context.Context, input manager.ScanMetadataInput) (string, error) {
@@ -99,6 +101,33 @@ func (r *mutationResolver) MetadataClean(ctx context.Context, input manager.Clea
 	return strconv.Itoa(jobID), nil
 }
 
+func (r *mutationResolver) VerifyPaths(ctx context.Context, input VerifyPathsInput) (string, error) {
+	dryRun := utils.IsTrue(input.DryRun)
+
+	options := file.VerifyOptions{
+		Paths:                 input.Paths,
+		IgnoreZipFileContents: !utils.IsTrue(input.CheckZipFileContents),
+		DryRun:                dryRun,
+		PurgeMissing:          !dryRun && utils.IsTrue(input.PurgeMissing),
+	}
+
+	jobID := manager.GetInstance().VerifyPaths(ctx, options)
+	return strconv.Itoa(jobID), nil
+}
+
+func (r *mutationResolver) PurgeMissing(ctx context.Context, input PurgeMissingInput) (string, error) {
+	dryRun := utils.IsTrue(input.DryRun)
+
+	options := file.PurgeMissingOptions{
+		Paths:              input.Paths,
+		DryRun:             dryRun,
+		MissingSinceBefore: input.MissingSinceBefore,
+	}
+
+	jobID := manager.GetInstance().PurgeMissing(ctx, options)
+	return strconv.Itoa(jobID), nil
+}
+
 func (r *mutationResolver) MetadataCleanGenerated(ctx context.Context, input task.CleanGeneratedOptions) (string, error) {
 	mgr := manager.GetInstance()
 	t := &task.CleanGeneratedJob{

internal/manager/manager_tasks.go

@@ -319,25 +319,56 @@ type CleanMetadataInput struct {
 }
 
 func (s *Manager) Clean(ctx context.Context, input CleanMetadataInput) int {
-	cleaner := &file.Cleaner{
+	// deprecated - run verify paths task with purge missing turned on
+	return s.VerifyPaths(ctx, file.VerifyOptions{
+		Paths:                 input.Paths,
+		IgnoreZipFileContents: input.IgnoreZipFileContents,
+		DryRun:                input.DryRun,
+		PurgeMissing:          !input.DryRun,
+	})
+}
+
+func (s *Manager) VerifyPaths(ctx context.Context, options file.VerifyOptions) int {
+	verifier := &file.Verifier{
 		FS:         &file.OsFS{},
 		Repository: file.NewRepository(s.Repository),
-		Handlers: []file.CleanHandler{
-			&cleanHandler{},
+		PurgeHandlers: []file.PurgeHandler{
+			&purgeHandler{},
+		},
+		TrashPath: s.Config.GetDeleteTrashPath(),
+	}
+
+	j := verifyJob{
+		verifier:     verifier,
+		repository:   s.Repository,
+		sceneService: s.SceneService,
+		imageService: s.ImageService,
+		options:      options,
+		scanSubs:     s.scanSubs,
+	}
+
+	return s.JobManager.Add(ctx, "Verifying paths...", &j)
+}
+
+func (s *Manager) PurgeMissing(ctx context.Context, options file.PurgeMissingOptions) int {
+	purger := &file.MissingPurger{
+		Repository: file.NewRepository(s.Repository),
+		PurgeHandlers: []file.PurgeHandler{
+			&purgeHandler{},
 		},
 		TrashPath: s.Config.GetDeleteTrashPath(),
 	}
 
-	j := cleanJob{
-		cleaner:      cleaner,
+	j := purgeMissingJob{
+		purger:       purger,
 		repository:   s.Repository,
 		sceneService: s.SceneService,
 		imageService: s.ImageService,
-		input:        input,
+		options:      options,
 		scanSubs:     s.scanSubs,
 	}
 
-	return s.JobManager.Add(ctx, "Cleaning...", &j)
+	return s.JobManager.Add(ctx, "Purging missing files and folders...", &j)
 }
 
 func (s *Manager) OptimiseDatabase(ctx context.Context) int {