diff --git a/pkg/sqlite/driver.go b/pkg/sqlite/driver.go index 01a86d253a..89f2c609fa 100644 --- a/pkg/sqlite/driver.go +++ b/pkg/sqlite/driver.go @@ -30,6 +30,7 @@ func (d *CustomSQLiteDriver) Open(dsn string) (driver.Conn, error) { "durationToTinyInt": durationToTinyIntFn, "basename": basenameFn, "phash_distance": phashDistanceFn, + "lower_unicode": lowerUnicodeFn, } for name, fn := range funcs { diff --git a/pkg/sqlite/functions.go b/pkg/sqlite/functions.go index f639d0b5b9..03aa4afb26 100644 --- a/pkg/sqlite/functions.go +++ b/pkg/sqlite/functions.go @@ -1,6 +1,7 @@ package sqlite import ( + "fmt" "path/filepath" "strconv" "strings" @@ -35,3 +36,27 @@ func durationToTinyIntFn(str string) (int64, error) { func basenameFn(str string) (string, error) { return filepath.Base(str), nil } + +// custom SQLite function to enable case-insensitive searches +// that properly handle unicode characters +func lowerUnicodeFn(str interface{}) (string, error) { + // handle NULL values + if str == nil { + return "", nil + } + + // handle different types + switch v := str.(type) { + case string: + return strings.ToLower(v), nil + case int64: + // convert int64 to string (for phash fingerprints) + return strings.ToLower(strconv.FormatInt(v, 10)), nil + case []byte: + // handle BLOB type if needed + return strings.ToLower(string(v)), nil + default: + // for any other type, try converting to string + return strings.ToLower(fmt.Sprintf("%v", v)), nil + } +} diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index a6add7f7be..18b0132bdf 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -2698,6 +2698,137 @@ func TestPerformerStore_FindByStashIDStatus(t *testing.T) { } } +func TestPerformerQueryUnicodeSearchCaseInsensitive(t *testing.T) { + withTxn(func(ctx context.Context) error { + qb := db.Performer + + // test cases with various Unicode characters + testCases := []struct { + name string + performerName string + searchTerm string + }{ + { + "Cyrillic lowercase search", + "Анна", + "анна", + }, + { + "Cyrillic uppercase search", + "мария", + "МАРИЯ", + }, + { + "Accented Latin lowercase", + "Zoë", + "zoë", + }, + { + "Accented Latin uppercase", + "chloé", + "CHLOÉ", + }, + { + "Greek lowercase search", + "Έλενα", + "έλενα", + }, + { + "Pure ASCII term keeps working", + "John SMITH", + "smith", + }, + { + "Mixed ASCII and Cyrillic terms in one query", + "ANNA МАРИЯ", + "anna мария", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // create performer with unicode name + performer := models.Performer{ + Name: tc.performerName, + } + err := qb.Create(ctx, &models.CreatePerformerInput{Performer: &performer}) + if err != nil { + t.Fatalf("Error creating performer: %s", err.Error()) + } + + // search using different case + findFilter := &models.FindFilterType{ + Q: &tc.searchTerm, + } + + performers, _, err := qb.Query(ctx, nil, findFilter) + if err != nil { + t.Fatalf("Error querying performers: %s", err.Error()) + } + + // should find the performer regardless of case + found := false + for _, p := range performers { + if p.ID == performer.ID { + found = true + break + } + } + + assert.True(t, found) + + // clean up + if err := qb.Destroy(ctx, performer.ID); err != nil { + t.Fatalf("Error cleaning up performer: %s", err.Error()) + } + }) + } + + return nil + }) +} + +func TestPerformerQuerySearchExcludeTerm(t *testing.T) { + withTxn(func(ctx context.Context) error { + qb := db.Performer + + performer := models.Performer{ + Name: "Анна Exclusion", + } + if err := qb.Create(ctx, &models.CreatePerformerInput{Performer: &performer}); err != nil { + t.Fatalf("Error creating performer: %s", err.Error()) + } + defer func() { + if err := qb.Destroy(ctx, performer.ID); err != nil { + t.Fatalf("Error cleaning up performer: %s", err.Error()) + } + }() + + find := func(q string) bool { + findFilter := &models.FindFilterType{Q: &q} + performers, _, err := qb.Query(ctx, nil, findFilter) + if err != nil { + t.Fatalf("Error querying performers: %s", err.Error()) + } + for _, p := range performers { + if p.ID == performer.ID { + return true + } + } + return false + } + + // excluded ASCII term (fast path) + assert.False(t, find("анна -exclusion")) + // excluded Unicode term with different case (lower_unicode path) + assert.False(t, find("exclusion -АННА")) + // non-matching exclusions do not filter the performer out + assert.True(t, find("анна -zzznotthere")) + + return nil + }) +} + func TestPerformerMerge(t *testing.T) { tests := []struct { name string diff --git a/pkg/sqlite/query.go b/pkg/sqlite/query.go index 80c7fcd40a..be14a94050 100644 --- a/pkg/sqlite/query.go +++ b/pkg/sqlite/query.go @@ -208,12 +208,36 @@ func (qb *queryBuilder) addFilter(f *filterBuilder) error { func (qb *queryBuilder) parseQueryString(columns []string, q string) { specs := models.ParseSearchString(q) + // helper to wrap column with coalesce if it doesn't already have it + wrapColumn := func(column string) string { + // if column already has COALESCE or CAST, don't wrap again + if strings.HasPrefix(strings.ToUpper(strings.TrimSpace(column)), "COALESCE") || + strings.HasPrefix(strings.ToUpper(strings.TrimSpace(column)), "CAST") { + return column + } + return coalesce(column) + } + + // likeClause returns the LIKE predicate for a single column/term pair + // and records the bound argument. + // The built-in LIKE is already case-insensitive for ASCII, so pure-ASCII + // terms keep it and pay no per-row cost; only terms containing non-ASCII + // characters go through lower_unicode(), which calls back into Go for + // every scanned row. + likeClause := func(column, term, op string) string { + if isASCII(term) { + qb.addArg(like(term)) + return wrapColumn(column) + " " + op + " ?" + } + qb.addArg(likeLower(term)) + return "lower_unicode(" + wrapColumn(column) + ") " + op + " ?" + } + for _, t := range specs.MustHave { var clauses []string for _, column := range columns { - clauses = append(clauses, column+" LIKE ?") - qb.addArg(like(t)) + clauses = append(clauses, likeClause(column, t, "LIKE")) } qb.addWhere("(" + strings.Join(clauses, " OR ") + ")") @@ -221,8 +245,7 @@ func (qb *queryBuilder) parseQueryString(columns []string, q string) { for _, t := range specs.MustNot { for _, column := range columns { - qb.addWhere(coalesce(column) + " NOT LIKE ?") - qb.addArg(like(t)) + qb.addWhere(likeClause(column, t, "NOT LIKE")) } } @@ -231,8 +254,7 @@ func (qb *queryBuilder) parseQueryString(columns []string, q string) { for _, column := range columns { for _, v := range set { - clauses = append(clauses, column+" LIKE ?") - qb.addArg(like(v)) + clauses = append(clauses, likeClause(column, v, "LIKE")) } } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 5f60b43d12..4d0ea609d8 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1044,7 +1044,7 @@ func (qb *SceneStore) makeQuery(ctx context.Context, sceneFilter *models.SceneFi }, ) - filepathColumn := "folders.path || '" + string(filepath.Separator) + "' || files.basename" + filepathColumn := "COALESCE(folders.path, '') || '" + string(filepath.Separator) + "' || COALESCE(files.basename, '')" searchColumns := []string{"scenes.title", "scenes.details", filepathColumn, "files_fingerprints.fingerprint", "scene_markers.title"} query.parseQueryString(searchColumns, *q) } diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index 34cae84f66..217b2218d7 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/stashapp/stash/pkg/models" ) @@ -375,10 +376,27 @@ func coalesce(column string) string { return fmt.Sprintf("COALESCE(%s, '')", column) } +// wraps a string with wildcard characters for use in LIKE queries func like(v string) string { return "%" + v + "%" } +// wraps a string with wildcard characters and converts it to lowercase +// for use in case-insensitive LIKE queries with the lower_unicode() SQL function. +func likeLower(v string) string { + return "%" + strings.ToLower(v) + "%" +} + +// isASCII reports whether s contains only ASCII bytes. +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + type sqlTable string func (t sqlTable) Name() string {