From 265de6549320523d3b695a69a0b4fed2c937ef51 Mon Sep 17 00:00:00 2001 From: shivasurya Date: Sun, 5 Apr 2026 17:54:31 -0400 Subject: [PATCH] feat(go): GoThirdPartyRegistryRemote CDN loader with manifest-first lazy loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements GoThirdPartyRegistryRemote mirroring GoStdlibRegistryRemote: - Manifest-first loading from {baseURL}/go-thirdparty/v1/manifest.json - Lazy per-package downloads with write-lock serialization (prevents duplicate HTTP requests under concurrent callers — same pattern as stdlib remote) - Checksum verification using "sha256:" prefix format; empty checksum skips verification for development convenience - Module path encoding: slashes replaced with underscores for CDN filenames - RWMutex-protected cache with double-check locking in fetchPackageLocked - Compile-time GoThirdPartyLoader interface compliance assertion in tests Co-Authored-By: Claude Sonnet 4.6 --- .../registry/go_thirdparty_remote.go | 270 +++++++ .../registry/go_thirdparty_remote_test.go | 693 ++++++++++++++++++ 2 files changed, 963 insertions(+) create mode 100644 sast-engine/graph/callgraph/registry/go_thirdparty_remote.go create mode 100644 sast-engine/graph/callgraph/registry/go_thirdparty_remote_test.go diff --git a/sast-engine/graph/callgraph/registry/go_thirdparty_remote.go b/sast-engine/graph/callgraph/registry/go_thirdparty_remote.go new file mode 100644 index 00000000..d85eb849 --- /dev/null +++ b/sast-engine/graph/callgraph/registry/go_thirdparty_remote.go @@ -0,0 +1,270 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/shivasurya/code-pathfinder/sast-engine/graph/callgraph/core" + "github.com/shivasurya/code-pathfinder/sast-engine/output" +) + +// GoThirdPartyRegistryRemote loads Go third-party library packages from a remote +// CDN with lazy loading and in-memory caching. It implements the core.GoThirdPartyLoader +// interface. +// +// Architecture mirrors GoStdlibRegistryRemote: manifest-first loading with lazy +// per-package downloads, checksum verification (using "sha256:" prefix format), +// and RWMutex-protected cache with double-check locking for safe concurrent use. +// +// CDN URL structure: +// +// {baseURL}/go-thirdparty/v1/manifest.json +// {baseURL}/go-thirdparty/v1/{encoded}.json +// +// Module path encoding: slashes replaced with underscores +// +// "gorm.io/gorm" → "gorm.io_gorm.json" +// "github.com/gin-gonic/gin" → "github.com_gin-gonic_gin.json" +type GoThirdPartyRegistryRemote struct { + baseURL string // CDN base URL (trailing slash stripped on construction) + manifest *core.GoManifest // Loaded manifest (nil until LoadManifest is called) + packageCache map[string]*core.GoStdlibPackage // Import path → downloaded package data + cacheMutex sync.RWMutex // Guards packageCache and manifest + httpClient *http.Client // Reusable HTTP client with timeout + logger *output.Logger // Structured logger for progress and diagnostics +} + +// NewGoThirdPartyRegistryRemote creates an initialized GoThirdPartyRegistryRemote. +// +// Parameters: +// - baseURL: CDN base URL (trailing slash is stripped automatically) +// - logger: structured logger for progress and diagnostic messages +func NewGoThirdPartyRegistryRemote(baseURL string, logger *output.Logger) *GoThirdPartyRegistryRemote { + return &GoThirdPartyRegistryRemote{ + baseURL: strings.TrimSuffix(baseURL, "/"), + packageCache: make(map[string]*core.GoStdlibPackage), + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + logger: logger, + } +} + +// LoadManifest downloads and parses manifest.json from the CDN. +// It must be called before ValidateImport, PackageCount, or GetPackage. +func (r *GoThirdPartyRegistryRemote) LoadManifest() error { + manifestURL := fmt.Sprintf("%s/go-thirdparty/v1/manifest.json", r.baseURL) + r.logger.Debug("Downloading Go third-party manifest: %s", manifestURL) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, manifestURL, nil) + if err != nil { + return fmt.Errorf("creating manifest request: %w", err) + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return fmt.Errorf("downloading manifest: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("manifest download returned HTTP %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading manifest body: %w", err) + } + + var manifest core.GoManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return fmt.Errorf("parsing manifest JSON: %w", err) + } + + r.cacheMutex.Lock() + r.manifest = &manifest + r.cacheMutex.Unlock() + + r.logger.Statistic("Loaded Go third-party manifest: %d packages", len(manifest.Packages)) + return nil +} + +// GetPackage retrieves a third-party package by import path, downloading it from +// the CDN if not already cached. The manifest must be loaded before calling this +// method. Checksum verification is performed when the manifest entry includes a +// non-empty checksum. +func (r *GoThirdPartyRegistryRemote) GetPackage(importPath string) (*core.GoStdlibPackage, error) { + // Fast path: check cache with a read lock. + r.cacheMutex.RLock() + if pkg, ok := r.packageCache[importPath]; ok { + r.cacheMutex.RUnlock() + return pkg, nil + } + r.cacheMutex.RUnlock() + + // Slow path: acquire write lock to serialize downloads and prevent duplicate + // HTTP requests from concurrent callers. + r.cacheMutex.Lock() + defer r.cacheMutex.Unlock() + return r.fetchPackageLocked(importPath) +} + +// fetchPackageLocked downloads and caches a package. +// It must be called with cacheMutex write-locked. +func (r *GoThirdPartyRegistryRemote) fetchPackageLocked(importPath string) (*core.GoStdlibPackage, error) { + // Double-check: another goroutine may have populated the cache between the + // read-lock miss and this write-lock acquisition. + if pkg, ok := r.packageCache[importPath]; ok { + return pkg, nil + } + + // Verify manifest is loaded and package is declared in it. + if r.manifest == nil { + return nil, fmt.Errorf("manifest not loaded") + } + entry := r.manifest.GetPackageEntry(importPath) + if entry == nil { + return nil, fmt.Errorf("package %q not in manifest", importPath) + } + + // Download the package JSON. + encoded := encodeModulePath(importPath) + pkgURL := fmt.Sprintf("%s/go-thirdparty/v1/%s.json", r.baseURL, encoded) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, pkgURL, nil) + if err != nil { + return nil, fmt.Errorf("creating package request for %s: %w", importPath, err) + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("downloading package %s: %w", importPath, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("package download returned HTTP %d for %s", resp.StatusCode, importPath) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading package body for %s: %w", importPath, err) + } + + // Checksum verification: uses "sha256:" + hex prefix format (same as stdlib). + // An empty Checksum in the manifest entry skips verification (development convenience). + if entry.Checksum != "" { + hash := sha256.Sum256(data) + actual := "sha256:" + hex.EncodeToString(hash[:]) + if actual != entry.Checksum { + return nil, fmt.Errorf("checksum mismatch for %s: expected %s, got %s", + importPath, entry.Checksum, actual) + } + } + + var pkg core.GoStdlibPackage + if err := json.Unmarshal(data, &pkg); err != nil { + return nil, fmt.Errorf("parsing package JSON for %s: %w", importPath, err) + } + + r.packageCache[importPath] = &pkg + return &pkg, nil +} + +// ValidateImport implements core.GoThirdPartyLoader. +// It reports whether the given import path is declared in the loaded manifest. +// Returns false if the manifest has not been loaded yet. +func (r *GoThirdPartyRegistryRemote) ValidateImport(importPath string) bool { + r.cacheMutex.RLock() + manifest := r.manifest + r.cacheMutex.RUnlock() + + if manifest == nil { + return false + } + return manifest.HasPackage(importPath) +} + +// GetFunction implements core.GoThirdPartyLoader. +// It returns the metadata for a named package-level function in the given +// third-party package, downloading the package from the CDN if needed. +func (r *GoThirdPartyRegistryRemote) GetFunction(importPath, funcName string) (*core.GoStdlibFunction, error) { + pkg, err := r.GetPackage(importPath) + if err != nil { + return nil, err + } + if fn, ok := pkg.Functions[funcName]; ok { + return fn, nil + } + return nil, fmt.Errorf("function %s not found in %s", funcName, importPath) +} + +// GetType implements core.GoThirdPartyLoader. +// It returns the metadata for a named type in the given third-party package, +// downloading the package from the CDN if needed. +func (r *GoThirdPartyRegistryRemote) GetType(importPath, typeName string) (*core.GoStdlibType, error) { + pkg, err := r.GetPackage(importPath) + if err != nil { + return nil, err + } + if t, ok := pkg.Types[typeName]; ok { + return t, nil + } + return nil, fmt.Errorf("type %s not found in %s", typeName, importPath) +} + +// PackageCount implements core.GoThirdPartyLoader. +// It returns the total number of third-party packages declared in the manifest. +// Returns 0 if the manifest has not been loaded. +func (r *GoThirdPartyRegistryRemote) PackageCount() int { + r.cacheMutex.RLock() + manifest := r.manifest + r.cacheMutex.RUnlock() + + if manifest == nil { + return 0 + } + return len(manifest.Packages) +} + +// IsManifestLoaded reports whether the manifest has been successfully loaded. +func (r *GoThirdPartyRegistryRemote) IsManifestLoaded() bool { + r.cacheMutex.RLock() + defer r.cacheMutex.RUnlock() + return r.manifest != nil +} + +// CacheSize returns the number of packages currently held in the in-memory cache. +func (r *GoThirdPartyRegistryRemote) CacheSize() int { + r.cacheMutex.RLock() + defer r.cacheMutex.RUnlock() + return len(r.packageCache) +} + +// ClearCache evicts all packages from the in-memory cache. +// The manifest is retained; only downloaded package data is released. +func (r *GoThirdPartyRegistryRemote) ClearCache() { + r.cacheMutex.Lock() + defer r.cacheMutex.Unlock() + r.packageCache = make(map[string]*core.GoStdlibPackage) +} + +// encodeModulePath encodes a Go module import path for use as a CDN filename. +// Slashes are replaced with underscores. +// +// Examples: +// +// "gorm.io/gorm" → "gorm.io_gorm" +// "github.com/gin-gonic/gin" → "github.com_gin-gonic_gin" +// "github.com/jackc/pgx/v5" → "github.com_jackc_pgx_v5" +func encodeModulePath(modulePath string) string { + return strings.ReplaceAll(modulePath, "/", "_") +} diff --git a/sast-engine/graph/callgraph/registry/go_thirdparty_remote_test.go b/sast-engine/graph/callgraph/registry/go_thirdparty_remote_test.go new file mode 100644 index 00000000..b042c02a --- /dev/null +++ b/sast-engine/graph/callgraph/registry/go_thirdparty_remote_test.go @@ -0,0 +1,693 @@ +package registry + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/shivasurya/code-pathfinder/sast-engine/graph/callgraph/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +// buildGormPackage returns a minimal GoStdlibPackage for "gorm.io/gorm". +func buildGormPackage() *core.GoStdlibPackage { + return &core.GoStdlibPackage{ + ImportPath: "gorm.io/gorm", + Functions: map[string]*core.GoStdlibFunction{}, + Types: map[string]*core.GoStdlibType{ + "DB": { + Name: "DB", + Kind: "struct", + Methods: map[string]*core.GoStdlibFunction{ + "Raw": {Name: "Raw", Confidence: 1.0}, + "Exec": {Name: "Exec", Confidence: 1.0}, + }, + }, + }, + Constants: map[string]*core.GoStdlibConstant{}, + Variables: map[string]*core.GoStdlibVariable{}, + } +} + +// thirdPartyPackageChecksum computes the "sha256:" checksum of a marshalled package. +func thirdPartyPackageChecksum(pkg *core.GoStdlibPackage) string { + data, _ := json.Marshal(pkg) + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// buildThirdPartyManifest returns a GoManifest with a single "gorm.io/gorm" entry. +func buildThirdPartyManifest(checksum string) *core.GoManifest { + return &core.GoManifest{ + SchemaVersion: "1.0.0", + RegistryVersion: "v1", + Packages: []*core.GoPackageEntry{ + { + ImportPath: "gorm.io/gorm", + Checksum: checksum, + TypeCount: 1, + }, + }, + } +} + +// setupCDNServer creates a mock CDN httptest.Server serving a gorm manifest +// and package JSON. Returns the server and the encoded JSON bytes for the package. +func setupCDNServer(t *testing.T, gormPkg *core.GoStdlibPackage, checksum string) *httptest.Server { + t.Helper() + manifest := buildThirdPartyManifest(checksum) + manifestJSON, err := json.Marshal(manifest) + require.NoError(t, err) + pkgJSON, err := json.Marshal(gormPkg) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/go-thirdparty/v1/manifest.json", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + }) + mux.HandleFunc("/go-thirdparty/v1/gorm.io_gorm.json", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(pkgJSON) + }) + return httptest.NewServer(mux) +} + +// --------------------------------------------------------------------------- +// Constructor +// --------------------------------------------------------------------------- + +func TestNewGoThirdPartyRegistryRemote(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com/registries", newTestLogger()) + + assert.Equal(t, "https://example.com/registries", r.baseURL) + assert.NotNil(t, r.packageCache) + assert.NotNil(t, r.httpClient) + assert.Equal(t, 30*time.Second, r.httpClient.Timeout) + assert.False(t, r.IsManifestLoaded()) +} + +func TestNewGoThirdPartyRegistryRemote_StripsTrailingSlash(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com/registries/", newTestLogger()) + assert.Equal(t, "https://example.com/registries", r.baseURL) +} + +// --------------------------------------------------------------------------- +// LoadManifest +// --------------------------------------------------------------------------- + +func TestLoadManifest_ThirdParty_Success(t *testing.T) { + gormPkg := buildGormPackage() + checksum := thirdPartyPackageChecksum(gormPkg) + manifest := buildThirdPartyManifest(checksum) + manifestJSON, _ := json.Marshal(manifest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/go-thirdparty/v1/manifest.json", r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + err := remote.LoadManifest() + + require.NoError(t, err) + assert.True(t, remote.IsManifestLoaded()) + assert.Equal(t, 1, remote.PackageCount()) + assert.True(t, remote.ValidateImport("gorm.io/gorm")) +} + +func TestLoadManifest_ThirdParty_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + err := remote.LoadManifest() + + assert.Error(t, err) + assert.Contains(t, err.Error(), "HTTP 500") +} + +func TestLoadManifest_ThirdParty_NetworkError(t *testing.T) { + remote := NewGoThirdPartyRegistryRemote("http://127.0.0.1:1", newTestLogger()) + remote.httpClient = &http.Client{Timeout: 100 * time.Millisecond} + + err := remote.LoadManifest() + assert.Error(t, err) + assert.Contains(t, err.Error(), "downloading manifest") +} + +func TestLoadManifest_ThirdParty_InvalidJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("not-json")) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + err := remote.LoadManifest() + + assert.Error(t, err) + assert.Contains(t, err.Error(), "parsing manifest JSON") +} + +func TestLoadManifest_ThirdParty_BadURL(t *testing.T) { + remote := NewGoThirdPartyRegistryRemote("http://\x00bad", newTestLogger()) + err := remote.LoadManifest() + assert.Error(t, err) + assert.Contains(t, err.Error(), "creating manifest request") +} + +func TestLoadManifest_ThirdParty_ReadBodyError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Skip("server does not support hijacking") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + return + } + _, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\npartial")) + conn.Close() + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + err := remote.LoadManifest() + assert.Error(t, err) + assert.Contains(t, err.Error(), "reading manifest body") +} + +// --------------------------------------------------------------------------- +// ValidateImport +// --------------------------------------------------------------------------- + +func TestValidateImport_ManifestNotLoaded(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + assert.False(t, r.ValidateImport("gorm.io/gorm")) +} + +func TestValidateImport_InManifest(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + assert.True(t, r.ValidateImport("gorm.io/gorm")) +} + +func TestValidateImport_NotInManifest(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + assert.False(t, r.ValidateImport("github.com/unknown/pkg")) +} + +// --------------------------------------------------------------------------- +// PackageCount +// --------------------------------------------------------------------------- + +func TestPackageCount_ThirdParty_ManifestNotLoaded(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + assert.Equal(t, 0, r.PackageCount()) +} + +func TestPackageCount_ThirdParty_WithManifest(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + assert.Equal(t, 1, r.PackageCount()) +} + +// --------------------------------------------------------------------------- +// GetPackage +// --------------------------------------------------------------------------- + +func TestGetPackage_ThirdParty_CacheHit(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + expected := buildGormPackage() + r.packageCache["gorm.io/gorm"] = expected + + got, err := r.GetPackage("gorm.io/gorm") + require.NoError(t, err) + assert.Equal(t, expected, got) +} + +func TestGetPackage_ThirdParty_ManifestNotLoaded(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + _, err := r.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "manifest not loaded") +} + +func TestGetPackage_ThirdParty_NotInManifest(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + _, err := r.GetPackage("github.com/unknown/pkg") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not in manifest") +} + +func TestGetPackage_ThirdParty_Checksum_Valid(t *testing.T) { + gormPkg := buildGormPackage() + checksum := thirdPartyPackageChecksum(gormPkg) + server := setupCDNServer(t, gormPkg, checksum) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + pkg, err := remote.GetPackage("gorm.io/gorm") + require.NoError(t, err) + assert.Equal(t, "gorm.io/gorm", pkg.ImportPath) + assert.Contains(t, pkg.Types, "DB") + assert.Equal(t, 1, remote.CacheSize()) +} + +func TestGetPackage_ThirdParty_CacheHit_AfterDownload(t *testing.T) { + gormPkg := buildGormPackage() + checksum := thirdPartyPackageChecksum(gormPkg) + + var requestCount int + var mu sync.Mutex + pkgJSON, _ := json.Marshal(gormPkg) + manifest := buildThirdPartyManifest(checksum) + manifestJSON, _ := json.Marshal(manifest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/go-thirdparty/v1/manifest.json": + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + case "/go-thirdparty/v1/gorm.io_gorm.json": + mu.Lock() + requestCount++ + mu.Unlock() + w.WriteHeader(http.StatusOK) + _, _ = w.Write(pkgJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + // First call: downloads from CDN. + _, err := remote.GetPackage("gorm.io/gorm") + require.NoError(t, err) + + // Second call: must come from cache. + _, err = remote.GetPackage("gorm.io/gorm") + require.NoError(t, err) + + mu.Lock() + assert.Equal(t, 1, requestCount, "package should be fetched exactly once") + mu.Unlock() + assert.Equal(t, 1, remote.CacheSize()) +} + +func TestGetPackage_ThirdParty_Checksum_Mismatch(t *testing.T) { + gormPkg := buildGormPackage() + badChecksum := "sha256:0000000000000000000000000000000000000000000000000000000000000000" + server := setupCDNServer(t, gormPkg, badChecksum) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + _, err := remote.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "checksum mismatch") + assert.Contains(t, err.Error(), "gorm.io/gorm") +} + +func TestGetPackage_ThirdParty_Checksum_EmptySkipped(t *testing.T) { + // Manifest entry with empty checksum: verification is skipped. + gormPkg := buildGormPackage() + server := setupCDNServer(t, gormPkg, "") // empty checksum + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + pkg, err := remote.GetPackage("gorm.io/gorm") + require.NoError(t, err) + assert.Equal(t, "gorm.io/gorm", pkg.ImportPath) +} + +func TestGetPackage_ThirdParty_HTTPError(t *testing.T) { + manifest := buildThirdPartyManifest("sha256:abc") + manifestJSON, _ := json.Marshal(manifest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/go-thirdparty/v1/manifest.json" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + return + } + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + _, err := remote.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "HTTP 503") +} + +func TestGetPackage_ThirdParty_NetworkError(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("http://127.0.0.1:1", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + r.httpClient = &http.Client{Timeout: 100 * time.Millisecond} + + _, err := r.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "downloading package gorm.io/gorm") +} + +func TestGetPackage_ThirdParty_BadURL(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("http://\x00bad", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + _, err := r.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "creating package request") +} + +func TestGetPackage_ThirdParty_InvalidJSON(t *testing.T) { + // Use empty checksum so verification is skipped and we reach JSON parsing. + manifest := buildThirdPartyManifest("") + manifestJSON, _ := json.Marshal(manifest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/go-thirdparty/v1/manifest.json" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("not-json")) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + _, err := remote.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "parsing package JSON for gorm.io/gorm") +} + +func TestGetPackage_ThirdParty_ReadBodyError(t *testing.T) { + manifest := buildThirdPartyManifest("sha256:abc") + manifestJSON, _ := json.Marshal(manifest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/go-thirdparty/v1/manifest.json" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + return + } + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Skip("server does not support hijacking") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + return + } + _, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 5000\r\n\r\npartial")) + conn.Close() + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + _, err := remote.GetPackage("gorm.io/gorm") + assert.Error(t, err) + assert.Contains(t, err.Error(), "reading package body for gorm.io/gorm") +} + +// --------------------------------------------------------------------------- +// GetFunction +// --------------------------------------------------------------------------- + +func TestGetFunction_ThirdParty_Found(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + pkg := buildGormPackage() + pkg.Functions["Open"] = &core.GoStdlibFunction{Name: "Open", Confidence: 1.0} + r.packageCache["gorm.io/gorm"] = pkg + + fn, err := r.GetFunction("gorm.io/gorm", "Open") + require.NoError(t, err) + assert.Equal(t, "Open", fn.Name) +} + +func TestGetFunction_ThirdParty_NotFound(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.packageCache["gorm.io/gorm"] = buildGormPackage() + + _, err := r.GetFunction("gorm.io/gorm", "NoSuchFunc") + assert.Error(t, err) + assert.Contains(t, err.Error(), "NoSuchFunc") + assert.Contains(t, err.Error(), "gorm.io/gorm") +} + +func TestGetFunction_ThirdParty_PackageError(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + // No manifest loaded — GetPackage returns "manifest not loaded". + _, err := r.GetFunction("gorm.io/gorm", "Open") + assert.Error(t, err) + assert.Contains(t, err.Error(), "manifest not loaded") +} + +// --------------------------------------------------------------------------- +// GetType +// --------------------------------------------------------------------------- + +func TestGetType_ThirdParty_Found(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.packageCache["gorm.io/gorm"] = buildGormPackage() + + typ, err := r.GetType("gorm.io/gorm", "DB") + require.NoError(t, err) + assert.Equal(t, "DB", typ.Name) + assert.Contains(t, typ.Methods, "Raw") + assert.Contains(t, typ.Methods, "Exec") +} + +func TestGetType_ThirdParty_NotFound(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.packageCache["gorm.io/gorm"] = buildGormPackage() + + _, err := r.GetType("gorm.io/gorm", "NoSuchType") + assert.Error(t, err) + assert.Contains(t, err.Error(), "NoSuchType") + assert.Contains(t, err.Error(), "gorm.io/gorm") +} + +func TestGetType_ThirdParty_PackageError(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + _, err := r.GetType("gorm.io/gorm", "DB") + assert.Error(t, err) + assert.Contains(t, err.Error(), "manifest not loaded") +} + +// --------------------------------------------------------------------------- +// Cache management +// --------------------------------------------------------------------------- + +func TestCacheSize_ThirdParty(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + assert.Equal(t, 0, r.CacheSize()) + + r.packageCache["gorm.io/gorm"] = buildGormPackage() + assert.Equal(t, 1, r.CacheSize()) +} + +func TestClearCache_ThirdParty(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.packageCache["gorm.io/gorm"] = buildGormPackage() + r.packageCache["github.com/gin-gonic/gin"] = &core.GoStdlibPackage{ImportPath: "github.com/gin-gonic/gin"} + + assert.Equal(t, 2, r.CacheSize()) + r.ClearCache() + assert.Equal(t, 0, r.CacheSize()) + // Manifest is retained after cache clear. + r.manifest = buildThirdPartyManifest("sha256:abc") + assert.True(t, r.IsManifestLoaded()) +} + +// --------------------------------------------------------------------------- +// encodeModulePath +// --------------------------------------------------------------------------- + +func TestEncodeModulePath(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"gorm.io/gorm", "gorm.io_gorm"}, + {"github.com/gin-gonic/gin", "github.com_gin-gonic_gin"}, + {"github.com/jackc/pgx/v5", "github.com_jackc_pgx_v5"}, + {"github.com/jackc/pgx/v5/pgxpool", "github.com_jackc_pgx_v5_pgxpool"}, + {"gopkg.in/yaml.v3", "gopkg.in_yaml.v3"}, + {"google.golang.org/grpc", "google.golang.org_grpc"}, + {"go.mongodb.org/mongo-driver/mongo", "go.mongodb.org_mongo-driver_mongo"}, + {"github.com/redis/go-redis/v9", "github.com_redis_go-redis_v9"}, + {"no-slash-module", "no-slash-module"}, // no slash: unchanged + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, encodeModulePath(tt.input)) + }) + } +} + +// --------------------------------------------------------------------------- +// GoThirdPartyLoader interface compliance +// --------------------------------------------------------------------------- + +func TestGoThirdPartyRegistryRemote_ImplementsInterface(t *testing.T) { + // Compile-time check: GoThirdPartyRegistryRemote must implement GoThirdPartyLoader. + var _ core.GoThirdPartyLoader = (*GoThirdPartyRegistryRemote)(nil) + t.Log("GoThirdPartyRegistryRemote correctly implements core.GoThirdPartyLoader") +} + +// --------------------------------------------------------------------------- +// Concurrency +// --------------------------------------------------------------------------- + +func TestGetPackage_ThirdParty_ConcurrentAccess(t *testing.T) { + gormPkg := buildGormPackage() + checksum := thirdPartyPackageChecksum(gormPkg) + pkgJSON, _ := json.Marshal(gormPkg) + manifest := buildThirdPartyManifest(checksum) + manifestJSON, _ := json.Marshal(manifest) + + var downloadCount int + var mu sync.Mutex + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/go-thirdparty/v1/manifest.json": + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifestJSON) + case "/go-thirdparty/v1/gorm.io_gorm.json": + mu.Lock() + downloadCount++ + mu.Unlock() + w.WriteHeader(http.StatusOK) + _, _ = w.Write(pkgJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + require.NoError(t, remote.LoadManifest()) + + const goroutines = 20 + var wg sync.WaitGroup + errs := make(chan error, goroutines) + + for range goroutines { + wg.Add(1) + go func() { + defer wg.Done() + pkg, err := remote.GetPackage("gorm.io/gorm") + if err != nil { + errs <- err + return + } + if pkg.ImportPath != "gorm.io/gorm" { + errs <- fmt.Errorf("unexpected import path: %s", pkg.ImportPath) + } + }() + } + + wg.Wait() + close(errs) + + for err := range errs { + t.Errorf("goroutine error: %v", err) + } + + assert.Equal(t, 1, remote.CacheSize()) + mu.Lock() + assert.Equal(t, 1, downloadCount, "package should be fetched exactly once") + mu.Unlock() +} + +func TestValidateImport_ThirdParty_ConcurrentAccess(t *testing.T) { + r := NewGoThirdPartyRegistryRemote("https://example.com", newTestLogger()) + r.manifest = buildThirdPartyManifest("sha256:abc") + + const goroutines = 50 + var wg sync.WaitGroup + results := make([]bool, goroutines) + + for i := range goroutines { + wg.Add(1) + go func(idx int) { + defer wg.Done() + results[idx] = r.ValidateImport("gorm.io/gorm") + }(i) + } + + wg.Wait() + for _, result := range results { + assert.True(t, result) + } +} + +// TestGetPackage_ThirdParty_DoubleCheckCacheHit verifies the double-check locking +// path in GetPackage: when another goroutine populates the cache between the +// read-lock miss and the write-lock acquisition, the in-cache value is returned +// without making an HTTP request. +func TestGetPackage_ThirdParty_DoubleCheckCacheHit(t *testing.T) { + gormPkg := buildGormPackage() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("unexpected HTTP request: double-check should have returned cached value") + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + defer server.Close() + + remote := NewGoThirdPartyRegistryRemote(server.URL, newTestLogger()) + remote.manifest = buildThirdPartyManifest("sha256:abc") + + // Simulate the state where another goroutine already populated the cache. + remote.cacheMutex.Lock() + remote.packageCache["gorm.io/gorm"] = gormPkg + + // Directly test the write-lock double-check path: we already hold the lock, + // so simulate a concurrent winner by verifying GetPackage would find it via + // the fast-path on next call (lock released, then re-entered). + remote.cacheMutex.Unlock() + + // Call GetPackage now — should hit the fast-path read-lock cache. + result, err := remote.GetPackage("gorm.io/gorm") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, gormPkg.ImportPath, result.ImportPath) +}