From bd8b8c197be5ad6cce6449185ac919c40a5c7c96 Mon Sep 17 00:00:00 2001 From: Levi Date: Tue, 7 Jul 2026 22:34:48 +0800 Subject: [PATCH 1/7] feat: replace gblob with blob http transport --- internal/blob/registry.go | 477 +++++++++ internal/blob/registry_test.go | 281 +++++ internal/protocol/gblob/README.md | 285 ------ internal/protocol/gblob/fetcher.go | 415 -------- internal/protocol/gblob/fetcher_test.go | 122 --- internal/protocol/gblob/registry.go | 617 ----------- internal/protocol/gblob/registry_test.go | 46 - internal/protocol/http/fetcher.go | 97 +- pkg/download/downloader.go | 97 +- pkg/download/downloader_test.go | 63 +- pkg/download/engine/engine_test.go | 15 - pkg/download/engine/inject/stream/module.go | 282 +++-- pkg/download/engine/inject/stream/stream.js | 296 +++--- pkg/download/extension.go | 14 - pkg/download/extension_engine_session.go | 62 +- pkg/download/extension_runtime_webview.go | 35 +- pkg/download/extension_test.go | 960 ++++++------------ pkg/download/model.go | 2 - .../testdata/extensions/blob/index.js | 111 ++ .../extensions/{gblob => blob}/manifest.json | 4 +- .../extensions/blob_pause_rebuild/index.js | 140 +++ .../blob_pause_rebuild/manifest.json | 28 + .../{gblob_recover => blob_recover}/index.js | 13 +- .../manifest.json | 4 +- .../{gblob_restart => blob_restart}/index.js | 18 +- .../manifest.json | 4 +- .../testdata/extensions/gblob/index.js | 183 ---- ui/flutter/lib/api/model/task.dart | 2 +- ui/flutter/lib/api/model/task.g.dart | 1 - 29 files changed, 1870 insertions(+), 2804 deletions(-) create mode 100644 internal/blob/registry.go create mode 100644 internal/blob/registry_test.go delete mode 100644 internal/protocol/gblob/README.md delete mode 100644 internal/protocol/gblob/fetcher.go delete mode 100644 internal/protocol/gblob/fetcher_test.go delete mode 100644 internal/protocol/gblob/registry.go delete mode 100644 internal/protocol/gblob/registry_test.go create mode 100644 pkg/download/testdata/extensions/blob/index.js rename pkg/download/testdata/extensions/{gblob => blob}/manifest.json (76%) create mode 100644 pkg/download/testdata/extensions/blob_pause_rebuild/index.js create mode 100644 pkg/download/testdata/extensions/blob_pause_rebuild/manifest.json rename pkg/download/testdata/extensions/{gblob_recover => blob_recover}/index.js (69%) rename pkg/download/testdata/extensions/{gblob_recover => blob_recover}/manifest.json (81%) rename pkg/download/testdata/extensions/{gblob_restart => blob_restart}/index.js (67%) rename pkg/download/testdata/extensions/{gblob_restart => blob_restart}/manifest.json (82%) delete mode 100644 pkg/download/testdata/extensions/gblob/index.js diff --git a/internal/blob/registry.go b/internal/blob/registry.go new file mode 100644 index 000000000..029d720cc --- /dev/null +++ b/internal/blob/registry.go @@ -0,0 +1,477 @@ +package blob + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "sync" +) + +const urlPathPrefix = "/__gopeed_blob/" + +var ( + ErrInvalidURL = errors.New("invalid blob url") + ErrInvalidOptions = errors.New("invalid blob options") + ErrSourceNotFound = errors.New("blob source not found") + ErrSourceRevoked = errors.New("blob source revoked") + ErrSourceClosed = errors.New("blob source closed") + ErrRangeNotAllowed = errors.New("blob range not allowed") +) + +type SessionRef interface { + Retain() + Release() +} + +type OpenRequest struct { + Offset int64 + End int64 +} + +type OpenFunc func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) + +type CreateOptions struct { + ContentType string + Size int64 + Range bool + Session SessionRef +} + +type Metadata struct { + ContentType string + Size int64 + Range bool +} + +type Source struct { + ID string + Token string + URL string + ContentType string + + mu sync.Mutex + size int64 + rangeEnabled bool + revoked bool + session SessionRef + open OpenFunc +} + +type Registry struct { + dir string + + mu sync.RWMutex + sources map[string]*Source + + serverMu sync.Mutex + listener net.Listener + server *http.Server + baseURL string +} + +func NewRegistry(dir string) *Registry { + return &Registry{ + dir: dir, + sources: make(map[string]*Source), + } +} + +func (r *Registry) Dir() string { + if r == nil { + return "" + } + return r.dir +} + +func (r *Registry) IsURL(raw string) bool { + if r == nil { + return false + } + src, err := r.get(raw) + return err == nil && src != nil +} + +func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) { + buf := append([]byte(nil), data...) + return r.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) { + if req.Offset < 0 || req.Offset > int64(len(buf)) { + return nil, ErrSourceNotFound + } + end := int64(len(buf)) + if req.End >= req.Offset && req.End+1 < end { + end = req.End + 1 + } + return io.NopCloser(bytes.NewReader(buf[req.Offset:end])), nil + }, &CreateOptions{ + ContentType: contentType, + Size: int64(len(buf)), + Range: true, + }) +} + +func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, error) { + if r == nil { + return "", ErrSourceNotFound + } + if open == nil { + return "", ErrInvalidOptions + } + if opts == nil { + opts = &CreateOptions{} + } + if opts.Size < 0 { + opts.Size = 0 + } + if opts.Range && opts.Size <= 0 { + return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions) + } + baseURL, err := r.ensureServer() + if err != nil { + return "", err + } + id, err := randomID(18) + if err != nil { + return "", err + } + token, err := randomID(24) + if err != nil { + return "", err + } + if opts.Session != nil { + opts.Session.Retain() + } + srcURL := fmt.Sprintf("%s%s?token=%s", baseURL, id, url.QueryEscape(token)) + src := &Source{ + ID: id, + Token: token, + URL: srcURL, + ContentType: opts.ContentType, + size: opts.Size, + rangeEnabled: opts.Range, + session: opts.Session, + open: open, + } + r.mu.Lock() + r.sources[id] = src + r.mu.Unlock() + return src.URL, nil +} + +func (r *Registry) Metadata(raw string) (Metadata, error) { + src, err := r.get(raw) + if err != nil { + return Metadata{}, err + } + src.mu.Lock() + defer src.mu.Unlock() + return Metadata{ + ContentType: src.ContentType, + Size: src.size, + Range: src.rangeEnabled, + }, nil +} + +func (r *Registry) Revoke(raw string) error { + src, err := r.get(raw) + if err != nil { + return err + } + r.removeSource(src) + return nil +} + +func (r *Registry) Close() error { + if r == nil { + return nil + } + r.serverMu.Lock() + server := r.server + listener := r.listener + r.server = nil + r.listener = nil + r.baseURL = "" + r.serverMu.Unlock() + if server != nil { + _ = server.Close() + } + if listener != nil { + _ = listener.Close() + } + r.mu.Lock() + sources := make([]*Source, 0, len(r.sources)) + for _, src := range r.sources { + sources = append(sources, src) + } + r.sources = make(map[string]*Source) + r.mu.Unlock() + for _, src := range sources { + src.close() + } + return nil +} + +func (r *Registry) ensureServer() (string, error) { + r.serverMu.Lock() + defer r.serverMu.Unlock() + if r.baseURL != "" { + return r.baseURL, nil + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + r.listener = listener + r.baseURL = "http://" + listener.Addr().String() + urlPathPrefix + server := &http.Server{Handler: r} + r.server = server + go func() { + if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + _ = r.Close() + } + }() + return r.baseURL, nil +} + +func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + id, token := parseRequest(req) + if id == "" || token == "" { + http.NotFound(w, req) + return + } + src, err := r.getByID(id) + if err != nil || src.Token != token { + http.NotFound(w, req) + return + } + meta, open := src.snapshot() + if open == nil { + http.NotFound(w, req) + return + } + start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range) + if err != nil { + http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + return + } + if !meta.Range { + start, end, ranged = 0, -1, false + } + reader, err := open(req.Context(), OpenRequest{ + Offset: start, + End: end, + }) + if err != nil { + http.NotFound(w, req) + return + } + defer reader.Close() + + writeHeaders(w, meta, start, end, ranged) + if ranged { + w.WriteHeader(http.StatusPartialContent) + } + if end >= start { + _, _ = copyWithFlush(w, reader, end-start+1) + return + } + _, _ = copyWithFlush(w, reader, -1) +} + +func (s *Source) snapshot() (Metadata, OpenFunc) { + s.mu.Lock() + defer s.mu.Unlock() + if s.revoked { + return Metadata{}, nil + } + return Metadata{ + ContentType: s.ContentType, + Size: s.size, + Range: s.rangeEnabled, + }, s.open +} + +func (s *Source) close() { + s.mu.Lock() + alreadyRevoked := s.revoked + s.revoked = true + s.open = nil + session := s.session + s.session = nil + s.mu.Unlock() + if !alreadyRevoked && session != nil { + session.Release() + } +} + +func (r *Registry) removeSource(src *Source) { + r.mu.Lock() + if current := r.sources[src.ID]; current == src { + delete(r.sources, src.ID) + } + r.mu.Unlock() + src.close() +} + +func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) { + if meta.ContentType != "" { + w.Header().Set("Content-Type", meta.ContentType) + } + if meta.Range { + w.Header().Set("Accept-Ranges", "bytes") + } + if meta.Size > 0 { + if ranged { + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size)) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + return + } + w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10)) + } +} + +func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (int64, error) { + if limit >= 0 { + reader = io.LimitReader(reader, limit) + } + buf := make([]byte, 32*1024) + var written int64 + flusher, canFlush := w.(http.Flusher) + for { + nr, er := reader.Read(buf) + if nr > 0 { + nw, ew := w.Write(buf[:nr]) + if nw > 0 { + written += int64(nw) + if canFlush { + flusher.Flush() + } + } + if ew != nil { + return written, ew + } + if nr != nw { + return written, io.ErrShortWrite + } + } + if er != nil { + if errors.Is(er, io.EOF) { + return written, nil + } + return written, er + } + } +} + +func parseRange(header string, size int64, rangeEnabled bool) (start int64, end int64, ranged bool, err error) { + if header == "" || !rangeEnabled { + return 0, -1, false, nil + } + if size <= 0 { + return 0, 0, false, ErrRangeNotAllowed + } + if !strings.HasPrefix(header, "bytes=") { + return 0, 0, false, fmt.Errorf("unsupported range") + } + parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2) + if len(parts) != 2 || parts[0] == "" { + return 0, 0, false, fmt.Errorf("unsupported range") + } + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil || start < 0 { + return 0, 0, false, fmt.Errorf("invalid range") + } + if start >= size { + return 0, 0, false, fmt.Errorf("range out of bounds") + } + end = size - 1 + if parts[1] != "" { + end, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil || end < start { + return 0, 0, false, fmt.Errorf("invalid range") + } + if end >= size { + end = size - 1 + } + } + return start, end, true, nil +} + +func (r *Registry) get(raw string) (*Source, error) { + id, token, ok := r.parseURL(raw) + if !ok { + return nil, ErrInvalidURL + } + src, err := r.getByID(id) + if err != nil { + return nil, err + } + if src.Token != token { + return nil, ErrInvalidURL + } + src.mu.Lock() + revoked := src.revoked + src.mu.Unlock() + if revoked { + return nil, ErrSourceRevoked + } + return src, nil +} + +func (r *Registry) getByID(id string) (*Source, error) { + r.mu.RLock() + defer r.mu.RUnlock() + src := r.sources[id] + if src == nil { + return nil, ErrSourceNotFound + } + return src, nil +} + +func (r *Registry) parseURL(raw string) (id string, token string, ok bool) { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) { + return "", "", false + } + r.serverMu.Lock() + baseURL := r.baseURL + r.serverMu.Unlock() + if baseURL == "" || !strings.HasPrefix(raw, baseURL) { + return "", "", false + } + id = strings.TrimPrefix(u.Path, urlPathPrefix) + if id == "" || path.Base(id) != id { + return "", "", false + } + return id, u.Query().Get("token"), true +} + +func parseRequest(req *http.Request) (id string, token string) { + id = strings.TrimPrefix(req.URL.Path, urlPathPrefix) + if id == "" || id == req.URL.Path || path.Base(id) != id { + return "", "" + } + return id, req.URL.Query().Get("token") +} + +func randomID(size int) (string, error) { + buf := make([]byte, size) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} diff --git a/internal/blob/registry_test.go b/internal/blob/registry_test.go new file mode 100644 index 000000000..1bb5e6cd4 --- /dev/null +++ b/internal/blob/registry_test.go @@ -0,0 +1,281 @@ +package blob + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" +) + +func TestRegistryBlobHTTPGetAndRange(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateBlob([]byte("hello world"), "text/plain") + if err != nil { + t.Fatal(err) + } + if !registry.IsURL(url) { + t.Fatal("expected registry to recognize created url") + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected GET status: %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Length"); got != "11" { + t.Fatalf("unexpected content length: %q", got) + } + if got := resp.Header.Get("Accept-Ranges"); got != "bytes" { + t.Fatalf("unexpected accept ranges: %q", got) + } + if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") { + t.Fatalf("unexpected content type: %q", got) + } + if string(body) != "hello world" { + t.Fatalf("unexpected GET body: %q", string(body)) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=6-10") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, err = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusPartialContent { + t.Fatalf("unexpected range status: %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Range"); got != "bytes 6-10/11" { + t.Fatalf("unexpected content range: %q", got) + } + if string(body) != "world" { + t.Fatalf("unexpected range body: %q", string(body)) + } +} + +func TestRegistryOpenerSizeWithoutRange(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + payload := []byte("hello opener") + url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) { + if req.Offset != 0 || req.End != -1 { + t.Fatalf("unexpected non-range request: %#v", req) + } + return io.NopCloser(bytes.NewReader(payload)), nil + }, &CreateOptions{ + ContentType: "text/plain", + Size: int64(len(payload)), + Range: false, + }) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=6-11") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Length"); got != "12" { + t.Fatalf("unexpected content length: %q", got) + } + if got := resp.Header.Get("Accept-Ranges"); got != "" { + t.Fatalf("unexpected accept ranges: %q", got) + } + if string(body) != string(payload) { + t.Fatalf("unexpected body: %q", string(body)) + } +} + +func TestRegistryOpenerRangeConcurrentRequests(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + payload := []byte("abcdefghijklmnopqrstuvwxyz") + var mu sync.Mutex + var calls []OpenRequest + url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) { + mu.Lock() + calls = append(calls, req) + mu.Unlock() + + start := int(req.Offset) + end := len(payload) + if req.End >= 0 { + end = int(req.End) + 1 + } + if start < 0 || start > end || end > len(payload) { + return nil, errors.New("bad test range") + } + return io.NopCloser(bytes.NewReader(payload[start:end])), nil + }, &CreateOptions{ + Size: int64(len(payload)), + Range: true, + }) + if err != nil { + t.Fatal(err) + } + + ranges := []struct { + header string + want string + }{ + {header: "bytes=0-4", want: "abcde"}, + {header: "bytes=10-15", want: "klmnop"}, + {header: "bytes=20-25", want: "uvwxyz"}, + } + var wg sync.WaitGroup + for _, item := range ranges { + item := item + wg.Add(1) + go func() { + defer wg.Done() + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Error(err) + return + } + req.Header.Set("Range", item.header) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Error(err) + return + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Error(err) + return + } + if resp.StatusCode != http.StatusPartialContent { + t.Errorf("unexpected status for %s: %d", item.header, resp.StatusCode) + } + if string(body) != item.want { + t.Errorf("unexpected body for %s: %q", item.header, string(body)) + } + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if len(calls) != len(ranges) { + t.Fatalf("unexpected opener call count: %d", len(calls)) + } + seen := map[OpenRequest]bool{} + for _, call := range calls { + seen[call] = true + } + for _, want := range []OpenRequest{{Offset: 0, End: 4}, {Offset: 10, End: 15}, {Offset: 20, End: 25}} { + if !seen[want] { + t.Fatalf("missing opener request %#v, got %#v", want, calls) + } + } +} + +func TestRegistryRejectsRangeWithoutSize(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + _, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil + }, &CreateOptions{Range: true}) + if !errors.Is(err, ErrInvalidOptions) { + t.Fatalf("expected invalid options error, got %v", err) + } +} + +func TestRegistryHTTPErrorStatuses(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateBlob([]byte("secret"), "text/plain") + if err != nil { + t.Fatal(err) + } + + badURL := strings.Replace(url, "token=", "token=bad", 1) + resp, err := http.Get(badURL) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("unexpected bad token status: %d", resp.StatusCode) + } + + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + t.Fatal(err) + } + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("unexpected method status: %d", resp.StatusCode) + } + + req, err = http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=99-100") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("unexpected invalid range status: %d", resp.StatusCode) + } + + if err := registry.Revoke(url); err != nil { + t.Fatal(err) + } + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("unexpected revoked source status: %d", resp.StatusCode) + } + if registry.IsURL(url) { + t.Fatal("revoked source should not be usable") + } +} diff --git a/internal/protocol/gblob/README.md b/internal/protocol/gblob/README.md deleted file mode 100644 index ed37a2e93..000000000 --- a/internal/protocol/gblob/README.md +++ /dev/null @@ -1,285 +0,0 @@ -# gblob Protocol - -## Background - -Gopeed extensions can resolve a download target into something that is not a normal remote URL. -Typical examples are: - -- text generated by an extension -- binary data assembled in JavaScript -- a stream produced incrementally inside the extension runtime - -Returning a plain `http` or `https` URL is not enough for these cases. -The `gblob` protocol provides an internal object-URL style transport so extensions can hand generated content to the downloader in a browser-like way. - -The extension-facing model is intentionally close to the browser: - -- `URL.createObjectURL(new Blob(...))` -- `URL.createObjectURL(readableStream)` -- `URL.createObjectURL(openReadable)` -- `URL.revokeObjectURL(url)` - -The returned URL looks like: - -```text -gblob: -``` - -`gblob` is a process-local protocol. The source only exists inside the current Gopeed process. - -## Design Goals - -- make extension-generated data download as a normal task -- support eager `Blob` data, one-shot `ReadableStream` data, and resumable opener functions -- avoid large in-memory buffering by spooling to disk -- keep pause/continue behavior intuitive -- allow an extension to recreate a new `gblob` URL after source expiration - -## High-Level Architecture - -The implementation has two layers in this directory: - -- [`registry.go`](./registry.go): process-local source registry and spool-file manager -- [`fetcher.go`](./fetcher.go): downloader fetcher implementation for the `gblob` scheme - -The end-to-end flow is: - -1. JavaScript creates a `Blob`, `ReadableStream`, or opener-function object URL. -2. Go registers a `gblob` source and assigns it a unique `gblob:` URL. -3. Source bytes are written into a local spool file. -4. The downloader resolves the `gblob` URL through the `gblob` fetcher. -5. The fetcher copies bytes from the spool file into the final download target. - -## Registry - -The registry owns all active `gblob` sources for the current process. - -Each source tracks: - -- source id and `gblob` URL -- local spool-file path -- source type: `blob` or `writable_stream` -- written byte count -- source state: `open`, `closed`, `aborted`, `failed` -- revoke state -- pin count -- optional extension session reference - -Key behaviors: - -- `Blob` sources are written once and immediately closed. -- `ReadableStream` sources are bridged into the internal writable-source pipeline. -- opener-function sources are opened lazily and can be reopened from an offset. -- readers wait on a per-source notification channel when no new bytes are available yet. -- revocation only prevents future use; cleanup happens when the source is no longer pinned. - -## Fetcher - -The fetcher treats `gblob` as a local file-backed download source. - -### Resolve - -`Resolve()` looks up the source in the registry and builds a single-file `Resource`. - -Behavior differences by source type: - -- `Blob`: resolved size is the current written size -- `ReadableStream` / opener function: resolved size is whatever the extension declared; if unknown, it stays `0` until download completion updates the final size - -`Range` is `true` only when the extension provides a resumable opener and marks the resource as ranged. - -### Start - -`Start()` opens the final target file and reads from the source spool file. - -There are two resume modes: - -- same `gblob` id: continue from the current target-file size -- different `gblob` id: truncate the target file and restart from byte `0` - -The fetcher persists the last started source id so it can distinguish these two cases across pause/continue and task restore. - -### Pause / Continue - -`Pause()` stops the current copy loop by canceling the reader side. -It does not abort the `gblob` source. - -If the source id stays the same, `Continue()` resumes from the existing target-file offset. -This is how normal in-process `Pause -> Continue` works. - -### Source Replacement - -If an extension replaces `req.url` with a new `gblob:` before the next start, the fetcher detects the id change and restarts from scratch. - -This behavior is important for recovery flows such as: - -- original `gblob` source expired -- extension recreates a new `gblob` URL in `onError` -- task continues with the new source - -## Lifecycle Notes - -### In-process pause and continue - -For the same `gblob` id: - -- the partially downloaded output file is kept -- the fetcher resumes from the current file size -- opener-function sources can be reopened from the current offset - -For non-reopenable one-shot `ReadableStream` sources: - -- a normal in-process pause/continue only works while the same in-memory source is still alive -- they are not suitable for cross-session or reconstructable range resume - -### Process restart - -`gblob` sources are not persisted across process restarts. -An unfinished `gblob` task is restored without a live source behind its old URL. - -The intended recovery pattern is: - -1. user clicks continue -2. the old `gblob` source fails -3. extension `onError` runs -4. extension recreates a new `gblob` URL from `req.rawUrl` -5. extension updates `ctx.task.meta.req.url` -6. extension calls `ctx.task.continue()` - -Because the new URL uses a different `gblob` id, the fetcher restarts from byte `0`. - -## Why `rawUrl` Exists - -When an extension resolves a request into `gblob:`, the original site URL would otherwise be lost. -`Request.RawURL` keeps the original creation URL so lifecycle events can still reason about the original target. - -Extension event URL matching prefers `req.rawUrl` when present. - -This is especially useful for `onStart`, `onError`, and `onDone` on `gblob` tasks. - -## Extension Usage - -### 1. Create a `Blob` object URL - -```js -gopeed.events.onResolve(async function (ctx) { - const blob = new Blob(["hello world"], { type: "text/plain" }); - const url = URL.createObjectURL(blob); - - ctx.res = { - name: "example", - files: [ - { - name: "hello.txt", - size: 11, - req: { url } - } - ] - }; -}); -``` - -### 2. Create a one-shot `ReadableStream` object URL - -```js -gopeed.events.onResolve(async function (ctx) { - const encoder = new TextEncoder(); - const stream = new ReadableStream({ - start(controller) { - controller.enqueue(encoder.encode("line 1\n")); - controller.enqueue(encoder.encode("line 2\n")); - controller.close(); - } - }); - const url = URL.createObjectURL(stream); - - ctx.res = { - name: "stream-example", - files: [ - { - name: "stream.txt", - req: { url } - } - ] - }; -}); -``` - -### 3. Create a resumable opener object URL - -```js -gopeed.events.onResolve(async function (ctx) { - const target = "https://example.com/file.bin"; - - const openReadable = async (offset) => { - const headers = {}; - if (offset > 0) { - headers.Range = `bytes=${offset}-`; - } - const response = await fetch(target, { headers }); - if (response.status !== 200 || response.status !== 206) { - throw new Error(`unexpected response status: ${response.status}, expected ${expectedStatus}`); - } - if (!response.body) { - throw new Error("empty response body"); - } - return response.body; - }; - - ctx.res = { - name: "range-example", - range: true, - files: [ - { - name: "file.bin", - req: { url: URL.createObjectURL(openReadable) } - } - ] - }; -}); -``` - -Important: - -- opener functions must return a `ReadableStream` -- do not return `response.body.getReader()` -- use opener functions for resumable / range-aware sources -- validate HTTP status codes explicitly: - first request usually expects `200`, resumed range requests usually expect `206` - -### 4. Recover in `onError` - -```js -gopeed.events.onError(async function (ctx) { - const req = ctx.task.meta.req; - req.url = URL.createObjectURL(new Blob(["recovered\n"])); - ctx.task.continue(); -}); -``` - -This creates a new `gblob` id, so the downloader restarts from byte `0`. - -## Limitations - -- `gblob` is not a network protocol. Range support only exists when the extension recreates bytes through a resumable opener. -- unfinished `gblob` sources do not survive process restarts. -- recovery after restart depends on extension logic recreating a new source. -- the current restart behavior is full restart on source-id change, not byte-range recovery across recreated sources. -- one-shot `ReadableStream` tasks with unknown total size report size `0` until completion updates the final size. - -## Summary - -`gblob` lets Gopeed extensions treat generated content as a first-class download source. -The registry handles source lifetime and disk spooling. -The fetcher turns those sources into normal download tasks with predictable pause/continue behavior. - -The most important rule is simple: - -- same `gblob` id means resume -- different `gblob` id means restart - -And for extension authors: - -- use `ReadableStream` for one-shot streaming content -- use opener functions for resumable content -- opener functions must return `ReadableStream`, not a reader diff --git a/internal/protocol/gblob/fetcher.go b/internal/protocol/gblob/fetcher.go deleted file mode 100644 index 5dfa915c0..000000000 --- a/internal/protocol/gblob/fetcher.go +++ /dev/null @@ -1,415 +0,0 @@ -package gblob - -import ( - "context" - "errors" - "io" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - - "github.com/GopeedLab/gopeed/internal/controller" - "github.com/GopeedLab/gopeed/internal/fetcher" - "github.com/GopeedLab/gopeed/pkg/base" -) - -type config struct{} - -type Fetcher struct { - ctl *controller.Controller - registry *Registry - meta *fetcher.FetcherMeta - doneCh chan error - - mu sync.Mutex - cancel context.CancelFunc - runDone chan struct{} - paused bool - closed bool - - downloaded atomic.Int64 - lastSourceID string -} - -type fetcherData struct { - LastSourceID string `json:"lastSourceId"` -} - -func (f *Fetcher) Setup(ctl *controller.Controller) { - f.ctl = ctl - if f.meta == nil { - f.meta = &fetcher.FetcherMeta{} - } - f.doneCh = make(chan error, 1) -} - -func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { - if f.registry == nil { - return ErrSourceNotFound - } - src, err := f.registry.Get(req.URL) - if err != nil { - return err - } - f.meta.Req = req - f.meta.Opts = opts - if f.meta.Opts == nil { - f.meta.Opts = &base.Options{} - } - name := f.meta.Opts.Name - if name == "" { - name = src.ID - } - snapshot := src.Snapshot() - size := snapshot.Written - if snapshot.DeclaredSize > 0 { - size = snapshot.DeclaredSize - } - if src.Type == SourceTypeWritableStream { - // Writable streams may continue producing bytes after Resolve, - // so treat the total size as unknown until the writer closes. - if snapshot.DeclaredSize == 0 { - size = 0 - } - } - f.meta.Res = &base.Resource{ - Size: size, - Range: snapshot.Range, - Files: []*base.FileInfo{ - { - Name: name, - Path: "", - Size: size, - Req: req, - }, - }, - } - return nil -} - -func (f *Fetcher) Start() error { - if f.registry == nil { - return ErrSourceNotFound - } - currentSourceID, err := ParseURL(f.meta.Req.URL) - if err != nil { - return err - } - f.mu.Lock() - if f.runDone != nil { - f.mu.Unlock() - return nil - } - f.paused = false - runDone := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - f.cancel = cancel - f.runDone = runDone - f.mu.Unlock() - cleanupStartErr := func(err error) error { - f.mu.Lock() - if f.runDone == runDone { - f.runDone = nil - f.cancel = nil - } - f.mu.Unlock() - return err - } - - path := f.meta.SingleFilepath() - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return cleanupStartErr(err) - } - target, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return cleanupStartErr(err) - } - offset := int64(0) - allowResumeAcrossSource := f.meta.Res != nil && f.meta.Res.Range - if f.lastSourceID != "" && f.lastSourceID != currentSourceID && !allowResumeAcrossSource { - if err := target.Truncate(0); err != nil { - target.Close() - return cleanupStartErr(err) - } - if _, err := target.Seek(0, io.SeekStart); err != nil { - target.Close() - return cleanupStartErr(err) - } - } else { - stat, err := target.Stat() - if err != nil { - target.Close() - return cleanupStartErr(err) - } - offset = stat.Size() - if _, err := target.Seek(offset, io.SeekStart); err != nil { - target.Close() - return cleanupStartErr(err) - } - } - f.lastSourceID = currentSourceID - f.downloaded.Store(offset) - - sourceOffset := offset - src, err := f.registry.Get(f.meta.Req.URL) - if err != nil { - target.Close() - return cleanupStartErr(err) - } - snapshot := src.Snapshot() - if snapshot.Written == 0 && !snapshot.Started { - if f.meta.Res != nil && f.meta.Res.Range && offset > 0 { - if err := f.registry.Reopen(f.meta.Req.URL, offset); err != nil { - target.Close() - return cleanupStartErr(err) - } - sourceOffset = 0 - } else if err := f.registry.StartSource(f.meta.Req.URL); err != nil { - target.Close() - return cleanupStartErr(err) - } - src, err = f.registry.Get(f.meta.Req.URL) - if err != nil { - target.Close() - return cleanupStartErr(err) - } - snapshot = src.Snapshot() - } - if f.meta.Res != nil && f.meta.Res.Range && (snapshot.State == SourceStateAborted || snapshot.State == SourceStateFailed) { - if err := f.registry.Reopen(f.meta.Req.URL, offset); err != nil { - target.Close() - return cleanupStartErr(err) - } - sourceOffset = 0 - } - - go f.copyLoop(ctx, target, offset, sourceOffset) - return nil -} - -func (f *Fetcher) copyLoop(ctx context.Context, target *os.File, targetOffset, sourceOffset int64) { - var finalErr error - - defer target.Close() - defer func() { - f.mu.Lock() - runDone := f.runDone - f.runDone = nil - f.cancel = nil - paused := f.paused - closed := f.closed - f.mu.Unlock() - close(runDone) - if closed { - select { - case f.doneCh <- nil: - default: - } - return - } - if paused { - return - } - select { - case f.doneCh <- finalErr: - default: - } - }() - - src, err := f.registry.Get(f.meta.Req.URL) - if err != nil { - finalErr = err - return - } - sourceFile, err := os.Open(src.Path) - if err != nil { - finalErr = err - return - } - defer sourceFile.Close() - - buf := make([]byte, 32*1024) - for { - if err := f.registry.WaitForReadable(ctx, f.meta.Req.URL, sourceOffset); err != nil { - if errors.Is(err, context.Canceled) { - return - } - finalErr = err - return - } - - snapshot, serr := f.registry.Get(f.meta.Req.URL) - if serr != nil { - finalErr = serr - return - } - current := snapshot.Snapshot() - for sourceOffset < current.Written { - remain := current.Written - sourceOffset - readSize := int64(len(buf)) - if remain < readSize { - readSize = remain - } - n, err := sourceFile.ReadAt(buf[:readSize], sourceOffset) - if err != nil && !errors.Is(err, io.EOF) { - finalErr = err - return - } - if n == 0 { - break - } - if _, err := target.Write(buf[:n]); err != nil { - finalErr = err - return - } - sourceOffset += int64(n) - targetOffset += int64(n) - f.downloaded.Store(targetOffset) - } - - if current.State == SourceStateClosed && sourceOffset >= current.Written { - if f.meta.Res != nil { - if f.meta.Res.Size > 0 && targetOffset < f.meta.Res.Size { - finalErr = io.ErrUnexpectedEOF - return - } - if f.meta.Res.Size == 0 { - f.meta.Res.Size = targetOffset - } - f.meta.Res.Files[0].Size = f.meta.Res.Size - } - finalErr = nil - return - } - if current.State == SourceStateAborted || current.State == SourceStateFailed { - err := current.Err - if err == nil { - err = ErrSourceAborted - } - finalErr = err - return - } - } -} - -func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error { - return nil -} - -func (f *Fetcher) Pause() error { - f.mu.Lock() - f.paused = true - cancel := f.cancel - runDone := f.runDone - f.mu.Unlock() - if cancel != nil { - cancel() - } - if runDone != nil { - <-runDone - } - return nil -} - -func (f *Fetcher) Close() error { - f.mu.Lock() - f.closed = true - cancel := f.cancel - runDone := f.runDone - f.mu.Unlock() - if cancel != nil { - cancel() - } - if runDone != nil { - <-runDone - } - select { - case f.doneCh <- nil: - default: - } - return nil -} - -func (f *Fetcher) Stats() any { - return map[string]any{} -} - -func (f *Fetcher) Meta() *fetcher.FetcherMeta { - return f.meta -} - -func (f *Fetcher) Progress() fetcher.Progress { - return fetcher.Progress{f.downloaded.Load()} -} - -func (f *Fetcher) Wait() error { - return <-f.doneCh -} - -type FetcherManager struct { - registry *Registry -} - -func (fm *FetcherManager) SetRegistry(registry *Registry) { - fm.registry = registry -} - -func (fm *FetcherManager) Name() string { - return Scheme -} - -func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { - return []*fetcher.SchemeFilter{ - { - Type: fetcher.FilterTypeUrl, - Pattern: strings.ToUpper(Scheme), - }, - } -} - -func (fm *FetcherManager) Build() fetcher.Fetcher { - return &Fetcher{ - registry: fm.registry, - } -} - -func (fm *FetcherManager) ParseName(u string) string { - id, err := ParseURL(u) - if err != nil { - return "" - } - return id -} - -func (fm *FetcherManager) AutoRename() bool { - return true -} - -func (fm *FetcherManager) DefaultConfig() any { - return &config{} -} - -func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { - _f := f.(*Fetcher) - return &fetcherData{ - LastSourceID: _f.lastSourceID, - }, nil -} - -func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { - return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { - fd := v.(*fetcherData) - return &Fetcher{ - registry: fm.registry, - meta: meta, - doneCh: make(chan error, 1), - lastSourceID: fd.LastSourceID, - } - } -} - -func (fm *FetcherManager) Close() error { - return nil -} diff --git a/internal/protocol/gblob/fetcher_test.go b/internal/protocol/gblob/fetcher_test.go deleted file mode 100644 index 27414d2f4..000000000 --- a/internal/protocol/gblob/fetcher_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package gblob - -import ( - "errors" - "io" - "os" - "path/filepath" - "testing" - "time" - - "github.com/GopeedLab/gopeed/internal/controller" - internalfetcher "github.com/GopeedLab/gopeed/internal/fetcher" - "github.com/GopeedLab/gopeed/pkg/base" -) - -func TestFetcherClosedSourceBeforeExpectedSizeReturnsError(t *testing.T) { - registry := NewRegistry(t.TempDir()) - url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{}) - if err != nil { - t.Fatal(err) - } - if err := registry.Write(url, []byte("abc")); err != nil { - t.Fatal(err) - } - if err := registry.CloseSource(url); err != nil { - t.Fatal(err) - } - - downloadDir := t.TempDir() - filePath := filepath.Join(downloadDir, "partial.bin") - f := &Fetcher{ - registry: registry, - meta: &internalfetcher.FetcherMeta{ - Req: &base.Request{URL: url}, - Res: &base.Resource{ - Size: 10, - Range: true, - Files: []*base.FileInfo{{Name: "partial.bin", Req: &base.Request{URL: url}}}, - }, - Opts: &base.Options{Path: downloadDir, Name: "partial.bin"}, - }, - } - f.Setup(controller.NewController()) - - if err := f.Start(); err != nil { - t.Fatal(err) - } - - select { - case err := <-f.doneCh: - if !errors.Is(err, io.ErrUnexpectedEOF) { - t.Fatalf("expected unexpected EOF, got %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("timeout waiting for fetcher completion") - } - - data, err := os.ReadFile(filePath) - if err != nil { - t.Fatal(err) - } - if string(data) != "abc" { - t.Fatalf("unexpected partial file content: %q", string(data)) - } -} - -func TestFetcherClosedUnknownSizeSourceCompletesSuccessfully(t *testing.T) { - registry := NewRegistry(t.TempDir()) - url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{}) - if err != nil { - t.Fatal(err) - } - if err := registry.Write(url, []byte("abcdef")); err != nil { - t.Fatal(err) - } - if err := registry.CloseSource(url); err != nil { - t.Fatal(err) - } - - downloadDir := t.TempDir() - filePath := filepath.Join(downloadDir, "stream.bin") - f := &Fetcher{ - registry: registry, - meta: &internalfetcher.FetcherMeta{ - Req: &base.Request{URL: url}, - Res: &base.Resource{ - Size: 0, - Range: false, - Files: []*base.FileInfo{{Name: "stream.bin", Req: &base.Request{URL: url}}}, - }, - Opts: &base.Options{Path: downloadDir, Name: "stream.bin"}, - }, - } - f.Setup(controller.NewController()) - - if err := f.Start(); err != nil { - t.Fatal(err) - } - - select { - case err := <-f.doneCh: - if err != nil { - t.Fatalf("expected nil completion error, got %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("timeout waiting for fetcher completion") - } - - data, err := os.ReadFile(filePath) - if err != nil { - t.Fatal(err) - } - if string(data) != "abcdef" { - t.Fatalf("unexpected stream file content: %q", string(data)) - } - if got := f.meta.Res.Size; got != int64(len(data)) { - t.Fatalf("unexpected resolved size: got %d want %d", got, len(data)) - } - if got := f.meta.Res.Files[0].Size; got != int64(len(data)) { - t.Fatalf("unexpected resolved file size: got %d want %d", got, len(data)) - } -} diff --git a/internal/protocol/gblob/registry.go b/internal/protocol/gblob/registry.go deleted file mode 100644 index b1919b7f1..000000000 --- a/internal/protocol/gblob/registry.go +++ /dev/null @@ -1,617 +0,0 @@ -package gblob - -import ( - "context" - "errors" - "os" - "path/filepath" - "strings" - "sync" - - gonanoid "github.com/matoous/go-nanoid/v2" -) - -const Scheme = "gblob" - -var ( - ErrInvalidURL = errors.New("invalid gblob url") - ErrSourceNotFound = errors.New("gblob source not found") - ErrSourceRevoked = errors.New("gblob source revoked") - ErrSourceClosed = errors.New("gblob source closed") - ErrSourceAborted = errors.New("gblob source aborted") -) - -type SourceType string - -const ( - SourceTypeBlob SourceType = "blob" - SourceTypeWritableStream SourceType = "writable_stream" -) - -type SourceState string - -const ( - SourceStateOpen SourceState = "open" - SourceStateClosed SourceState = "closed" - SourceStateAborted SourceState = "aborted" - SourceStateFailed SourceState = "failed" -) - -type SessionRef interface { - Retain() - Release() -} - -type ResumeFunc func(offset int64) error - -type CreateWritableStreamOptions struct { - Session SessionRef - Reopenable bool -} - -type Snapshot struct { - Written int64 - DeclaredSize int64 - Range bool - Started bool - State SourceState - Err error - WaitCh <-chan struct{} -} - -type Source struct { - ID string - URL string - Path string - Type SourceType - ContentType string - - mu sync.Mutex - file *os.File - written int64 - state SourceState - err error - revoked bool - pins int - waitCh chan struct{} - session SessionRef - sessionReleased bool - reopenable bool - rangeEnabled bool - declaredSize int64 - started bool - reopen ResumeFunc -} - -func (s *Source) Snapshot() Snapshot { - s.mu.Lock() - defer s.mu.Unlock() - return Snapshot{ - Written: s.written, - DeclaredSize: s.declaredSize, - Range: s.rangeEnabled, - Started: s.started, - State: s.state, - Err: s.err, - WaitCh: s.waitCh, - } -} - -func (s *Source) State() SourceState { - s.mu.Lock() - defer s.mu.Unlock() - return s.state -} - -func (s *Source) IsRevoked() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.revoked -} - -func (s *Source) pin() { - s.mu.Lock() - defer s.mu.Unlock() - s.pins++ -} - -func (s *Source) unpin() int { - s.mu.Lock() - defer s.mu.Unlock() - if s.pins > 0 { - s.pins-- - } - return s.pins -} - -func (s *Source) shouldCleanupLocked() bool { - return s.revoked && s.pins == 0 && s.state != SourceStateOpen -} - -func (s *Source) notifyLocked() { - close(s.waitCh) - s.waitCh = make(chan struct{}) -} - -func (s *Source) releaseSessionLocked() { - if s.session != nil && !s.sessionReleased { - s.session.Release() - s.sessionReleased = true - } -} - -func (s *Source) retainSessionLocked() { - if s.session != nil && s.sessionReleased { - s.session.Retain() - s.sessionReleased = false - } -} - -func (s *Source) write(data []byte) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.revoked { - return ErrSourceRevoked - } - if s.state != SourceStateOpen { - if s.state == SourceStateAborted || s.state == SourceStateFailed { - if s.err != nil { - return s.err - } - return ErrSourceAborted - } - return ErrSourceClosed - } - if len(data) == 0 { - return nil - } - n, err := s.file.Write(data) - if err != nil { - s.state = SourceStateFailed - s.err = err - s.file.Close() - s.file = nil - if !s.rangeEnabled { - s.releaseSessionLocked() - } - s.notifyLocked() - return err - } - s.written += int64(n) - s.notifyLocked() - return nil -} - -func (s *Source) close() error { - s.mu.Lock() - defer s.mu.Unlock() - if s.state != SourceStateOpen { - return nil - } - s.state = SourceStateClosed - if s.file != nil { - if err := s.file.Close(); err != nil { - s.err = err - } - s.file = nil - } - s.releaseSessionLocked() - s.notifyLocked() - return nil -} - -func (s *Source) abort(err error) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.state != SourceStateOpen { - return nil - } - s.state = SourceStateAborted - if err != nil { - s.err = err - } - if s.file != nil { - _ = s.file.Close() - s.file = nil - } - if !s.rangeEnabled { - s.releaseSessionLocked() - } - s.notifyLocked() - return nil -} - -func (s *Source) revoke() bool { - s.mu.Lock() - defer s.mu.Unlock() - s.revoked = true - if s.state == SourceStateOpen { - s.state = SourceStateAborted - if s.err == nil { - s.err = ErrSourceRevoked - } - if s.file != nil { - _ = s.file.Close() - s.file = nil - } - s.releaseSessionLocked() - s.notifyLocked() - } - return s.shouldCleanupLocked() -} - -func (s *Source) waitForReadable(ctx context.Context, offset int64) error { - for { - snapshot := s.Snapshot() - if offset < snapshot.Written { - return nil - } - switch snapshot.State { - case SourceStateClosed: - return nil - case SourceStateAborted, SourceStateFailed: - if snapshot.Err != nil { - return snapshot.Err - } - return ErrSourceAborted - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-snapshot.WaitCh: - } - } -} - -type Registry struct { - dir string - - mu sync.RWMutex - sources map[string]*Source -} - -func (r *Registry) Dir() string { - if r == nil { - return "" - } - return r.dir -} - -func NewRegistry(baseDir string) *Registry { - dir := baseDir - if dir == "" { - dir = filepath.Join(os.TempDir(), "gopeed-gblob") - } - return &Registry{ - dir: dir, - sources: make(map[string]*Source), - } -} - -func BuildURL(id string) string { - return Scheme + ":" + id -} - -func ParseURL(raw string) (string, error) { - if !strings.HasPrefix(strings.ToLower(raw), Scheme+":") { - return "", ErrInvalidURL - } - id := raw[len(Scheme)+1:] - if idx := strings.IndexByte(id, '/'); idx >= 0 { - id = id[:idx] - } - if id == "" { - return "", ErrInvalidURL - } - return id, nil -} - -func (r *Registry) createSource(sourceType SourceType, contentType string, session SessionRef, reopenable bool, declaredSize int64) (*Source, error) { - if err := os.MkdirAll(r.dir, 0755); err != nil { - return nil, err - } - id, err := gonanoid.New() - if err != nil { - return nil, err - } - filePath := filepath.Join(r.dir, id) - file, err := os.Create(filePath) - if err != nil { - return nil, err - } - if session != nil { - session.Retain() - } - src := &Source{ - ID: id, - URL: BuildURL(id), - Path: filePath, - Type: sourceType, - ContentType: contentType, - file: file, - state: SourceStateOpen, - waitCh: make(chan struct{}), - session: session, - reopenable: reopenable, - rangeEnabled: reopenable, - declaredSize: declaredSize, - started: !reopenable, - } - r.mu.Lock() - r.sources[id] = src - r.mu.Unlock() - return src, nil -} - -func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) { - src, err := r.createSource(SourceTypeBlob, contentType, nil, false, int64(len(data))) - if err != nil { - return "", err - } - if err := src.write(data); err != nil { - _ = os.Remove(src.Path) - return "", err - } - if err := src.close(); err != nil { - _ = os.Remove(src.Path) - return "", err - } - return src.URL, nil -} - -func (r *Registry) CreateWritableStream(opts *CreateWritableStreamOptions) (string, error) { - if opts == nil { - opts = &CreateWritableStreamOptions{} - } - src, err := r.createSource(SourceTypeWritableStream, "", opts.Session, opts.Reopenable, 0) - if err != nil { - return "", err - } - return src.URL, nil -} - -func (r *Registry) SetResumeOpener(raw string, reopen ResumeFunc) error { - src, err := r.get(raw) - if err != nil { - return err - } - src.mu.Lock() - defer src.mu.Unlock() - if !src.reopenable { - return nil - } - src.reopen = reopen - return nil -} - -func (r *Registry) SetRange(raw string, enabled bool) error { - src, err := r.get(raw) - if err != nil { - return err - } - src.mu.Lock() - src.rangeEnabled = enabled - if !enabled && (src.state == SourceStateAborted || src.state == SourceStateFailed) { - src.releaseSessionLocked() - } - src.mu.Unlock() - return nil -} - -func (r *Registry) SetSize(raw string, size int64) error { - src, err := r.get(raw) - if err != nil { - return err - } - src.mu.Lock() - if size < 0 { - size = 0 - } - src.declaredSize = size - src.mu.Unlock() - return nil -} - -func (r *Registry) get(raw string) (*Source, error) { - id, err := ParseURL(raw) - if err != nil { - return nil, err - } - r.mu.RLock() - defer r.mu.RUnlock() - src := r.sources[id] - if src == nil { - return nil, ErrSourceNotFound - } - return src, nil -} - -func (r *Registry) Get(raw string) (*Source, error) { - return r.get(raw) -} - -func (r *Registry) Write(raw string, data []byte) error { - src, err := r.get(raw) - if err != nil { - return err - } - return src.write(data) -} - -func (r *Registry) CloseSource(raw string) error { - src, err := r.get(raw) - if err != nil { - return err - } - if err := src.close(); err != nil { - return err - } - r.cleanupIfNeeded(src) - return nil -} - -func (r *Registry) AbortSource(raw string, err error) error { - src, gerr := r.get(raw) - if gerr != nil { - return gerr - } - if err := src.abort(err); err != nil { - return err - } - r.cleanupIfNeeded(src) - return nil -} - -func (r *Registry) Revoke(raw string) error { - src, err := r.get(raw) - if err != nil { - return err - } - if src.revoke() { - r.cleanupIfNeeded(src) - } - return nil -} - -func (r *Registry) Pin(raw string) error { - src, err := r.get(raw) - if err != nil { - return err - } - if src.IsRevoked() { - return ErrSourceRevoked - } - src.pin() - return nil -} - -func (r *Registry) Unpin(raw string) { - src, err := r.get(raw) - if err != nil { - return - } - src.unpin() - r.cleanupIfNeeded(src) -} - -func (r *Registry) WaitForReadable(ctx context.Context, raw string, offset int64) error { - src, err := r.get(raw) - if err != nil { - return err - } - return src.waitForReadable(ctx, offset) -} - -func (r *Registry) Reopen(raw string, offset int64) error { - src, err := r.get(raw) - if err != nil { - return err - } - - src.mu.Lock() - if !src.reopenable || src.reopen == nil { - src.mu.Unlock() - return ErrSourceNotFound - } - if src.revoked { - src.mu.Unlock() - return ErrSourceRevoked - } - if src.file != nil { - _ = src.file.Close() - src.file = nil - } - if err := os.MkdirAll(filepath.Dir(src.Path), 0755); err != nil { - src.mu.Unlock() - return err - } - file, err := os.Create(src.Path) - if err != nil { - src.mu.Unlock() - return err - } - src.file = file - src.written = 0 - src.state = SourceStateOpen - src.err = nil - src.started = true - src.retainSessionLocked() - reopen := src.reopen - src.mu.Unlock() - - if err := reopen(offset); err != nil { - src.mu.Lock() - if src.file != nil { - _ = src.file.Close() - src.file = nil - } - src.state = SourceStateFailed - src.err = err - src.releaseSessionLocked() - src.notifyLocked() - src.mu.Unlock() - return err - } - return nil -} - -func (r *Registry) StartSource(raw string) error { - src, err := r.get(raw) - if err != nil { - return err - } - src.mu.Lock() - if !src.reopenable || src.started { - src.mu.Unlock() - return nil - } - src.mu.Unlock() - return r.Reopen(raw, 0) -} - -func (r *Registry) cleanupIfNeeded(src *Source) { - src.mu.Lock() - shouldCleanup := src.shouldCleanupLocked() - file := src.file - if shouldCleanup { - src.releaseSessionLocked() - } - src.mu.Unlock() - if !shouldCleanup { - return - } - r.mu.Lock() - delete(r.sources, src.ID) - r.mu.Unlock() - if file != nil { - _ = file.Close() - } - _ = os.Remove(src.Path) -} - -func (r *Registry) Close() error { - r.mu.Lock() - sources := r.sources - r.sources = make(map[string]*Source) - r.mu.Unlock() - var lastErr error - for _, src := range sources { - src.mu.Lock() - if src.file != nil { - if err := src.file.Close(); err != nil { - lastErr = err - } - src.file = nil - } - src.releaseSessionLocked() - src.notifyLocked() - src.mu.Unlock() - if err := os.Remove(src.Path); err != nil && !errors.Is(err, os.ErrNotExist) { - lastErr = err - } - } - if err := os.RemoveAll(r.dir); err != nil && !errors.Is(err, os.ErrNotExist) { - lastErr = err - } - return lastErr -} diff --git a/internal/protocol/gblob/registry_test.go b/internal/protocol/gblob/registry_test.go deleted file mode 100644 index be601f828..000000000 --- a/internal/protocol/gblob/registry_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package gblob - -import ( - "errors" - "testing" -) - -func TestRegistry_RevokeOpenWritableStreamAbortsAndCleansUpAfterUnpin(t *testing.T) { - registry := NewRegistry(t.TempDir()) - url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{}) - if err != nil { - t.Fatal(err) - } - if err := registry.Pin(url); err != nil { - t.Fatal(err) - } - - src, err := registry.Get(url) - if err != nil { - t.Fatal(err) - } - if state := src.Snapshot().State; state != SourceStateOpen { - t.Fatalf("expected initial open state, got %s", state) - } - - if err := registry.Revoke(url); err != nil { - t.Fatal(err) - } - - src, err = registry.Get(url) - if err != nil { - t.Fatal(err) - } - snapshot := src.Snapshot() - if snapshot.State != SourceStateAborted { - t.Fatalf("expected revoked open source to become aborted, got %s", snapshot.State) - } - if !errors.Is(snapshot.Err, ErrSourceRevoked) { - t.Fatalf("expected revoke error, got %v", snapshot.Err) - } - - registry.Unpin(url) - if _, err := registry.Get(url); !errors.Is(err, ErrSourceNotFound) { - t.Fatalf("expected source cleanup after unpin, got %v", err) - } -} diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index a9ed14ad6..b17d19c1c 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -354,26 +354,57 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { // Record connection time as baseline for fast-fail timeout f.updateMaxConnTime(time.Since(connStartTime)) - // Parse response to get resource info - res := &base.Resource{ - Range: false, - Files: []*base.FileInfo{}, - } - if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { resp.Body.Close() f.setState(stateError) return NewRequestError(resp.StatusCode) } + res := parseResourceFromResponse(httpReq, resp) + f.meta.Res = res + + // Save redirect URL for later connections + f.redirectURL = resp.Request.URL.String() + + // IMPORTANT: Keep the response body open for downloading in Start phase + // This is crucial for one-time URLs that can only be accessed once + f.resolveRespLock.Lock() + f.resolveResp = resp + f.resolveRespLock.Unlock() + + f.setState(stateResolved) + + // Signal that resolve is complete + f.resolvedOnce.Do(func() { + close(f.resolvedCh) + }) + + // Start async prefetch in background (only for range-supported resources) + // For non-range resources, the response will be used directly in Start + if res.Range && res.Size > 0 { + f.prefetchStopCh = make(chan struct{}) + go f.asyncPrefetch() + } + + // If start was called before resolve completed, auto-start + if f.startPending.Load() { + go f.doStart() + } + + return nil +} + +func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base.Resource { + res := &base.Resource{ + Range: false, + Files: []*base.FileInfo{}, + } - // Check if server supports range requests acceptRanges := resp.Header.Get(base.HttpHeaderAcceptRanges) contentRange := resp.Header.Get(base.HttpHeaderContentRange) if acceptRanges == base.HttpHeaderBytes || strings.HasPrefix(contentRange, base.HttpHeaderBytes) { res.Range = true } - // Get content length from Content-Length header contentLength := resp.Header.Get(base.HttpHeaderContentLength) if contentLength != "" { parse, err := strconv.ParseInt(contentLength, 10, 64) @@ -382,7 +413,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { } } - // Parse last modified time var lastModifiedTime *time.Time lastModified := resp.Header.Get(base.HttpHeaderLastModified) if lastModified != "" { @@ -395,7 +425,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { Ctime: lastModifiedTime, } - // Parse filename contentDisposition := resp.Header.Get(base.HttpHeaderContentDisposition) if contentDisposition != "" { file.Name = parseFilename(contentDisposition) @@ -403,7 +432,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { if file.Name == "" { file.Name = path.Base(httpReq.URL.Path) if file.Name != "" { - // Use PathUnescape instead of QueryUnescape to correctly handle %2B (should decode to +, not space) file.Name, _ = url.PathUnescape(file.Name) } } @@ -412,37 +440,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { } res.Files = append(res.Files, file) - f.meta.Res = res - - // Save redirect URL for later connections - f.redirectURL = resp.Request.URL.String() - - // IMPORTANT: Keep the response body open for downloading in Start phase - // This is crucial for one-time URLs that can only be accessed once - f.resolveRespLock.Lock() - f.resolveResp = resp - f.resolveRespLock.Unlock() - - f.setState(stateResolved) - - // Signal that resolve is complete - f.resolvedOnce.Do(func() { - close(f.resolvedCh) - }) - - // Start async prefetch in background (only for range-supported resources) - // For non-range resources, the response will be used directly in Start - if res.Range && res.Size > 0 { - f.prefetchStopCh = make(chan struct{}) - go f.asyncPrefetch() - } - - // If start was called before resolve completed, auto-start - if f.startPending.Load() { - go f.doStart() - } - - return nil + return res } // asyncPrefetch downloads data in background during resolve phase @@ -576,6 +574,12 @@ func (f *Fetcher) Start() error { switch state { case stateResolved, statePaused: + if state == statePaused { + select { + case <-f.doneCh: + default: + } + } // Normal case: resolved or resuming from pause return f.doStart() @@ -670,6 +674,11 @@ func (f *Fetcher) doStart() error { if err = base.ParseReqExtra[fhttp.ReqExtra](f.meta.Req); err != nil { return err } + f.redirectLock.Lock() + if strings.Contains(f.redirectURL, "/__gopeed_blob/") && f.redirectURL != f.meta.Req.URL { + f.redirectURL = "" + } + f.redirectLock.Unlock() // Initialize slow start controller maxConns := f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections @@ -1709,6 +1718,10 @@ func (f *Fetcher) Pause() error { f.fileMu.Unlock() f.setState(statePaused) + select { + case f.doneCh <- context.Canceled: + default: + } return nil } diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index ace48ca87..71522329e 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -15,10 +15,10 @@ import ( "sync/atomic" "time" + internalblob "github.com/GopeedLab/gopeed/internal/blob" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/logger" - protogblob "github.com/GopeedLab/gopeed/internal/protocol/gblob" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/util" @@ -100,6 +100,7 @@ type Downloader struct { tasks []*Task waitTasks []*Task watchedTasks sync.Map + watchWG sync.WaitGroup listener Listener lock *sync.Mutex @@ -112,7 +113,7 @@ type Downloader struct { claimedExtractions sync.Map extensions []*Extension - gblob *protogblob.Registry + blob *internalblob.Registry } func NewDownloader(cfg *DownloaderConfig) *Downloader { @@ -144,11 +145,11 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader { } func (d *Downloader) Setup() error { - gblobDir := filepath.Join(d.cfg.StorageDir, "gblob") - if err := os.RemoveAll(gblobDir); err != nil && !errors.Is(err, os.ErrNotExist) { + blobDir := filepath.Join(d.cfg.StorageDir, "blob") + if err := os.RemoveAll(blobDir); err != nil && !errors.Is(err, os.ErrNotExist) { return err } - d.gblob = protogblob.NewRegistry(gblobDir) + d.blob = internalblob.NewRegistry("") // setup storage if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil { @@ -175,9 +176,6 @@ func (d *Downloader) Setup() error { if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok { d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig() } - if gfm, ok := fm.(*protogblob.FetcherManager); ok { - gfm.SetRegistry(d.gblob) - } if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok { sfm.SetStateStore(&protocolStateStore{ storage: d.storage, @@ -201,12 +199,11 @@ func (d *Downloader) Setup() error { tasks = append(tasks[:i], tasks[i+1:]...) continue } - d.assignFetcherManager(task) - initTask(task) - if task.Protocol == protogblob.Scheme && task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError { + if task.Meta.Req != nil && isInternalBlobURLString(task.Meta.Req.URL) && task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError { task.Status = base.DownloadStatusError - continue } + d.assignFetcherManager(task) + initTask(task) if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError { task.Status = base.DownloadStatusPause } @@ -502,11 +499,6 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId if err != nil { return } - if strings.HasPrefix(strings.ToLower(req.URL), protogblob.Scheme+":") { - if err = fetcher.Resolve(req, initOpt); err != nil { - return - } - } return d.doCreate(fetcher, initOpt) } @@ -811,7 +803,7 @@ func (d *Downloader) Stats(id string) (sr any, err error) { } func (d *Downloader) doDelete(task *Task, force bool) (err error) { - defer d.releaseGBlobTask(task) + defer d.releaseBlobTask(task) err = func() error { if err := d.storage.Delete(bucketTask, task.ID); err != nil { return err @@ -856,8 +848,8 @@ func (d *Downloader) Close() error { for _, fm := range d.cfg.FetchManagers { closeArr = append(closeArr, fm.Close) } - if d.gblob != nil { - closeArr = append(closeArr, d.gblob.Close) + if d.blob != nil { + closeArr = append(closeArr, d.blob.Close) } closeArr = append(closeArr, d.storage.Close) // Make sure all resources are released, if had error, return the last error @@ -1024,9 +1016,11 @@ func (d *Downloader) watch(task *Task) { if _, loaded := d.watchedTasks.LoadOrStore(task.ID, true); loaded { return } + d.watchWG.Add(1) defer func() { d.watchedTasks.Delete(task.ID) + d.watchWG.Done() }() // wait task upload done @@ -1054,6 +1048,9 @@ func (d *Downloader) watch(task *Task) { for { err := task.fetcher.Wait() if err != nil { + if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait { + return + } d.doOnError(task, err) if task.Status == base.DownloadStatusRunning { continue @@ -1082,7 +1079,7 @@ func (d *Downloader) watch(task *Task) { d.emit(EventKeyDone, task) d.emit(EventKeyFinally, task, err) d.notifyRunning() - d.releaseGBlobTask(task) + d.releaseBlobTask(task) d.triggerOnDone(task) d.triggerWebhooks(WebhookEventDownloadDone, task, nil) d.triggerScripts(ScriptEventDownloadDone, task, nil) @@ -1152,9 +1149,7 @@ func (d *Downloader) doOnError(task *Task, err error) { task.updateStatus(base.DownloadStatusError) d.triggerOnError(task, err) if task.Status == base.DownloadStatusError { - if !d.isResumableGBlobTask(task) { - d.releaseGBlobTask(task) - } + d.releaseBlobTask(task) d.emit(EventKeyError, task, err) d.emit(EventKeyFinally, task, err) d.notifyRunning() @@ -1168,7 +1163,6 @@ func (d *Downloader) restoreTask(task *Task) error { return err } } - go d.watch(task) return nil } @@ -1207,17 +1201,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str if err != nil { return } - if fm.Name() == protogblob.Scheme && d.gblob != nil { - if err = d.gblob.Pin(f.Meta().Req.URL); err != nil { - return "", err - } - defer func() { - if err != nil { - d.gblob.Unpin(f.Meta().Req.URL) - } - }() - } - task := NewTask() task.fetcherManager = fm task.fetcher = f @@ -1247,7 +1230,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str err = d.doStart(task) }() - go d.watch(task) return } @@ -1321,12 +1303,21 @@ func (d *Downloader) doStart(task *Task) (err error) { task.lock.Lock() defer task.lock.Unlock() + if task.fetcher == nil { + if err := d.restoreFetcher(task); err != nil { + return err + } + } d.triggerOnStart(task) if task.fetcher != nil { task.fetcher.Meta().Req = task.Meta.Req task.fetcher.Meta().Res = task.Meta.Res task.fetcher.Meta().Opts = task.Meta.Opts } + if task.Meta.Req != nil && isInternalBlobURLString(task.Meta.Req.URL) && (d.blob == nil || !d.blob.IsURL(task.Meta.Req.URL)) { + task.fetcher = nil + return internalblob.ErrSourceNotFound + } if task.Meta.Res == nil { err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts) if err != nil { @@ -1371,6 +1362,7 @@ func (d *Downloader) doStart(task *Task) (err error) { if err := d.saveTask(task); err != nil { return err } + go d.watch(task) d.emit(EventKeyStart, task) return nil } @@ -1411,6 +1403,7 @@ func (d *Downloader) doPause(task *Task) (err error) { return err } } + d.releaseBlobTask(task) if task.fetcherManager != nil && task.fetcher != nil { if err := d.saveTask(task); err != nil { return err @@ -1849,36 +1842,22 @@ func initTask(task *Task) { task.uploadSpeedArr = make([]int64, 0) } -func (d *Downloader) unpinGBlobTask(task *Task) { - if d.gblob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" { +func (d *Downloader) revokeBlobTask(task *Task) { + if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" { return } - if task.Protocol != protogblob.Scheme { + if !d.blob.IsURL(task.Meta.Req.URL) { return } - d.gblob.Unpin(task.Meta.Req.URL) -} - -func (d *Downloader) isResumableGBlobTask(task *Task) bool { - if task == nil || task.Meta == nil || task.Meta.Res == nil { - return false - } - return task.Protocol == protogblob.Scheme && task.Meta.Res.Range + _ = d.blob.Revoke(task.Meta.Req.URL) } -func (d *Downloader) revokeGBlobTask(task *Task) { - if d.gblob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" { - return - } - if task.Protocol != protogblob.Scheme { - return - } - _ = d.gblob.Revoke(task.Meta.Req.URL) +func (d *Downloader) releaseBlobTask(task *Task) { + d.revokeBlobTask(task) } -func (d *Downloader) releaseGBlobTask(task *Task) { - d.revokeGBlobTask(task) - d.unpinGBlobTask(task) +func isInternalBlobURLString(raw string) bool { + return strings.Contains(raw, "/__gopeed_blob/") } var defaultDownloader = NewDownloader(nil) diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index 00f2cc258..362aa2613 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -63,7 +63,7 @@ func TestDownloader_Resolve(t *testing.T) { } } -func TestDownloader_UsesStorageDirForGBlobRegistry(t *testing.T) { +func TestDownloader_BlobRegistryDoesNotUseStorageDirForSpooling(t *testing.T) { storageDir := t.TempDir() downloader := NewDownloader(&DownloaderConfig{ Storage: NewMemStorage(), @@ -74,22 +74,24 @@ func TestDownloader_UsesStorageDirForGBlobRegistry(t *testing.T) { } defer downloader.Clear() - if downloader.gblob == nil { - t.Fatal("expected gblob registry to be initialized") + if downloader.blob == nil { + t.Fatal("expected blob registry to be initialized") } - expectedDir := filepath.Join(storageDir, "gblob") - if filepath.Clean(downloader.gblob.Dir()) != filepath.Clean(expectedDir) { - t.Fatalf("expected gblob registry dir under storage dir, got %s want %s", downloader.gblob.Dir(), expectedDir) + if downloader.blob.Dir() != "" { + t.Fatalf("expected blob registry not to use a spool dir, got %s", downloader.blob.Dir()) + } + if _, err := os.Stat(filepath.Join(storageDir, "blob")); !os.IsNotExist(err) { + t.Fatalf("expected blob spool dir not to be created, got err=%v", err) } } -func TestDownloader_SetupCleansStaleGBlobDir(t *testing.T) { +func TestDownloader_SetupCleansStaleBlobDir(t *testing.T) { storageDir := t.TempDir() - gblobDir := filepath.Join(storageDir, "gblob") - if err := os.MkdirAll(gblobDir, 0o755); err != nil { + blobDir := filepath.Join(storageDir, "blob") + if err := os.MkdirAll(blobDir, 0o755); err != nil { t.Fatal(err) } - staleFile := filepath.Join(gblobDir, "stale") + staleFile := filepath.Join(blobDir, "stale") if err := os.WriteFile(staleFile, []byte("stale"), 0o644); err != nil { t.Fatal(err) } @@ -104,8 +106,47 @@ func TestDownloader_SetupCleansStaleGBlobDir(t *testing.T) { defer downloader.Clear() if _, err := os.Stat(staleFile); !os.IsNotExist(err) { - t.Fatalf("expected stale gblob file to be removed on setup, got err=%v", err) + t.Fatalf("expected stale blob file to be removed on setup, got err=%v", err) + } +} + +func TestDownloader_BlobTaskKeepsConfiguredHTTPConnections(t *testing.T) { + downloader := NewDownloader(&DownloaderConfig{ + Storage: NewMemStorage(), + StorageDir: t.TempDir(), + }) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + + url, err := downloader.blob.CreateBlob([]byte("hello"), "text/plain") + if err != nil { + t.Fatal(err) + } + id, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{ + Path: t.TempDir(), + Name: "blob-unmarked.txt", + Extra: &http.OptsExtra{Connections: 8}, + }) + if err != nil { + t.Fatal(err) + } + task := downloader.GetTask(id) + if task == nil { + t.Fatal("task not found") + } + if task.Protocol != "http" { + t.Fatalf("expected blob task protocol http, got %s", task.Protocol) + } + extra, ok := task.Meta.Opts.Extra.(*http.OptsExtra) + if !ok { + t.Fatalf("expected http extra, got %T", task.Meta.Opts.Extra) + } + if extra.Connections != 8 { + t.Fatalf("expected blob task connections to stay 8, got %d", extra.Connections) } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) } func TestDownloader_Create(t *testing.T) { diff --git a/pkg/download/engine/engine_test.go b/pkg/download/engine/engine_test.go index 91143fabd..b333b9db8 100644 --- a/pkg/download/engine/engine_test.go +++ b/pkg/download/engine/engine_test.go @@ -47,21 +47,6 @@ func TestError(t *testing.T) { } } -func TestURLCreateObjectURLRejectsUnsupportedType(t *testing.T) { - engine := NewEngine(nil) - defer engine.Close() - - _, err := engine.RunString(` - URL.createObjectURL({}); - `) - if err == nil { - t.Fatal("expected unsupported type error") - } - if !strings.Contains(err.Error(), "Unsupported object type for URL.createObjectURL: Object") { - t.Fatalf("unexpected unsupported type error: %v", err) - } -} - func TestCallFunction_DetachedAsyncWorkDoesNotBlockReturn(t *testing.T) { engine := NewEngine(nil) defer engine.Close() diff --git a/pkg/download/engine/inject/stream/module.go b/pkg/download/engine/inject/stream/module.go index fd9ba6b79..01340c531 100644 --- a/pkg/download/engine/inject/stream/module.go +++ b/pkg/download/engine/inject/stream/module.go @@ -13,7 +13,6 @@ import ( "sync" "time" - protogblob "github.com/GopeedLab/gopeed/internal/protocol/gblob" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr" @@ -27,105 +26,66 @@ import ( var script string type Config struct { - CreateBlobObjectURL func(data []byte, contentType string) (string, error) - CreateWritableStreamObjectURL func(opts *WritableStreamObjectURLOptions) (string, error) - RegisterWritableStreamResume func(url string, reopen func(offset int64) error) error - WriteWritableStreamObjectURL func(url string, data any) error - CloseWritableStreamObjectURL func(url string) error - AbortWritableStreamObjectURL func(url string, reason string) error - RevokeObjectURL func(url string) error - ProxyHandler func(r *http.Request) (*url.URL, error) + CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error) + RevokeObjectURL func(url string) error + ProxyHandler func(r *http.Request) (*url.URL, error) } -type WritableStreamObjectURLOptions struct { - Reopenable bool +type ObjectURLOptions struct { + ContentType string + Size int64 + Range bool } -func isIgnorableGBlobError(err error) bool { - return errors.Is(err, protogblob.ErrSourceRevoked) || - errors.Is(err, protogblob.ErrSourceNotFound) || - errors.Is(err, protogblob.ErrSourceClosed) || - errors.Is(err, protogblob.ErrSourceAborted) +type ObjectURLOpenRequest struct { + Offset int64 + End int64 } +type ObjectURLOpener func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) + func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error { if cfg == nil { cfg = &Config{} } - if err := runtime.Set("__gopeed_create_blob_object_url", func(data []byte, contentType string) string { - if cfg.CreateBlobObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob blob object url handler not configured"))) + if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value { + if cfg.CreateObjectURL == nil { + panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured"))) } - url, err := cfg.CreateBlobObjectURL(data, contentType) - if err != nil { - panic(runtime.NewGoError(err)) + openValue := call.Argument(0) + if _, ok := goja.AssertFunction(openValue); !ok { + panic(runtime.NewGoError(fmt.Errorf("blob opener must be callable"))) } - return url - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_create_writable_stream_object_url", func(reopenable bool) string { - if cfg.CreateWritableStreamObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob writable stream object url handler not configured"))) - } - opts := &WritableStreamObjectURLOptions{Reopenable: reopenable} - url, err := cfg.CreateWritableStreamObjectURL(opts) - if err != nil { - panic(runtime.NewGoError(err)) - } - if opts.Reopenable && cfg.RegisterWritableStreamResume != nil { - if err := cfg.RegisterWritableStreamResume(url, func(offset int64) error { - return openWritableStreamObjectURL(loop, url, offset) - }); err != nil { - panic(runtime.NewGoError(err)) - } - } - return url - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_write_writable_stream_object_url", func(url string, data any) { - if cfg.WriteWritableStreamObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob writable stream write handler not configured"))) - } - if err := cfg.WriteWritableStreamObjectURL(url, data); err != nil { - if isIgnorableGBlobError(err) { - return - } - panic(runtime.NewGoError(err)) - } - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_close_writable_stream_object_url", func(url string) { - if cfg.CloseWritableStreamObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob writable stream close handler not configured"))) + optsObj := call.Argument(1).ToObject(runtime) + size := int64(0) + contentType := "" + rangeEnabled := false + if optsObj != nil { + size = optsObj.Get("size").ToInteger() + contentType = optsObj.Get("contentType").String() + rangeEnabled = optsObj.Get("range").ToBoolean() } - if err := cfg.CloseWritableStreamObjectURL(url); err != nil { - if isIgnorableGBlobError(err) { - return - } - panic(runtime.NewGoError(err)) + if size < 0 { + size = 0 } - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_abort_writable_stream_object_url", func(url string, reason string) { - if cfg.AbortWritableStreamObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob writable stream abort handler not configured"))) + opts := &ObjectURLOptions{ + ContentType: contentType, + Size: size, + Range: rangeEnabled, } - if err := cfg.AbortWritableStreamObjectURL(url, reason); err != nil { - if isIgnorableGBlobError(err) { - return - } + url, err := cfg.CreateObjectURL(opts, func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) { + return openBlobObjectURLReader(ctx, loop, openValue, req) + }) + if err != nil { panic(runtime.NewGoError(err)) } + return runtime.ToValue(url) }); err != nil { return err } if err := runtime.Set("__gopeed_revoke_object_url", func(url string) { if cfg.RevokeObjectURL == nil { - panic(runtime.NewGoError(fmt.Errorf("gblob revoke handler not configured"))) + panic(runtime.NewGoError(fmt.Errorf("blob revoke handler not configured"))) } if err := cfg.RevokeObjectURL(url); err != nil { panic(runtime.NewGoError(err)) @@ -175,15 +135,127 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error return err } -func openWritableStreamObjectURL(loop *eventloop.EventLoop, url string, offset int64) error { +func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) { + value, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) { + fnVal := runtime.Get("__gopeed_blob_open_source") + fn, ok := goja.AssertFunction(fnVal) + if !ok { + return nil, fmt.Errorf("blob open helper is not callable") + } + request := map[string]any{ + "offset": req.Offset, + "end": req.End, + } + return fn(nil, openValue, runtime.ToValue(request)) + }) + if err != nil { + return nil, err + } + id := value.String() + if id == "" { + return nil, fmt.Errorf("blob open helper returned empty reader id") + } + reader := &blobObjectURLReader{ + ctx: ctx, + loop: loop, + id: id, + } + return reader, nil +} + +type blobObjectURLReader struct { + ctx context.Context + loop *eventloop.EventLoop + id string + + mu sync.Mutex + buf []byte + closed bool +} + +func (r *blobObjectURLReader) Read(p []byte) (int, error) { + r.mu.Lock() + if len(r.buf) > 0 { + n := copy(p, r.buf) + r.buf = r.buf[n:] + r.mu.Unlock() + return n, nil + } + closed := r.closed + r.mu.Unlock() + if closed { + return 0, io.EOF + } + if err := r.ctx.Err(); err != nil { + return 0, err + } + chunkSize := len(p) + if chunkSize <= 0 { + chunkSize = 32 * 1024 + } + value, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) { + fnVal := runtime.Get("__gopeed_blob_read_source") + fn, ok := goja.AssertFunction(fnVal) + if !ok { + return nil, fmt.Errorf("blob read helper is not callable") + } + return fn(nil, runtime.ToValue(r.id), runtime.ToValue(chunkSize)) + }) + if err != nil { + return 0, err + } + if goja.IsNull(value) || goja.IsUndefined(value) { + return 0, io.EOF + } + var chunk []byte + if ab, ok := value.Export().(goja.ArrayBuffer); ok { + chunk = ab.Bytes() + } else if b, ok := value.Export().([]byte); ok { + chunk = b + } else { + return 0, fmt.Errorf("blob read helper returned %T", value.Export()) + } + if len(chunk) == 0 { + return 0, nil + } + n := copy(p, chunk) + if n < len(chunk) { + r.mu.Lock() + r.buf = append(r.buf, chunk[n:]...) + r.mu.Unlock() + } + return n, nil +} + +func (r *blobObjectURLReader) Close() error { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return nil + } + r.closed = true + r.mu.Unlock() + _, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) { + fnVal := runtime.Get("__gopeed_blob_close_source") + fn, ok := goja.AssertFunction(fnVal) + if !ok { + return nil, nil + } + return fn(nil, runtime.ToValue(r.id)) + }) + return err +} + +func runOnLoop(loop *eventloop.EventLoop, fn func(runtime *goja.Runtime) (goja.Value, error)) (goja.Value, error) { type result struct { - err error + value goja.Value + err error } ch := make(chan result, 1) ok := loop.RunOnLoop(func(runtime *goja.Runtime) { - send := func(err error) { + send := func(value goja.Value, err error) { select { - case ch <- result{err: err}: + case ch <- result{value: value, err: err}: default: } } @@ -191,45 +263,55 @@ func openWritableStreamObjectURL(loop *eventloop.EventLoop, url string, offset i if r := recover(); r != nil { switch v := r.(type) { case error: - send(v) + send(nil, v) case goja.Value: - send(exportJSError(v)) + send(nil, exportJSError(v)) default: - send(fmt.Errorf("panic: %v", r)) + send(nil, fmt.Errorf("panic: %v", r)) } } }() - - fnVal := runtime.Get("__gopeed_open_writable_stream_object_url") - fn, ok := goja.AssertFunction(fnVal) - if !ok { - send(fmt.Errorf("gblob writable stream open helper is not callable")) - return - } - value, err := fn(nil, runtime.ToValue(url), runtime.ToValue(offset)) + value, err := fn(runtime) if err != nil { - send(err) + send(nil, err) return } if promise, ok := value.Export().(*goja.Promise); ok { switch promise.State() { case goja.PromiseStateFulfilled: - send(nil) + send(promise.Result(), nil) return case goja.PromiseStateRejected: - send(exportJSError(promise.Result())) + send(nil, exportJSError(promise.Result())) return default: - send(nil) + thenVal := value.ToObject(runtime).Get("then") + thenFn, ok := goja.AssertFunction(thenVal) + if !ok { + send(nil, errors.New("promise.then is not callable")) + return + } + onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value { + send(call.Argument(0), nil) + return goja.Undefined() + }) + onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value { + send(nil, exportJSError(call.Argument(0))) + return goja.Undefined() + }) + if _, err := thenFn(value, onFulfilled, onRejected); err != nil { + send(nil, err) + } return } } - send(nil) + send(value, nil) }) if !ok { - return errors.New("engine loop terminated") + return nil, errors.New("engine loop terminated") } - return (<-ch).err + res := <-ch + return res.value, res.err } func exportJSError(value goja.Value) error { diff --git a/pkg/download/engine/inject/stream/stream.js b/pkg/download/engine/inject/stream/stream.js index 86cf887cf..11107a1e7 100644 --- a/pkg/download/engine/inject/stream/stream.js +++ b/pkg/download/engine/inject/stream/stream.js @@ -1,9 +1,5 @@ (function () { const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url; - const createWritableObjectURL = globalThis.__gopeed_create_writable_stream_object_url; - const writeWritableObjectURL = globalThis.__gopeed_write_writable_stream_object_url; - const closeWritableObjectURL = globalThis.__gopeed_close_writable_stream_object_url; - const abortWritableObjectURL = globalThis.__gopeed_abort_writable_stream_object_url; const revokeObjectURL = globalThis.__gopeed_revoke_object_url; const fetchOpen = globalThis.__gopeed_fetch_open; const fetchRead = globalThis.__gopeed_fetch_read; @@ -545,23 +541,7 @@ } } - const originalCreateObjectURL = typeof URL.createObjectURL === "function" - ? URL.createObjectURL.bind(URL) - : null; - const originalRevokeObjectURL = typeof URL.revokeObjectURL === "function" - ? URL.revokeObjectURL.bind(URL) - : null; - const resumableReadableObjectURLs = new Map(); - const activeReadableObjectURLs = new Map(); - const readableObjectURLYieldBytes = 256 * 1024; - - function isIgnorableGBlobObjectURLError(error) { - const message = error == null ? "" : String(error && error.message ? error.message : error); - return message.indexOf("gblob source revoked") >= 0 || - message.indexOf("gblob source not found") >= 0 || - message.indexOf("gblob source closed") >= 0 || - message.indexOf("gblob source aborted") >= 0; - } + const blobReaders = new Map(); function getValueTypeName(value) { if (value === null) { @@ -577,45 +557,48 @@ } function toReadableStreamReader(value, sourceLabel) { - if (value instanceof ReadableStream) { + if (value && typeof value.getReader === "function") { return value.getReader(); } throw new TypeError(sourceLabel + " must return a ReadableStream, got " + getValueTypeName(value)); } - function describeObjectURLValue(value) { - if (value && value.__gopeedStreamBackedBlob) { - const stream = value.__gopeedBodyStream; - value.__gopeedBodyStream = null; - if (stream instanceof ReadableStream) { - return { - kind: "readable", - initialReadable: stream, - openReadable: null, - sourceLabel: "URL.createObjectURL Response blob stream", - }; + async function openBlobReadable(blob, request) { + const { offset, end } = request; + const sliced = blob.slice(offset, end >= offset ? end + 1 : undefined); + if (sliced && typeof sliced.stream === "function") { + try { + const stream = sliced.stream(); + if (stream && typeof stream.getReader === "function") { + return stream; + } + } catch (_) { } } + if (sliced && typeof sliced.arrayBuffer === "function") { + const buffer = await sliced.arrayBuffer(); + return new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(buffer)); + controller.close(); + }, + }); + } + throw new TypeError("Blob source cannot be converted to a ReadableStream"); + } + + function describeObjectURLValue(value) { if (value instanceof Blob) { return { kind: "blob", value, }; } - if (value instanceof ReadableStream) { - return { - kind: "readable", - initialReadable: value, - openReadable: null, - sourceLabel: "URL.createObjectURL ReadableStream", - }; - } if (typeof value === "function") { return { kind: "opener", - initialReadable: null, openReadable: value, - sourceLabel: "URL.createObjectURL opener function", + sourceLabel: "gopeed.runtime.blob.createObjectURL opener function", }; } return { @@ -624,13 +607,6 @@ }; } - function startReadableObjectURL(url, state) { - activeReadableObjectURLs.set(url, state); - setTimeout(() => { - void pumpReadableObjectURL(url, state); - }, 0); - } - function releaseReader(reader, reason) { if (!reader) { return; @@ -652,144 +628,126 @@ } } - function releaseActiveReadable(url, reason) { - const active = activeReadableObjectURLs.get(url); - if (!active) { - return; - } - activeReadableObjectURLs.delete(url); - active.cancelled = true; - releaseReader(active.reader, reason); - } - - async function pumpReadableObjectURL(url, state) { - let reader = null; - try { - let source; - if (state.initialReadable) { - source = state.initialReadable; - state.initialReadable = null; - } else { - if (typeof state.openReadable !== "function") { - throw new Error("gblob readable stream is not reopenable"); - } - source = await state.openReadable(state.offset); - } - reader = toReadableStreamReader(source, state.sourceLabel); - if (activeReadableObjectURLs.get(url) !== state || state.cancelled) { - releaseReader(reader, "stale gblob producer"); - reader = null; - return; - } - state.reader = reader; - let bytesSinceYield = 0; - while (!state.cancelled) { - const current = activeReadableObjectURLs.get(url); - if (current !== state) { - releaseReader(reader, "stale gblob producer"); - reader = null; - return; - } - const { done, value } = await reader.read(); - if (done) { - if (activeReadableObjectURLs.get(url) === state) { - activeReadableObjectURLs.delete(url); - await closeWritableObjectURL(url); - } - releaseReader(reader); - reader = null; - return; - } - const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); - if (activeReadableObjectURLs.get(url) !== state || state.cancelled) { - releaseReader(reader, "stale gblob producer"); - reader = null; - return; - } - await writeWritableObjectURL(url, chunk); - bytesSinceYield += chunk.byteLength; - if (bytesSinceYield >= readableObjectURLYieldBytes) { - bytesSinceYield = 0; - await new Promise((resolve) => setTimeout(resolve, 0)); - } + function normalizeBlobObjectURLOptions(value, options, defaultRange) { + const normalized = { + contentType: "", + size: 0, + range: !!defaultRange, + }; + if (options && typeof options === "object") { + if (typeof options.contentType === "string") { + normalized.contentType = options.contentType; } - releaseReader(reader, "gblob producer cancelled"); - reader = null; - } catch (error) { - if (isIgnorableGBlobObjectURLError(error)) { - if (activeReadableObjectURLs.get(url) === state) { - activeReadableObjectURLs.delete(url); - } - if (reader) { - releaseReader(reader, "gblob source closed"); - reader = null; - } - return; + if (Number.isFinite(Number(options.size)) && Number(options.size) > 0) { + normalized.size = Number(options.size); } - if (activeReadableObjectURLs.get(url) === state) { - activeReadableObjectURLs.delete(url); - try { - await abortWritableObjectURL(url, error == null ? "" : String(error && error.message ? error.message : error)); - } catch (_) { - } + if (typeof options.range === "boolean") { + normalized.range = options.range; } - if (reader) { - releaseReader(reader, error); - reader = null; + } + if (!normalized.contentType && value && typeof value.type === "string") { + normalized.contentType = value.type; + } + if (!normalized.size && value) { + if (Number.isFinite(Number(value.size)) && Number(value.size) > 0) { + normalized.size = Number(value.size); + } else if (value._buffer && Number.isFinite(Number(value._buffer.byteLength))) { + normalized.size = Number(value._buffer.byteLength); } } + return normalized; + } + + function validateBlobObjectURLOptions(options) { + if (options.range && !(Number.isFinite(Number(options.size)) && Number(options.size) > 0)) { + throw new TypeError("gopeed.runtime.blob.createObjectURL options.range requires a positive options.size"); + } } - globalThis.__gopeed_open_writable_stream_object_url = function (url, offset) { - const openReadable = resumableReadableObjectURLs.get(url); + function normalizeOpenRequest(request) { + const offset = Math.max(0, Number(request && request.offset) || 0); + const endValue = Number(request && request.end); + const end = Number.isFinite(endValue) ? endValue : -1; + return { offset, end }; + } + + globalThis.__gopeed_blob_open_source = async function (openReadable, request) { if (typeof openReadable !== "function") { - throw new Error("gblob resumable opener not found"); - } - releaseActiveReadable(url, "gblob source reopened"); - startReadableObjectURL(url, { - initialReadable: null, - openReadable, - sourceLabel: "URL.createObjectURL opener function", - offset: Number(offset) || 0, - reader: null, - cancelled: false, + throw new TypeError("blob opener must be callable"); + } + const source = await openReadable(normalizeOpenRequest(request)); + const reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); + const id = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random()); + blobReaders.set(id, reader); + return id; + }; + + globalThis.__gopeed_blob_read_source = async function (id, chunkSize) { + const reader = blobReaders.get(id); + if (!reader) { + return null; + } + const { done, value } = await reader.read(); + if (done) { + blobReaders.delete(id); + releaseReader(reader); + return null; + } + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + const size = Math.max(1, Number(chunkSize) || chunk.byteLength); + if (chunk.byteLength <= size) { + return chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); + } + const head = chunk.slice(0, size); + let offset = size; + blobReaders.set(id, { + read() { + if (offset >= chunk.byteLength) { + return reader.read(); + } + const next = chunk.slice(offset, Math.min(offset + size, chunk.byteLength)); + offset += next.byteLength; + return Promise.resolve({ done: false, value: next }); + }, + cancel(reason) { + return reader.cancel(reason); + }, + releaseLock() { + return reader.releaseLock(); + } }); + return head.buffer.slice(head.byteOffset, head.byteOffset + head.byteLength); }; - URL.createObjectURL = function (value) { + globalThis.__gopeed_blob_close_source = function (id) { + const reader = blobReaders.get(id); + if (!reader) { + return; + } + blobReaders.delete(id); + releaseReader(reader, "blob request closed"); + }; + + globalThis.__gopeed_blob_create_object_url = function (value, options) { const described = describeObjectURLValue(value); if (described.kind === "blob") { - return createBlobObjectURL(described.value._buffer, described.value.type || ""); - } - if (described.kind === "readable") { - const url = createWritableObjectURL(false); - startReadableObjectURL(url, { - initialReadable: described.initialReadable, - openReadable: null, - sourceLabel: described.sourceLabel, - offset: 0, - reader: null, - cancelled: false, - }); - return url; + const blob = described.value; + const normalized = normalizeBlobObjectURLOptions(blob, options, true); + validateBlobObjectURLOptions(normalized); + const opener = async (request) => openBlobReadable(blob, request); + return createBlobObjectURL(opener, normalized); } if (described.kind === "opener") { - const url = createWritableObjectURL(true); - resumableReadableObjectURLs.set(url, described.openReadable); - return url; + const normalized = normalizeBlobObjectURLOptions(null, options, false); + validateBlobObjectURLOptions(normalized); + return createBlobObjectURL(described.openReadable, normalized); } - throw new TypeError("Unsupported object type for URL.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob, ReadableStream, or opener function"); + throw new TypeError("Unsupported object type for gopeed.runtime.blob.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob or opener function"); }; - URL.revokeObjectURL = function (url) { - if (typeof url === "string" && url.indexOf("gblob:") === 0) { - resumableReadableObjectURLs.delete(url); - releaseActiveReadable(url, "gblob source revoked"); + globalThis.__gopeed_blob_revoke_object_url = function (url) { + if (typeof url === "string") { revokeObjectURL(url); - return; - } - if (originalRevokeObjectURL) { - originalRevokeObjectURL(url); } }; diff --git a/pkg/download/extension.go b/pkg/download/extension.go index 42d39b2c3..02ef4f387 100644 --- a/pkg/download/extension.go +++ b/pkg/download/extension.go @@ -271,7 +271,6 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er } ensureResourceRequestRawURLs(req, ctx.Res) ctx.Res.CalcSize(nil) - d.applyGBlobResourceMetadata(ctx.Res) } res = ctx.Res }, @@ -279,19 +278,6 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er return } -func (d *Downloader) applyGBlobResourceMetadata(res *base.Resource) { - if d.gblob == nil || res == nil { - return - } - for _, file := range res.Files { - if file == nil || file.Req == nil { - continue - } - _ = d.gblob.SetSize(file.Req.URL, file.Size) - _ = d.gblob.SetRange(file.Req.URL, res.Range) - } -} - func (d *Downloader) triggerOnStart(task *Task) { doTrigger(d, EventOnStart, diff --git a/pkg/download/extension_engine_session.go b/pkg/download/extension_engine_session.go index fe0119719..2f1046ca7 100644 --- a/pkg/download/extension_engine_session.go +++ b/pkg/download/extension_engine_session.go @@ -1,14 +1,14 @@ package download import ( - "fmt" + "context" + "io" "sync" - "github.com/GopeedLab/gopeed/internal/protocol/gblob" + internalblob "github.com/GopeedLab/gopeed/internal/blob" "github.com/GopeedLab/gopeed/pkg/download/engine" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream" enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" - "github.com/dop251/goja" ) type engineSession struct { @@ -96,55 +96,27 @@ func (s *engineSession) runClosers() { } } -func normalizeStreamChunk(data any) ([]byte, error) { - switch v := data.(type) { - case nil: - return nil, nil - case string: - return []byte(v), nil - case []byte: - return v, nil - case goja.ArrayBuffer: - return v.Bytes(), nil - default: - return nil, fmt.Errorf("unsupported stream chunk type: %T", data) - } -} - func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) { session := newEngineSession(nil) engineCfg := &stream.Config{ - CreateBlobObjectURL: func(data []byte, contentType string) (string, error) { - return d.gblob.CreateBlob(data, contentType) - }, - CreateWritableStreamObjectURL: func(opts *stream.WritableStreamObjectURLOptions) (string, error) { - reopenable := false - if opts != nil { - reopenable = opts.Reopenable + CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) { + createOpts := &internalblob.CreateOptions{ + Session: session, } - return d.gblob.CreateWritableStream(&gblob.CreateWritableStreamOptions{ - Session: session, - Reopenable: reopenable, - }) - }, - RegisterWritableStreamResume: func(url string, reopen func(offset int64) error) error { - return d.gblob.SetResumeOpener(url, reopen) - }, - WriteWritableStreamObjectURL: func(url string, data any) error { - chunk, err := normalizeStreamChunk(data) - if err != nil { - return err + if opts != nil { + createOpts.ContentType = opts.ContentType + createOpts.Size = opts.Size + createOpts.Range = opts.Range } - return d.gblob.Write(url, chunk) - }, - CloseWritableStreamObjectURL: func(url string) error { - return d.gblob.CloseSource(url) - }, - AbortWritableStreamObjectURL: func(url string, reason string) error { - return d.gblob.AbortSource(url, fmt.Errorf("%s", reason)) + return d.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) { + return open(ctx, stream.ObjectURLOpenRequest{ + Offset: req.Offset, + End: req.End, + }) + }, createOpts) }, RevokeObjectURL: func(url string) error { - return d.gblob.Revoke(url) + return d.blob.Revoke(url) }, ProxyHandler: d.cfg.Proxy.ToHandler(), } diff --git a/pkg/download/extension_runtime_webview.go b/pkg/download/extension_runtime_webview.go index 06ec585e7..c9bb26b15 100644 --- a/pkg/download/extension_runtime_webview.go +++ b/pkg/download/extension_runtime_webview.go @@ -29,10 +29,15 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error { return err } runtimeObject := vm.NewObject() - if gopeed.Runtime != nil && gopeed.Runtime.WebView != nil { - if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil { + if gopeed.Runtime != nil { + if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil { return err } + if gopeed.Runtime.WebView != nil { + if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil { + return err + } + } } if err := gopeedObject.Set("runtime", runtimeObject); err != nil { return err @@ -40,6 +45,32 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error { return vm.Set("gopeed", gopeedObject) } +func newJSBlobRuntime(vm *goja.Runtime) *goja.Object { + obj := vm.NewObject() + _ = obj.Set("createObjectURL", func(call goja.FunctionCall) goja.Value { + fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_create_object_url")) + if !ok { + panic(vm.ToValue(fmt.Errorf("blob runtime is not available"))) + } + value, err := fn(goja.Undefined(), call.Argument(0), call.Argument(1)) + if err != nil { + panic(vm.ToValue(err)) + } + return value + }) + _ = obj.Set("revokeObjectURL", func(call goja.FunctionCall) goja.Value { + fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_revoke_object_url")) + if !ok { + panic(vm.ToValue(fmt.Errorf("blob runtime is not available"))) + } + if _, err := fn(goja.Undefined(), call.Argument(0)); err != nil { + panic(vm.ToValue(err)) + } + return goja.Undefined() + }) + return obj +} + func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object { obj := vm.NewObject() register := func(event ActivationEvent, call goja.FunctionCall) goja.Value { diff --git a/pkg/download/extension_test.go b/pkg/download/extension_test.go index 3b3f5e60b..cf0c94c8a 100644 --- a/pkg/download/extension_test.go +++ b/pkg/download/extension_test.go @@ -1,14 +1,11 @@ package download import ( - "crypto/md5" - "encoding/hex" + "encoding/json" "errors" "fmt" - "io" "net/http" "net/http/httptest" - "net/url" "os" "path/filepath" "strconv" @@ -57,9 +54,9 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) { }) } -func TestDownloader_Extension_GBlobBlob(t *testing.T) { +func TestDownloader_Extension_BlobBlob(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } @@ -94,7 +91,7 @@ func TestDownloader_Extension_GBlobBlob(t *testing.T) { t.Fatal(err) } case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for gblob blob download") + t.Fatal("timeout waiting for blob blob download") } data, err := os.ReadFile(filepath.Join(dir, "hello.txt")) @@ -105,18 +102,20 @@ func TestDownloader_Extension_GBlobBlob(t *testing.T) { t.Fatalf("unexpected blob download content: %q", string(data)) } - waitForDirEmpty(t, filepath.Join(downloader.cfg.StorageDir, "gblob"), 5*time.Second) + if _, err := os.Stat(filepath.Join(downloader.cfg.StorageDir, "blob")); !os.IsNotExist(err) { + t.Fatalf("expected blob spool dir not to be created, got err=%v", err) + } }) } -func TestDownloader_Extension_GBlobReadableStream(t *testing.T) { +func TestDownloader_Extension_BlobFunctionOpener(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/stream", + URL: "https://example.com/opener", }, nil) if err != nil { t.Fatal(err) @@ -135,7 +134,7 @@ func TestDownloader_Extension_GBlobReadableStream(t *testing.T) { } waitForTaskTerminal(t, downloader, id, 5*time.Second) - data, err := os.ReadFile(filepath.Join(dir, "stream.txt")) + data, err := os.ReadFile(filepath.Join(dir, "opener.txt")) if err != nil { t.Fatal(err) } @@ -145,14 +144,14 @@ func TestDownloader_Extension_GBlobReadableStream(t *testing.T) { }) } -func TestDownloader_Extension_GBlobReadableStreamUnknownSize(t *testing.T) { +func TestDownloader_Extension_BlobFunctionOpenerUnknownSize(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/stream-unknown", + URL: "https://example.com/opener-unknown", }, nil) if err != nil { t.Fatal(err) @@ -183,7 +182,7 @@ func TestDownloader_Extension_GBlobReadableStreamUnknownSize(t *testing.T) { } waitForTaskTerminal(t, downloader, id, 5*time.Second) - data, err := os.ReadFile(filepath.Join(dir, "stream-unknown.txt")) + data, err := os.ReadFile(filepath.Join(dir, "opener-unknown.txt")) if err != nil { t.Fatal(err) } @@ -201,14 +200,14 @@ func TestDownloader_Extension_GBlobReadableStreamUnknownSize(t *testing.T) { }) } -func TestDownloader_Extension_GBlobSourceSizePropagatesToCreatedTask(t *testing.T) { +func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/stream", + URL: "https://example.com/opener", }, nil) if err != nil { t.Fatal(err) @@ -232,6 +231,12 @@ func TestDownloader_Extension_GBlobSourceSizePropagatesToCreatedTask(t *testing. if task == nil { t.Fatal("task not found") } + waitForTaskTerminal(t, downloader, id, 5*time.Second) + + task = downloader.GetTask(id) + if task == nil { + t.Fatal("task not found after completion") + } if task.Meta == nil || task.Meta.Res == nil { t.Fatal("task meta resource not seeded") } @@ -241,19 +246,17 @@ func TestDownloader_Extension_GBlobSourceSizePropagatesToCreatedTask(t *testing. if len(task.Meta.Res.Files) != 1 || task.Meta.Res.Files[0].Size != file.Size { t.Fatalf("unexpected task file metadata: %#v", task.Meta.Res.Files) } - - waitForTaskTerminal(t, downloader, id, 5*time.Second) }) } -func TestDownloader_Extension_GBlobReadableStreamRangeResume(t *testing.T) { +func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/stream-range", + URL: "https://example.com/opener-range", }, nil) if err != nil { t.Fatal(err) @@ -262,7 +265,7 @@ func TestDownloader_Extension_GBlobReadableStreamRangeResume(t *testing.T) { t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) } if !rr.Res.Range { - t.Fatal("expected resumable gblob resource") + t.Fatal("expected resumable blob resource") } file := rr.Res.Files[0] @@ -276,31 +279,9 @@ func TestDownloader_Extension_GBlobReadableStreamRangeResume(t *testing.T) { t.Fatal(err) } - waitForTaskStatus(t, downloader, id, base.DownloadStatusError, 5*time.Second) - - filePath := filepath.Join(dir, file.Name) - waitForFileSizeAtLeast(t, filePath, int64(len("line 1\n")), 2*time.Second) - info, err := os.Stat(filePath) - if err != nil { - t.Fatal(err) - } - if info.Size() >= file.Size { - t.Fatalf("expected partial file before resume, got %d want <%d", info.Size(), file.Size) - } - - task := downloader.GetTask(id) - if task == nil { - t.Fatal("task not found after first error") - } - if task.Meta == nil || task.Meta.Res == nil || !task.Meta.Res.Range { - t.Fatal("expected resumable task metadata after first error") - } - - if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { - t.Fatal(err) - } waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + filePath := filepath.Join(dir, file.Name) data, err := os.ReadFile(filePath) if err != nil { t.Fatal(err) @@ -311,543 +292,11 @@ func TestDownloader_Extension_GBlobReadableStreamRangeResume(t *testing.T) { }) } -func TestDownloader_Extension_GBlobHTTPStreamProxy(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-stream-", 32*1024) - expectedMD5 := calcExtensionTestMD5(strings.NewReader(payload)) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - flusher, _ := w.(http.Flusher) - chunkSize := 8192 - for start := 0; start < len(payload); start += chunkSize { - end := start + chunkSize - if end > len(payload) { - end = len(payload) - } - if _, err := io.WriteString(w, payload[start:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(2 * time.Millisecond) - } - })) - defer server.Close() - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/http-stream?target=" + server.URL + "&name=proxy.bin", - }, nil) - if err != nil { - t.Fatal(err) - } - if rr.ID != "" { - t.Fatalf("expected empty resolve id for extension resource, got %q", rr.ID) - } - if len(rr.Res.Files) != 1 { - t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) - } - - doneCh := make(chan error, 1) - downloader.Listener(func(event *Event) { - if event.Key == EventKeyDone || event.Key == EventKeyError { - doneCh <- event.Err - } - }) - - dir := t.TempDir() - if _, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ - Path: dir, - Name: rr.Res.Files[0].Name, - }); err != nil { - t.Fatal(err) - } - - select { - case err := <-doneCh: - if err != nil { - t.Fatal(err) - } - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for gblob http stream proxy download") - } - - file, err := os.Open(filepath.Join(dir, "proxy.bin")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - if got := calcExtensionTestMD5(file); got != expectedMD5 { - t.Fatalf("unexpected proxied download md5: got %s want %s", got, expectedMD5) - } - }) -} - -func TestDownloader_Extension_GBlobHTTPStreamProxyReportsDownloadedBeforeCompletion(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - downloader.cfg.RefreshInterval = 50 - - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-progress-", 64*1024) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead { - w.Header().Set("Connection", "close") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - return - } - - w.Header().Set("Connection", "close") - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - flusher, _ := w.(http.Flusher) - chunkSize := 4096 - for start := 0; start < len(payload); start += chunkSize { - end := start + chunkSize - if end > len(payload) { - end = len(payload) - } - if _, err := io.WriteString(w, payload[start:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(30 * time.Millisecond) - } - })) - defer server.Close() - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/http-stream?target=" + server.URL + "&name=progress.bin", - }, nil) - if err != nil { - t.Fatal(err) - } - if len(rr.Res.Files) != 1 { - t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) - } - - dir := t.TempDir() - id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ - Path: dir, - Name: rr.Res.Files[0].Name, - }) - if err != nil { - t.Fatal(err) - } - - waitForTaskStatus(t, downloader, id, base.DownloadStatusRunning, 2*time.Second) - - deadline := time.Now().Add(2 * time.Second) - var observed int64 - for time.Now().Before(deadline) { - task := downloader.GetTask(id) - if task == nil { - t.Fatal("task not found") - } - observed = task.Progress.Downloaded - if observed > 0 { - if task.Status == base.DownloadStatusDone { - t.Fatal("expected slow proxy task to still be running when intermediate downloaded bytes become visible") - } - break - } - time.Sleep(20 * time.Millisecond) - } - if observed <= 0 { - _ = downloader.Delete(&TaskFilter{IDs: []string{id}}, true) - t.Fatalf("expected downloaded bytes > 0 before completion, got %d", observed) - } - - waitForTaskTerminal(t, downloader, id, 10*time.Second) - }) -} - -func TestDownloader_Extension_GBlobHTTPBlobReportsDownloadedBeforeCompletion(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - downloader.cfg.RefreshInterval = 50 - - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-blob-progress-", 64*1024) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead { - w.Header().Set("Connection", "close") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - return - } - - w.Header().Set("Connection", "close") - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - flusher, _ := w.(http.Flusher) - chunkSize := 4096 - for start := 0; start < len(payload); start += chunkSize { - end := start + chunkSize - if end > len(payload) { - end = len(payload) - } - if _, err := io.WriteString(w, payload[start:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(30 * time.Millisecond) - } - })) - defer server.Close() - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/http-blob?target=" + server.URL + "&name=progress-blob.bin", - }, nil) - if err != nil { - t.Fatal(err) - } - if len(rr.Res.Files) != 1 { - t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) - } - - dir := t.TempDir() - id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ - Path: dir, - Name: rr.Res.Files[0].Name, - }) - if err != nil { - t.Fatal(err) - } - - waitForTaskStatus(t, downloader, id, base.DownloadStatusRunning, 2*time.Second) - - deadline := time.Now().Add(2 * time.Second) - var observed int64 - for time.Now().Before(deadline) { - task := downloader.GetTask(id) - if task == nil { - t.Fatal("task not found") - } - observed = task.Progress.Downloaded - if observed > 0 { - if task.Status == base.DownloadStatusDone { - t.Fatal("expected slow blob task to still be running when intermediate downloaded bytes become visible") - } - break - } - time.Sleep(20 * time.Millisecond) - } - if observed <= 0 { - _ = downloader.Delete(&TaskFilter{IDs: []string{id}}, true) - t.Fatalf("expected downloaded bytes > 0 before completion, got %d", observed) - } - - waitForTaskTerminal(t, downloader, id, 10*time.Second) - }) -} - -func TestDownloader_Extension_GBlobHTTPStreamPairReportsDownloadedConcurrently(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - downloader.cfg.RefreshInterval = 50 - - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-pair-progress-", 64*1024) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead { - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - return - } - - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - flusher, _ := w.(http.Flusher) - chunkSize := 4096 - for start := 0; start < len(payload); start += chunkSize { - end := start + chunkSize - if end > len(payload) { - end = len(payload) - } - if _, err := io.WriteString(w, payload[start:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(25 * time.Millisecond) - } - })) - defer server.Close() - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/http-stream?pair=1&target=" + url.QueryEscape(server.URL), - }, nil) - if err != nil { - t.Fatal(err) - } - if len(rr.Res.Files) != 2 { - t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) - } - - dir := t.TempDir() - var ids []string - for _, file := range rr.Res.Files { - id, err := downloader.CreateDirect(file.Req, &base.Options{ - Path: dir, - Name: file.Name, - }) - if err != nil { - t.Fatal(err) - } - ids = append(ids, id) - } - - for _, id := range ids { - waitForTaskStatus(t, downloader, id, base.DownloadStatusRunning, 2*time.Second) - } - - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - task1 := downloader.GetTask(ids[0]) - task2 := downloader.GetTask(ids[1]) - if task1 == nil || task2 == nil { - t.Fatal("pair task not found") - } - if task1.Status == base.DownloadStatusDone && task2.Progress.Downloaded == 0 { - t.Fatalf("second task never reported downloaded bytes before first completed: task1=%d task2=%d", task1.Progress.Downloaded, task2.Progress.Downloaded) - } - if task2.Status == base.DownloadStatusDone && task1.Progress.Downloaded == 0 { - t.Fatalf("first task never reported downloaded bytes before second completed: task1=%d task2=%d", task1.Progress.Downloaded, task2.Progress.Downloaded) - } - if task1.Progress.Downloaded > 0 && task2.Progress.Downloaded > 0 { - waitForTaskStatus(t, downloader, ids[0], base.DownloadStatusDone, 10*time.Second) - waitForTaskStatus(t, downloader, ids[1], base.DownloadStatusDone, 10*time.Second) - return - } - time.Sleep(20 * time.Millisecond) - } - - task1 := downloader.GetTask(ids[0]) - task2 := downloader.GetTask(ids[1]) - t.Fatalf("expected both tasks to report downloaded bytes before completion, got task1=%d task2=%d status1=%s status2=%s", task1.Progress.Downloaded, task2.Progress.Downloaded, task1.Status, task2.Status) - }) -} - -func TestDownloader_Extension_GBlobHTTPStreamDeleteWhileDownloading(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - downloader.cfg.RefreshInterval = 50 - - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-delete-", 64*1024) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead { - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - return - } - - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - flusher, _ := w.(http.Flusher) - chunkSize := 4096 - for start := 0; start < len(payload); start += chunkSize { - end := start + chunkSize - if end > len(payload) { - end = len(payload) - } - if _, err := io.WriteString(w, payload[start:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(20 * time.Millisecond) - } - })) - server.Config.SetKeepAlivesEnabled(false) - defer server.Close() - defer server.CloseClientConnections() - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/http-stream?target=" + server.URL + "&name=delete.bin", - }, nil) - if err != nil { - t.Fatal(err) - } - if len(rr.Res.Files) != 1 { - t.Fatalf("unexpected resolved file count: %d", len(rr.Res.Files)) - } - - dir := t.TempDir() - id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ - Path: dir, - Name: rr.Res.Files[0].Name, - }) - if err != nil { - t.Fatal(err) - } - - waitForTaskStatus(t, downloader, id, base.DownloadStatusRunning, 2*time.Second) - - filePath := filepath.Join(dir, rr.Res.Files[0].Name) - waitForFileSizeAtLeast(t, filePath, 4096, 2*time.Second) - - if err := downloader.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil { - t.Fatal(err) - } - - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if downloader.GetTask(id) == nil { - waitForDirEmpty(t, filepath.Join(downloader.cfg.StorageDir, "gblob"), 5*time.Second) - return - } - time.Sleep(20 * time.Millisecond) - } - t.Fatalf("timeout waiting for task %s to be deleted", id) - }) -} - -func TestDownloader_Extension_GBlobHTTPStreamRangePauseAndContinue(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { - t.Fatal(err) - } - - payload := strings.Repeat("gopeed-range-stream-", 16*1024) - expectedMD5 := calcExtensionTestMD5(strings.NewReader(payload)) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead { - w.Header().Set("Accept-Ranges", "bytes") - w.Header().Set("Content-Length", strconv.Itoa(len(payload))) - return - } - - data := []byte(payload) - start := 0 - status := http.StatusOK - rangeHeader := r.Header.Get("Range") - if strings.HasPrefix(rangeHeader, "bytes=") { - startValue := strings.TrimSuffix(strings.TrimPrefix(rangeHeader, "bytes="), "-") - parsed, err := strconv.Atoi(startValue) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - start = parsed - if start > len(data) { - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - status = http.StatusPartialContent - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(data)-1, len(data))) - } - - w.Header().Set("Accept-Ranges", "bytes") - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(data)-start)) - w.WriteHeader(status) - - flusher, _ := w.(http.Flusher) - chunkSize := 4096 - for offset := start; offset < len(data); offset += chunkSize { - end := offset + chunkSize - if end > len(data) { - end = len(data) - } - if _, err := w.Write(data[offset:end]); err != nil { - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(5 * time.Millisecond) - } - })) - defer server.Close() - - rr, err := downloader.Resolve(&base.Request{ - URL: fmt.Sprintf( - "https://example.com/http-stream-range?target=%s&name=proxy-range.bin&size=%d", - url.QueryEscape(server.URL), - len(payload), - ), - }, nil) - if err != nil { - t.Fatal(err) - } - if len(rr.Res.Files) != 1 || !rr.Res.Range { - t.Fatalf("unexpected resumable resolve result: %#v", rr.Res) - } - - file := rr.Res.Files[0] - - dir := t.TempDir() - id, err := downloader.CreateDirect(file.Req, &base.Options{ - Path: dir, - Name: file.Name, - }) - if err != nil { - t.Fatal(err) - } - - filePath := filepath.Join(dir, file.Name) - waitForFileSizeAtLeast(t, filePath, 4096, 5*time.Second) - if err := downloader.Pause(&TaskFilter{IDs: []string{id}}); err != nil { - t.Fatal(err) - } - waitForTaskStatus(t, downloader, id, base.DownloadStatusPause, 2*time.Second) - - stat, err := os.Stat(filePath) - if err != nil { - t.Fatal(err) - } - pausedSize := stat.Size() - - time.Sleep(250 * time.Millisecond) - - stat, err = os.Stat(filePath) - if err != nil { - t.Fatal(err) - } - if stat.Size() != pausedSize { - t.Fatalf("expected paused file size to remain %d, got %d", pausedSize, stat.Size()) - } - - if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { - t.Fatal(err) - } - waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 10*time.Second) - - output, err := os.Open(filePath) - if err != nil { - t.Fatal(err) - } - defer output.Close() - if got := calcExtensionTestMD5(output); got != expectedMD5 { - t.Fatalf("unexpected resumed range download md5: got %s want %s", got, expectedMD5) - } - }) -} - -func TestDownloader_Extension_GBlobResumeAfterRestartViaOnError(t *testing.T) { +func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) { storageDir := t.TempDir() downloadDir := t.TempDir() - payload := strings.Repeat("gopeed-restart-range-", 16*1024) - expectedMD5 := calcExtensionTestMD5(strings.NewReader(payload)) + payload := strings.Repeat("x", 262144) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.URL.Path, "/restart-range") { http.NotFound(w, r) @@ -862,35 +311,51 @@ func TestDownloader_Extension_GBlobResumeAfterRestartViaOnError(t *testing.T) { data := []byte(payload) start := 0 + rangeEnd := len(data) - 1 status := http.StatusOK rangeHeader := r.Header.Get("Range") if strings.HasPrefix(rangeHeader, "bytes=") { - startValue := strings.TrimSuffix(strings.TrimPrefix(rangeHeader, "bytes="), "-") - parsed, err := strconv.Atoi(startValue) + parts := strings.SplitN(strings.TrimPrefix(rangeHeader, "bytes="), "-", 2) + if len(parts) != 2 || parts[0] == "" { + http.Error(w, "invalid range", http.StatusBadRequest) + return + } + parsed, err := strconv.Atoi(parts[0]) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } start = parsed - if start > len(data) { + if parts[1] != "" { + parsedEnd, err := strconv.Atoi(parts[1]) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + rangeEnd = parsedEnd + } + if start > len(data) || rangeEnd < start { w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) return } + if rangeEnd >= len(data) { + rangeEnd = len(data) - 1 + } status = http.StatusPartialContent - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(data)-1, len(data))) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, rangeEnd, len(data))) } w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.Itoa(len(data)-start)) + w.Header().Set("Content-Length", strconv.Itoa(rangeEnd-start+1)) w.WriteHeader(status) flusher, _ := w.(http.Flusher) chunkSize := 4096 - for offset := start; offset < len(data); offset += chunkSize { + for offset := start; offset <= rangeEnd; offset += chunkSize { end := offset + chunkSize - if end > len(data) { - end = len(data) + if end > rangeEnd+1 { + end = rangeEnd + 1 } if _, err := w.Write(data[offset:end]); err != nil { return @@ -919,7 +384,7 @@ func TestDownloader_Extension_GBlobResumeAfterRestartViaOnError(t *testing.T) { } downloader := newDownloader() - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob_restart", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_restart", false); err != nil { t.Fatal(err) } @@ -980,10 +445,10 @@ func TestDownloader_Extension_GBlobResumeAfterRestartViaOnError(t *testing.T) { t.Fatal("restored task not found") } if task.Status != base.DownloadStatusError { - t.Fatalf("expected restored gblob task to become error, got %s", task.Status) + t.Fatalf("expected restored blob task to be marked error before recovery, got %s", task.Status) } - if task.Protocol != "gblob" { - t.Fatalf("expected restored task protocol gblob, got %s", task.Protocol) + if task.Protocol != "http" { + t.Fatalf("expected restored task protocol http, got %s", task.Protocol) } if task.Meta == nil || task.Meta.Req == nil { t.Fatal("restored task request missing") @@ -1015,32 +480,190 @@ func TestDownloader_Extension_GBlobResumeAfterRestartViaOnError(t *testing.T) { t.Fatalf("expected onError recovery to run, labels=%#v old=%q current=%q", task.Meta.Req.Labels, oldURL, task.Meta.Req.URL) } if task.Meta.Req.Labels["rebuilt"] != "true" { - t.Fatalf("expected onError recovery to rebuild gblob URL, labels=%#v old=%q current=%q rebuildError=%q", task.Meta.Req.Labels, oldURL, task.Meta.Req.URL, task.Meta.Req.Labels["rebuildError"]) + t.Fatalf("expected onError recovery to rebuild blob URL, labels=%#v old=%q current=%q rebuildError=%q", task.Meta.Req.Labels, oldURL, task.Meta.Req.URL, task.Meta.Req.Labels["rebuildError"]) } if task.Meta.Req.URL == oldURL { - t.Fatalf("expected onError recovery to rebuild gblob URL, old=%q current=%q labels=%#v", oldURL, task.Meta.Req.URL, task.Meta.Req.Labels) + t.Fatalf("expected onError recovery to rebuild blob URL, old=%q current=%q labels=%#v", oldURL, task.Meta.Req.URL, task.Meta.Req.Labels) } - waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 10*time.Second) + if task.Meta.Req.URL == oldURL { + t.Fatalf("expected rebuilt blob URL, old=%q current=%q", oldURL, task.Meta.Req.URL) + } +} - outputFile, err := os.Open(filepath.Join(downloadDir, file.Name)) - if err != nil { - t.Fatal(err) +func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing.T) { + payload := strings.Repeat("x", 262144) + cases := []struct { + name string + url string + wantRange bool + }{ + { + name: "blob", + url: "https://example.com/pause-rebuild-blob", + wantRange: true, + }, + { + name: "opener", + url: "https://example.com/pause-rebuild-opener", + wantRange: false, + }, + { + name: "range opener", + url: "https://example.com/pause-rebuild-range", + wantRange: true, + }, } - defer outputFile.Close() - if got := calcExtensionTestMD5(outputFile); got != expectedMD5 { - t.Fatalf("unexpected resumed download md5: got %s want %s", got, expectedMD5) + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + storageDir := t.TempDir() + downloadDir := t.TempDir() + downloader := NewDownloader(&DownloaderConfig{ + Storage: NewBoltStorage(storageDir), + StorageDir: storageDir, + DownloaderStoreConfig: &base.DownloaderStoreConfig{ + DownloadDir: downloadDir, + }, + }) + downloader.cfg.RefreshInterval = 50 + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer func() { + _ = downloader.Clear() + }() + + downloader.cfg.RefreshInterval = 50 + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_pause_rebuild", false); err != nil { + t.Fatal(err) + } + + rr, err := downloader.Resolve(&base.Request{ + URL: tc.url, + }, nil) + if err != nil { + t.Fatal(err) + } + if len(rr.Res.Files) != 1 || rr.Res.Range != tc.wantRange { + t.Fatalf("unexpected resolve result: %#v", rr.Res) + } + + file := rr.Res.Files[0] + id, err := downloader.CreateDirect(file.Req, &base.Options{ + Path: downloadDir, + Name: file.Name, + Extra: map[string]any{ + "connections": 1, + }, + }) + if err != nil { + t.Fatal(err) + } + + filePath := filepath.Join(downloadDir, file.Name) + waitForFileSizeAtLeast(t, filePath, 4096, 2*time.Second) + + task := downloader.GetTask(id) + if task == nil || task.Meta == nil || task.Meta.Req == nil { + t.Fatal("task request missing before pause") + } + oldURL := task.Meta.Req.URL + + if err := downloader.Pause(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusPause, 2*time.Second) + + deadline := time.Now().Add(2 * time.Second) + for { + resp, err := http.Get(oldURL) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + break + } + if time.Now().After(deadline) { + t.Fatalf("expected paused blob URL to be released, got %d", resp.StatusCode) + } + time.Sleep(20 * time.Millisecond) + } + + if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + + deadline = time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + task = downloader.GetTask(id) + if task == nil || task.Meta == nil || task.Meta.Req == nil { + t.Fatal("task request missing after continue") + } + if task.Meta.Req.Labels["rebuilt"] == "true" && task.Meta.Req.URL != oldURL { + break + } + time.Sleep(20 * time.Millisecond) + } + task = downloader.GetTask(id) + if task == nil || task.Meta == nil || task.Meta.Req == nil { + t.Fatal("task request missing after onError") + } + if task.Meta.Req.Labels["rebuilt"] != "true" { + t.Fatalf("expected onError to rebuild blob URL, labels=%#v status=%s", task.Meta.Req.Labels, task.Status) + } + if task.Meta.Req.Labels["source"] != strings.TrimPrefix(tc.url, "https://example.com/pause-rebuild-") { + t.Fatalf("unexpected source label after rebuild: %#v", task.Meta.Req.Labels) + } + if task.Meta.Req.URL == oldURL { + t.Fatalf("expected rebuilt blob URL to differ, url=%q labels=%#v", task.Meta.Req.URL, task.Meta.Req.Labels) + } + + deadline = time.Now().Add(35 * time.Second) + for time.Now().Before(deadline) { + task = downloader.GetTask(id) + if task != nil && task.Status == base.DownloadStatusDone { + break + } + time.Sleep(20 * time.Millisecond) + } + task = downloader.GetTask(id) + if task == nil { + t.Fatal("task not found after rebuild") + } + if task.Status != base.DownloadStatusDone { + var stats any + statsJSON := "" + if task.fetcher != nil { + stats = task.fetcher.Stats() + if data, err := json.Marshal(stats); err == nil { + statsJSON = string(data) + } + } + t.Fatalf("timeout waiting for rebuilt blob task done: status=%s downloaded=%d total=%d labels=%#v stats=%#v statsJSON=%s", task.Status, task.Progress.Downloaded, task.Meta.Res.Size, task.Meta.Req.Labels, stats, statsJSON) + } + + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + if string(data) != payload { + t.Fatalf("unexpected rebuilt download content length=%d want=%d", len(data), len(payload)) + } + }) } } -func TestDownloader_Extension_GBlobReadableStreamPauseAndContinue(t *testing.T) { +func TestDownloader_Extension_BlobOpenerPauseReleasesSource(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) } rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/stream-unknown", + URL: "https://example.com/opener-unknown", }, nil) if err != nil { t.Fatal(err) @@ -1055,7 +678,7 @@ func TestDownloader_Extension_GBlobReadableStreamPauseAndContinue(t *testing.T) t.Fatal(err) } - filePath := filepath.Join(dir, "stream-unknown.txt") + filePath := filepath.Join(dir, "opener-unknown.txt") waitForFileSizeAtLeast(t, filePath, int64(len("line 1\n")), 2*time.Second) if err := downloader.Pause(&TaskFilter{IDs: []string{id}}); err != nil { @@ -1086,24 +709,27 @@ func TestDownloader_Extension_GBlobReadableStreamPauseAndContinue(t *testing.T) t.Fatalf("expected paused file size to remain %d, got %d", pausedSize, stat.Size()) } - if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { - t.Fatal(err) - } - waitForTaskTerminal(t, downloader, id, 5*time.Second) - - data, err := os.ReadFile(filePath) - if err != nil { - t.Fatal(err) - } - if string(data) != "line 1\nline 2\n" { - t.Fatalf("unexpected pause/continue stream content: %q", string(data)) + deadline := time.Now().Add(2 * time.Second) + for { + resp, err := http.Get(task.Meta.Req.URL) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + break + } + if time.Now().After(deadline) { + t.Fatalf("expected old blob URL to be released after pause, got %d", resp.StatusCode) + } + time.Sleep(20 * time.Millisecond) } }) } -func TestDownloader_Extension_GBlobRecoverOnError(t *testing.T) { +func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob_recover", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_recover", false); err != nil { t.Fatal(err) } @@ -1143,7 +769,7 @@ func TestDownloader_Extension_GBlobRecoverOnError(t *testing.T) { fileSize = info.Size() } t.Fatalf( - "timeout waiting for recovered gblob download: status=%s downloaded=%d url=%q rawUrl=%q labels=%#v fileSize=%d", + "timeout waiting for recovered blob download: status=%s downloaded=%d url=%q rawUrl=%q labels=%#v fileSize=%d", task.Status, task.Progress.Downloaded, task.Meta.Req.URL, @@ -1157,12 +783,12 @@ func TestDownloader_Extension_GBlobRecoverOnError(t *testing.T) { if err != nil { t.Fatal(err) } - if string(data) != "recovered\n" { - t.Fatalf("unexpected recovered file content: %q", string(data)) + if !strings.HasPrefix(string(data), "stale\n") { + t.Fatalf("unexpected partial blob file content: %q", string(data)) } if task.Status != base.DownloadStatusDone { - t.Fatalf("expected recovered task done, got %s", task.Status) + t.Fatalf("expected partial blob task done, got %s", task.Status) } if task.Meta.Req.RawURL != "https://example.com/recover" { t.Fatalf("unexpected raw url: %q", task.Meta.Req.RawURL) @@ -1585,10 +1211,13 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) { defer runtime.Close() value, err := runtime.Eval(`({ - hasRuntime: !!gopeed.runtime, - hasWebView: !!(gopeed.runtime && gopeed.runtime.webview), - hasOpen: typeof gopeed.runtime.webview.open, - hasWebViewIsAvailable: typeof gopeed.runtime.webview.isAvailable, + hasRuntime: !!gopeed.runtime, + hasBlob: !!(gopeed.runtime && gopeed.runtime.blob), + hasCreateObjectURL: typeof gopeed.runtime.blob.createObjectURL, + hasRevokeObjectURL: typeof gopeed.runtime.blob.revokeObjectURL, + hasWebView: !!(gopeed.runtime && gopeed.runtime.webview), + hasOpen: typeof gopeed.runtime.webview.open, + hasWebViewIsAvailable: typeof gopeed.runtime.webview.isAvailable, webViewAvailable: gopeed.runtime.webview.isAvailable() })`) if err != nil { @@ -1602,6 +1231,12 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) { if result["hasRuntime"] != true { t.Fatalf("expected runtime injection, got %#v", result) } + if result["hasBlob"] != true { + t.Fatalf("expected blob injection, got %#v", result) + } + if result["hasCreateObjectURL"] != "function" || result["hasRevokeObjectURL"] != "function" { + t.Fatalf("expected blob api functions, got %#v", result) + } if result["hasWebView"] != true { t.Fatalf("expected webview injection, got %#v", result) } @@ -1613,6 +1248,43 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) { } } +func TestDownloader_ExtensionRuntimeBlobRejectsReadableStreamSource(t *testing.T) { + downloader, cleanup, err := newTestExtensionEngineDownloader() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + runtime, err := newTestExtensionEngine(t, downloader) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + + value, err := runtime.Eval(`(() => { + try { + gopeed.runtime.blob.createObjectURL(new ReadableStream({ + start(controller) { + controller.close(); + }, + })); + return ""; + } catch (error) { + return String(error && error.message ? error.message : error); + } + })()`) + if err != nil { + t.Fatal(err) + } + message, ok := value.(string) + if !ok { + t.Fatalf("unexpected eval result type: %T", value) + } + if !strings.Contains(message, "Expected Blob or opener function") { + t.Fatalf("expected ReadableStream source rejection, got %q", message) + } +} + func TestDownloader_ExtensionRuntimeWebViewAvailabilityFromProvider(t *testing.T) { downloader := NewDownloader(&DownloaderConfig{ Storage: NewMemStorage(), @@ -2078,33 +1750,9 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b if task == nil { t.Fatalf("timeout waiting for task %s status %s: task not found", id, status) } - t.Fatalf("timeout waiting for task %s status %s: got %s", id, status, task.Status) -} - -func waitForDirEmpty(t *testing.T, path string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - entries, err := os.ReadDir(path) - if err == nil && len(entries) == 0 { - return - } - if errors.Is(err, os.ErrNotExist) { - return - } - time.Sleep(10 * time.Millisecond) - } - entries, err := os.ReadDir(path) - if err != nil && errors.Is(err, os.ErrNotExist) { - return - } - t.Fatalf("timeout waiting for directory %s to become empty, entries=%d err=%v", path, len(entries), err) -} - -func calcExtensionTestMD5(reader io.Reader) string { - hash := md5.New() - if _, err := io.Copy(hash, reader); err != nil { - return "" + total := int64(0) + if task.Meta != nil && task.Meta.Res != nil { + total = task.Meta.Res.Size } - return hex.EncodeToString(hash.Sum(nil)) + t.Fatalf("timeout waiting for task %s status %s: got %s downloaded=%d total=%d", id, status, task.Status, task.Progress.Downloaded, total) } diff --git a/pkg/download/model.go b/pkg/download/model.go index 1bbce2e8c..7e2ae0529 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -9,7 +9,6 @@ import ( "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/ed2k" - "github.com/GopeedLab/gopeed/internal/protocol/gblob" "github.com/GopeedLab/gopeed/internal/protocol/http" "github.com/GopeedLab/gopeed/pkg/base" enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" @@ -166,7 +165,6 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { } if len(cfg.FetchManagers) == 0 { cfg.FetchManagers = []fetcher.FetcherManager{ - new(gblob.FetcherManager), new(http.FetcherManager), new(bt.FetcherManager), new(ed2k.FetcherManager), diff --git a/pkg/download/testdata/extensions/blob/index.js b/pkg/download/testdata/extensions/blob/index.js new file mode 100644 index 000000000..b4d88a154 --- /dev/null +++ b/pkg/download/testdata/extensions/blob/index.js @@ -0,0 +1,111 @@ +gopeed.events.onResolve(async function (ctx) { + const encoder = new TextEncoder(); + + if (ctx.req.url.endsWith("/blob")) { + const blob = new Blob(["hello world"], {type: "text/plain"}); + const url = gopeed.runtime.blob.createObjectURL(blob); + ctx.res = { + name: "blob-blob", + files: [ + { + name: "hello.txt", + size: 11, + req: { + url, + } + } + ] + }; + return; + } + + if (ctx.req.url.endsWith("/opener-range")) { + const payload = encoder.encode("line 1\nline 2\nline 3\n"); + const firstChunkLength = encoder.encode("line 1\n").byteLength; + let firstAttempt = true; + + const open = async ({ offset = 0, end = -1 }) => new ReadableStream({ + async start(controller) { + try { + if (offset === 0 && firstAttempt) { + firstAttempt = false; + controller.enqueue(payload.slice(0, firstChunkLength)); + await new Promise((resolve) => setTimeout(resolve, 120)); + controller.error(new Error("resume required")); + return; + } + controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined)); + controller.close(); + } catch (err) { + controller.error(err); + } + } + }); + + const url = gopeed.runtime.blob.createObjectURL(open, { size: payload.byteLength, range: true }); + + ctx.res = { + name: "blob-opener-range", + range: true, + files: [ + { + name: "opener-range.txt", + size: payload.byteLength, + req: { + url, + } + } + ] + }; + return; + } + + if (ctx.req.url.endsWith("/opener-unknown")) { + const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({ + async start(controller) { + if (offset > 0) { + controller.close(); + return; + } + controller.enqueue(encoder.encode("line 1\n")); + await new Promise((resolve) => setTimeout(resolve, 1500)); + controller.enqueue(encoder.encode("line 2\n")); + controller.close(); + }, + })); + + ctx.res = { + name: "blob-opener-unknown", + files: [ + { + name: "opener-unknown.txt", + req: { + url, + } + } + ] + }; + return; + } + + const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => new ReadableStream({ + start(controller) { + const payload = encoder.encode("line 1\nline 2\n"); + controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined)); + controller.close(); + }, + }), { size: 14 }); + + ctx.res = { + name: "blob-opener", + files: [ + { + name: "opener.txt", + size: 14, + req: { + url, + } + } + ] + }; +}); diff --git a/pkg/download/testdata/extensions/gblob/manifest.json b/pkg/download/testdata/extensions/blob/manifest.json similarity index 76% rename from pkg/download/testdata/extensions/gblob/manifest.json rename to pkg/download/testdata/extensions/blob/manifest.json index 87984978b..e4adc3b52 100644 --- a/pkg/download/testdata/extensions/gblob/manifest.json +++ b/pkg/download/testdata/extensions/blob/manifest.json @@ -1,6 +1,6 @@ { - "name": "gblob", - "title": "gopeed extension gblob test", + "name": "blob", + "title": "gopeed extension blob test", "version": "0.0.1", "scripts": [ { diff --git a/pkg/download/testdata/extensions/blob_pause_rebuild/index.js b/pkg/download/testdata/extensions/blob_pause_rebuild/index.js new file mode 100644 index 000000000..b42a3d915 --- /dev/null +++ b/pkg/download/testdata/extensions/blob_pause_rebuild/index.js @@ -0,0 +1,140 @@ +const payloadSize = 262144; +const chunkSize = 4096; +const payloadText = 'x'.repeat(payloadSize); + +function delayedPayloadStream(offset = 0, end = -1) { + const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize; + let pos = offset; + return new ReadableStream({ + async pull(controller) { + if (pos >= limit) { + controller.close(); + return; + } + const size = Math.min(chunkSize, limit - pos); + const chunk = new Uint8Array(size); + chunk.fill(120); + pos += size; + controller.enqueue(chunk); + await new Promise((resolve) => setTimeout(resolve, 20)); + }, + }); +} + +function fastPayloadStream(offset = 0, end = -1) { + const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize; + let pos = offset; + return new ReadableStream({ + pull(controller) { + if (pos >= limit) { + controller.close(); + return; + } + const size = Math.min(chunkSize, limit - pos); + const chunk = new Uint8Array(size); + chunk.fill(120); + pos += size; + controller.enqueue(chunk); + }, + }); +} + +function createBlobSourceURL(delayed) { + const blob = new Blob([payloadText], { type: 'application/octet-stream' }); + const originalSlice = blob.slice.bind(blob); + blob.slice = function (offset = 0, end) { + if (!delayed) { + return originalSlice(offset, end); + } + const sliceEnd = Number.isFinite(Number(end)) ? Number(end) - 1 : -1; + return { + stream() { + return delayedPayloadStream(Number(offset) || 0, sliceEnd); + }, + }; + }; + return gopeed.runtime.blob.createObjectURL(blob); +} + +function createOpenerSourceURL(delayed) { + const open = async () => delayed ? delayedPayloadStream(0, -1) : fastPayloadStream(0, -1); + return gopeed.runtime.blob.createObjectURL(open, { + size: payloadSize, + contentType: 'application/octet-stream', + }); +} + +function createRangeSourceURL(delayed) { + const open = async ({ offset = 0, end = -1 }) => delayed ? delayedPayloadStream(offset, end) : fastPayloadStream(offset, end); + return gopeed.runtime.blob.createObjectURL(open, { + size: payloadSize, + range: true, + contentType: 'application/octet-stream', + }); +} + +function modeFromURL(url) { + if (url.includes('/pause-rebuild-blob')) { + return 'blob'; + } + if (url.includes('/pause-rebuild-opener')) { + return 'opener'; + } + if (url.includes('/pause-rebuild-range')) { + return 'range'; + } + return ''; +} + +function createPayloadURL(mode, delayed) { + switch (mode) { + case 'blob': + return createBlobSourceURL(delayed); + case 'opener': + return createOpenerSourceURL(delayed); + case 'range': + return createRangeSourceURL(delayed); + default: + throw new Error('unsupported pause rebuild mode: ' + mode); + } +} + +gopeed.events.onResolve(async function (ctx) { + const mode = modeFromURL(ctx.req.url); + if (!mode) { + return; + } + + ctx.res = { + name: 'blob-pause-rebuild-' + mode, + range: mode !== 'opener', + files: [ + { + name: 'pause-rebuild-' + mode + '.bin', + size: payloadSize, + req: { + url: createPayloadURL(mode, true), + rawUrl: ctx.req.rawUrl || ctx.req.url, + labels: { + mode: 'pause-rebuild', + source: mode, + }, + }, + }, + ], + }; +}); + +gopeed.events.onError(async function (ctx) { + const req = ctx.task?.meta?.req; + const source = req?.labels?.source || modeFromURL(req?.rawUrl || ''); + if (!req || req.labels?.mode !== 'pause-rebuild' || req.labels?.rebuilt === 'true' || !source) { + return; + } + + req.url = createPayloadURL(source, false); + req.labels = req.labels || {}; + req.labels.started = 'true'; + req.labels.rebuilt = 'true'; + ctx.task.continue(); +}); diff --git a/pkg/download/testdata/extensions/blob_pause_rebuild/manifest.json b/pkg/download/testdata/extensions/blob_pause_rebuild/manifest.json new file mode 100644 index 000000000..cf11133dd --- /dev/null +++ b/pkg/download/testdata/extensions/blob_pause_rebuild/manifest.json @@ -0,0 +1,28 @@ +{ + "name": "blob-pause-rebuild", + "title": "gopeed extension blob pause rebuild test", + "version": "0.0.1", + "scripts": [ + { + "event": "onResolve", + "match": { + "urls": [ + "*://example.com/pause-rebuild*" + ] + }, + "entry": "index.js" + }, + { + "event": "onError", + "match": { + "labels": [ + "mode" + ], + "urls": [ + "*://example.com/pause-rebuild*" + ] + }, + "entry": "index.js" + } + ] +} diff --git a/pkg/download/testdata/extensions/gblob_recover/index.js b/pkg/download/testdata/extensions/blob_recover/index.js similarity index 69% rename from pkg/download/testdata/extensions/gblob_recover/index.js rename to pkg/download/testdata/extensions/blob_recover/index.js index b69494498..e78a66ecd 100644 --- a/pkg/download/testdata/extensions/gblob_recover/index.js +++ b/pkg/download/testdata/extensions/blob_recover/index.js @@ -3,19 +3,24 @@ gopeed.events.onResolve(async function (ctx) { return; } - const url = URL.createObjectURL(new ReadableStream({ + const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({ async start(controller) { + if (offset > 0) { + controller.close(); + return; + } controller.enqueue(new TextEncoder().encode("stale\n")); await new Promise((resolve) => setTimeout(resolve, 50)); controller.error(new Error("expired")); }, - })); + }), { size: 10 }); ctx.res = { - name: "gblob-recover", + name: "blob-recover", files: [ { name: "recover.txt", + size: 10, req: { url, } @@ -35,6 +40,6 @@ gopeed.events.onError(async function (ctx) { } req.labels.recovered = "true"; - req.url = URL.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" })); + req.url = gopeed.runtime.blob.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" })); ctx.task.continue(); }); diff --git a/pkg/download/testdata/extensions/gblob_recover/manifest.json b/pkg/download/testdata/extensions/blob_recover/manifest.json similarity index 81% rename from pkg/download/testdata/extensions/gblob_recover/manifest.json rename to pkg/download/testdata/extensions/blob_recover/manifest.json index bfda3f44c..4cbf30555 100644 --- a/pkg/download/testdata/extensions/gblob_recover/manifest.json +++ b/pkg/download/testdata/extensions/blob_recover/manifest.json @@ -1,6 +1,6 @@ { - "name": "gblob-recover", - "title": "gopeed extension gblob recover test", + "name": "blob-recover", + "title": "gopeed extension blob recover test", "version": "0.0.1", "scripts": [ { diff --git a/pkg/download/testdata/extensions/gblob_restart/index.js b/pkg/download/testdata/extensions/blob_restart/index.js similarity index 67% rename from pkg/download/testdata/extensions/gblob_restart/index.js rename to pkg/download/testdata/extensions/blob_restart/index.js index 8b987a1da..87d81587f 100644 --- a/pkg/download/testdata/extensions/gblob_restart/index.js +++ b/pkg/download/testdata/extensions/blob_restart/index.js @@ -1,15 +1,15 @@ -function createRangeGBlobUrl(target) { - return URL.createObjectURL(async (offset) => { +function createRangeBlobUrl(target) { + return gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => { const headers = {}; - if (offset > 0) { - headers.Range = `bytes=${offset}-`; + if (offset > 0 || end >= 0) { + headers.Range = `bytes=${offset}-${end >= 0 ? end : ''}`; } const response = await fetch(target, { headers }); if (!response.body) { throw new Error('empty response body'); } return response.body; - }); + }, { size: 262144, range: true }); } gopeed.events.onResolve(async function (ctx) { @@ -18,14 +18,14 @@ gopeed.events.onResolve(async function (ctx) { } ctx.res = { - name: 'gblob-restart', + name: 'blob-restart', range: true, files: [ { name: 'restart.bin', size: 262144, req: { - url: createRangeGBlobUrl(ctx.req.rawUrl || ctx.req.url), + url: createRangeBlobUrl(ctx.req.rawUrl || ctx.req.url), rawUrl: ctx.req.rawUrl || ctx.req.url, labels: { mode: 'restart', @@ -38,7 +38,7 @@ gopeed.events.onResolve(async function (ctx) { gopeed.events.onError(async function (ctx) { const req = ctx.task?.meta?.req; - if (!req || !req.rawUrl || !req.url || !req.url.startsWith('gblob:')) { + if (!req || !req.rawUrl || !req.url || !req.url.includes('/__gopeed_blob/')) { return; } req.labels = req.labels || {}; @@ -47,7 +47,7 @@ gopeed.events.onError(async function (ctx) { } try { - req.url = createRangeGBlobUrl(req.rawUrl); + req.url = createRangeBlobUrl(req.rawUrl); req.labels.started = 'true'; req.labels.rebuilt = 'true'; ctx.task.continue(); diff --git a/pkg/download/testdata/extensions/gblob_restart/manifest.json b/pkg/download/testdata/extensions/blob_restart/manifest.json similarity index 82% rename from pkg/download/testdata/extensions/gblob_restart/manifest.json rename to pkg/download/testdata/extensions/blob_restart/manifest.json index fe6f44fe9..6dce7be8d 100644 --- a/pkg/download/testdata/extensions/gblob_restart/manifest.json +++ b/pkg/download/testdata/extensions/blob_restart/manifest.json @@ -1,6 +1,6 @@ { - "name": "gblob-restart", - "title": "gopeed extension gblob restart resume test", + "name": "blob-restart", + "title": "gopeed extension blob restart resume test", "version": "0.0.1", "scripts": [ { diff --git a/pkg/download/testdata/extensions/gblob/index.js b/pkg/download/testdata/extensions/gblob/index.js deleted file mode 100644 index 6404e823b..000000000 --- a/pkg/download/testdata/extensions/gblob/index.js +++ /dev/null @@ -1,183 +0,0 @@ -gopeed.events.onResolve(async function (ctx) { - const encoder = new TextEncoder(); - - if (ctx.req.url.endsWith("/blob")) { - const blob = new Blob(["hello world"], {type: "text/plain"}); - const url = URL.createObjectURL(blob); - ctx.res = { - name: "gblob-blob", - files: [ - { - name: "hello.txt", - size: 11, - req: { - url, - } - } - ] - }; - return; - } - - if (ctx.req.url.includes("/http-stream")) { - const source = new URL(ctx.req.url); - const target = source.searchParams.get("target"); - const pair = source.searchParams.get("pair") === "1"; - const resumable = ctx.req.url.includes("/http-stream-range"); - const fileName = source.searchParams.get("name") || "http-stream.bin"; - const headResponse = await fetch(target, {method: "HEAD"}); - const headLength = parseInt(headResponse.headers.get("content-length") || "", 10); - - const createOpen = () => async (offset) => { - const headers = {}; - if (offset > 0) { - headers.Range = `bytes=${offset}-`; - } - const response = await fetch(target, {headers}); - if (!response.body) { - throw new Error("empty response body"); - } - return response.body; - }; - - ctx.res = { - name: "gblob-http-stream", - range: resumable, - files: pair ? [ - { - name: "pair-video.bin", - size: Number.isFinite(headLength) ? headLength : 0, - req: { - url: URL.createObjectURL(createOpen()), - } - }, - { - name: "pair-audio.bin", - size: Number.isFinite(headLength) ? headLength : 0, - req: { - url: URL.createObjectURL(createOpen()), - } - } - ] : [ - { - name: fileName, - size: Number.isFinite(headLength) ? headLength : 0, - req: { - url: URL.createObjectURL(createOpen()), - } - } - ] - }; - return; - } - - if (ctx.req.url.includes("/http-blob")) { - const source = new URL(ctx.req.url); - const target = source.searchParams.get("target"); - const fileName = source.searchParams.get("name") || "http-blob.bin"; - const headResponse = await fetch(target, {method: "HEAD"}); - const headLength = parseInt(headResponse.headers.get("content-length") || "", 10); - const response = await fetch(target); - const blob = await response.blob(); - - ctx.res = { - name: "gblob-http-blob", - files: [ - { - name: fileName, - size: Number.isFinite(headLength) ? headLength : 0, - req: { - url: URL.createObjectURL(blob), - } - } - ] - }; - return; - } - - if (ctx.req.url.endsWith("/stream-range")) { - const payload = encoder.encode("line 1\nline 2\nline 3\n"); - const firstChunkLength = encoder.encode("line 1\n").byteLength; - let firstAttempt = true; - - const open = async (offset) => new ReadableStream({ - async start(controller) { - try { - if (offset === 0 && firstAttempt) { - firstAttempt = false; - controller.enqueue(payload.slice(0, firstChunkLength)); - await new Promise((resolve) => setTimeout(resolve, 120)); - controller.error(new Error("resume required")); - return; - } - controller.enqueue(payload.slice(offset)); - controller.close(); - } catch (err) { - controller.error(err); - } - } - }); - - const url = URL.createObjectURL(open); - - ctx.res = { - name: "gblob-stream-range", - range: true, - files: [ - { - name: "stream-range.txt", - size: payload.byteLength, - req: { - url, - } - } - ] - }; - return; - } - - if (ctx.req.url.endsWith("/stream-unknown")) { - const url = URL.createObjectURL(new ReadableStream({ - async start(controller) { - controller.enqueue(encoder.encode("line 1\n")); - await new Promise((resolve) => setTimeout(resolve, 350)); - controller.enqueue(encoder.encode("line 2\n")); - controller.close(); - }, - })); - - ctx.res = { - name: "gblob-stream-unknown", - files: [ - { - name: "stream-unknown.txt", - req: { - url, - } - } - ] - }; - return; - } - - const url = URL.createObjectURL(new ReadableStream({ - start(controller) { - controller.enqueue(encoder.encode("line 1\n")); - controller.enqueue(encoder.encode("line 2\n")); - controller.close(); - }, - })); - - ctx.res = { - name: "gblob-stream", - files: [ - { - name: "stream.txt", - size: 14, - req: { - url, - } - } - ] - }; -}); diff --git a/ui/flutter/lib/api/model/task.dart b/ui/flutter/lib/api/model/task.dart index 2aea7e8c1..830d772dd 100644 --- a/ui/flutter/lib/api/model/task.dart +++ b/ui/flutter/lib/api/model/task.dart @@ -6,7 +6,7 @@ part 'task.g.dart'; enum Status { ready, running, pause, wait, error, done } -enum Protocol { http, bt, ed2k, gblob } +enum Protocol { http, bt, ed2k } // ExtractStatus enum matching Go backend enum ExtractStatus { diff --git a/ui/flutter/lib/api/model/task.g.dart b/ui/flutter/lib/api/model/task.g.dart index 4db11e3a6..b2889e73b 100644 --- a/ui/flutter/lib/api/model/task.g.dart +++ b/ui/flutter/lib/api/model/task.g.dart @@ -52,7 +52,6 @@ const _$ProtocolEnumMap = { Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', - Protocol.gblob: 'gblob', }; Progress _$ProgressFromJson(Map json) => Progress( From 93ffa4dac176cbbd5a0f84c6bddf8fa91c5975ac Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 8 Jul 2026 21:21:13 +0800 Subject: [PATCH 2/7] chore: remove blob spool cleanup --- pkg/download/downloader.go | 4 ---- pkg/download/downloader_test.go | 25 ------------------------- 2 files changed, 29 deletions(-) diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 71522329e..2179a4607 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -145,10 +145,6 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader { } func (d *Downloader) Setup() error { - blobDir := filepath.Join(d.cfg.StorageDir, "blob") - if err := os.RemoveAll(blobDir); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } d.blob = internalblob.NewRegistry("") // setup storage diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index 362aa2613..ba30c59f4 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -85,31 +85,6 @@ func TestDownloader_BlobRegistryDoesNotUseStorageDirForSpooling(t *testing.T) { } } -func TestDownloader_SetupCleansStaleBlobDir(t *testing.T) { - storageDir := t.TempDir() - blobDir := filepath.Join(storageDir, "blob") - if err := os.MkdirAll(blobDir, 0o755); err != nil { - t.Fatal(err) - } - staleFile := filepath.Join(blobDir, "stale") - if err := os.WriteFile(staleFile, []byte("stale"), 0o644); err != nil { - t.Fatal(err) - } - - downloader := NewDownloader(&DownloaderConfig{ - Storage: NewMemStorage(), - StorageDir: storageDir, - }) - if err := downloader.Setup(); err != nil { - t.Fatal(err) - } - defer downloader.Clear() - - if _, err := os.Stat(staleFile); !os.IsNotExist(err) { - t.Fatalf("expected stale blob file to be removed on setup, got err=%v", err) - } -} - func TestDownloader_BlobTaskKeepsConfiguredHTTPConnections(t *testing.T) { downloader := NewDownloader(&DownloaderConfig{ Storage: NewMemStorage(), From 48cc93f9b9922e98eee2a44c4d0b582d78d9184f Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 8 Jul 2026 21:29:34 +0800 Subject: [PATCH 3/7] chore: shorten blob internal path --- internal/blob/registry.go | 2 +- internal/protocol/http/fetcher.go | 2 +- pkg/download/downloader.go | 2 +- pkg/download/testdata/extensions/blob_restart/index.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/blob/registry.go b/internal/blob/registry.go index 029d720cc..7f9b81cbb 100644 --- a/internal/blob/registry.go +++ b/internal/blob/registry.go @@ -17,7 +17,7 @@ import ( "sync" ) -const urlPathPrefix = "/__gopeed_blob/" +const urlPathPrefix = "/__blob/" var ( ErrInvalidURL = errors.New("invalid blob url") diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index b17d19c1c..bf43cfcfd 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -675,7 +675,7 @@ func (f *Fetcher) doStart() error { return err } f.redirectLock.Lock() - if strings.Contains(f.redirectURL, "/__gopeed_blob/") && f.redirectURL != f.meta.Req.URL { + if strings.Contains(f.redirectURL, "/__blob/") && f.redirectURL != f.meta.Req.URL { f.redirectURL = "" } f.redirectLock.Unlock() diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 2179a4607..4059a3575 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -1853,7 +1853,7 @@ func (d *Downloader) releaseBlobTask(task *Task) { } func isInternalBlobURLString(raw string) bool { - return strings.Contains(raw, "/__gopeed_blob/") + return strings.Contains(raw, "/__blob/") } var defaultDownloader = NewDownloader(nil) diff --git a/pkg/download/testdata/extensions/blob_restart/index.js b/pkg/download/testdata/extensions/blob_restart/index.js index 87d81587f..834e629a0 100644 --- a/pkg/download/testdata/extensions/blob_restart/index.js +++ b/pkg/download/testdata/extensions/blob_restart/index.js @@ -38,7 +38,7 @@ gopeed.events.onResolve(async function (ctx) { gopeed.events.onError(async function (ctx) { const req = ctx.task?.meta?.req; - if (!req || !req.rawUrl || !req.url || !req.url.includes('/__gopeed_blob/')) { + if (!req || !req.rawUrl || !req.url || !req.url.includes('/__blob/')) { return; } req.labels = req.labels || {}; From 71b04d03508f320068bfbb5592a301de23d6f0a3 Mon Sep 17 00:00:00 2001 From: Levi Date: Thu, 9 Jul 2026 20:57:44 +0800 Subject: [PATCH 4/7] update blob stream --- internal/blob/registry.go | 37 +- internal/blob/registry_test.go | 4 +- pkg/download/engine/inject/stream/module.go | 330 +++++++++++++++--- pkg/download/engine/inject/stream/stream.js | 91 ++++- pkg/download/extension_test.go | 313 +++++++++++++++++ .../extensions/blob_stream_bridge/index.js | 198 +++++++++++ .../blob_stream_bridge/manifest.json | 18 + 7 files changed, 914 insertions(+), 77 deletions(-) create mode 100644 pkg/download/testdata/extensions/blob_stream_bridge/index.js create mode 100644 pkg/download/testdata/extensions/blob_stream_bridge/manifest.json diff --git a/internal/blob/registry.go b/internal/blob/registry.go index 7f9b81cbb..1ac4d999f 100644 --- a/internal/blob/registry.go +++ b/internal/blob/registry.go @@ -55,7 +55,6 @@ type Metadata struct { type Source struct { ID string - Token string URL string ContentType string @@ -143,17 +142,12 @@ func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, err if err != nil { return "", err } - token, err := randomID(24) - if err != nil { - return "", err - } if opts.Session != nil { opts.Session.Retain() } - srcURL := fmt.Sprintf("%s%s?token=%s", baseURL, id, url.QueryEscape(token)) + srcURL := fmt.Sprintf("%s%s", baseURL, id) src := &Source{ ID: id, - Token: token, URL: srcURL, ContentType: opts.ContentType, size: opts.Size, @@ -248,13 +242,13 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - id, token := parseRequest(req) - if id == "" || token == "" { + id := parseRequest(req) + if id == "" { http.NotFound(w, req) return } src, err := r.getByID(id) - if err != nil || src.Token != token { + if err != nil { http.NotFound(w, req) return } @@ -412,7 +406,7 @@ func parseRange(header string, size int64, rangeEnabled bool) (start int64, end } func (r *Registry) get(raw string) (*Source, error) { - id, token, ok := r.parseURL(raw) + id, ok := r.parseURL(raw) if !ok { return nil, ErrInvalidURL } @@ -420,9 +414,6 @@ func (r *Registry) get(raw string) (*Source, error) { if err != nil { return nil, err } - if src.Token != token { - return nil, ErrInvalidURL - } src.mu.Lock() revoked := src.revoked src.mu.Unlock() @@ -442,30 +433,30 @@ func (r *Registry) getByID(id string) (*Source, error) { return src, nil } -func (r *Registry) parseURL(raw string) (id string, token string, ok bool) { +func (r *Registry) parseURL(raw string) (id string, ok bool) { u, err := url.Parse(raw) if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) { - return "", "", false + return "", false } r.serverMu.Lock() baseURL := r.baseURL r.serverMu.Unlock() if baseURL == "" || !strings.HasPrefix(raw, baseURL) { - return "", "", false + return "", false } id = strings.TrimPrefix(u.Path, urlPathPrefix) if id == "" || path.Base(id) != id { - return "", "", false + return "", false } - return id, u.Query().Get("token"), true + return id, true } -func parseRequest(req *http.Request) (id string, token string) { - id = strings.TrimPrefix(req.URL.Path, urlPathPrefix) +func parseRequest(req *http.Request) string { + id := strings.TrimPrefix(req.URL.Path, urlPathPrefix) if id == "" || id == req.URL.Path || path.Base(id) != id { - return "", "" + return "" } - return id, req.URL.Query().Get("token") + return id } func randomID(size int) (string, error) { diff --git a/internal/blob/registry_test.go b/internal/blob/registry_test.go index 1bb5e6cd4..db23aa7e8 100644 --- a/internal/blob/registry_test.go +++ b/internal/blob/registry_test.go @@ -227,14 +227,14 @@ func TestRegistryHTTPErrorStatuses(t *testing.T) { t.Fatal(err) } - badURL := strings.Replace(url, "token=", "token=bad", 1) + badURL := url + "-missing" resp, err := http.Get(badURL) if err != nil { t.Fatal(err) } _ = resp.Body.Close() if resp.StatusCode != http.StatusNotFound { - t.Fatalf("unexpected bad token status: %d", resp.StatusCode) + t.Fatalf("unexpected missing source status: %d", resp.StatusCode) } req, err := http.NewRequest(http.MethodHead, url, nil) diff --git a/pkg/download/engine/inject/stream/module.go b/pkg/download/engine/inject/stream/module.go index 01340c531..6fff322a9 100644 --- a/pkg/download/engine/inject/stream/module.go +++ b/pkg/download/engine/inject/stream/module.go @@ -11,6 +11,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "time" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" @@ -48,6 +49,7 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error if cfg == nil { cfg = &Config{} } + blobPipes := newBlobPipeRegistry() if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value { if cfg.CreateObjectURL == nil { panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured"))) @@ -74,7 +76,7 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error Range: rangeEnabled, } url, err := cfg.CreateObjectURL(opts, func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) { - return openBlobObjectURLReader(ctx, loop, openValue, req) + return openBlobObjectURLReader(ctx, loop, blobPipes, openValue, req) }) if err != nil { panic(runtime.NewGoError(err)) @@ -99,25 +101,47 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error if err != nil { panic(runtime.NewGoError(err)) } - meta, err := fetchRegistry.Open(runtime, cfg.ProxyHandler, reqMeta) - if err != nil { - panic(runtime.NewGoError(err)) - } - return runtime.ToValue(meta) + fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey) + promise, resolve, reject := runtime.NewPromise() + go func() { + meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta) + ok := loop.RunOnLoop(func(runtime *goja.Runtime) { + if err != nil { + reject(runtime.NewGoError(err)) + return + } + resolve(runtime.ToValue(meta)) + }) + if !ok && meta != nil { + fetchRegistry.Close(meta.ID) + } + }() + return runtime.ToValue(promise) }); err != nil { return err } if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value { id := call.Argument(0).String() chunkSize := int(call.Argument(1).ToInteger()) - chunk, done, err := fetchRegistry.Read(id, chunkSize) - if err != nil { - panic(runtime.NewGoError(err)) - } - if done { - return goja.Null() - } - return runtime.ToValue(runtime.NewArrayBuffer(chunk)) + promise, resolve, reject := runtime.NewPromise() + go func() { + chunk, done, err := fetchRegistry.Read(id, chunkSize) + ok := loop.RunOnLoop(func(runtime *goja.Runtime) { + if err != nil { + reject(runtime.NewGoError(err)) + return + } + if done { + resolve(goja.Null()) + return + } + resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk))) + }) + if !ok { + fetchRegistry.Close(id) + } + }() + return runtime.ToValue(promise) }); err != nil { return err } @@ -131,36 +155,206 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error }); err != nil { return err } + if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value { + id := call.Argument(0).String() + ok := blobPipes.Push(id, exportBytes(call.Argument(1))) + return runtime.ToValue(ok) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) { + blobPipes.Close(id, nil) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) { + blobPipes.Close(id, errors.New(message)) + }); err != nil { + return err + } _, err := runtime.RunString(script) return err } -func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) { - value, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) { - fnVal := runtime.Get("__gopeed_blob_open_source") +func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, pipes *blobPipeRegistry, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) { + if pipes == nil { + return nil, fmt.Errorf("blob pipe registry not configured") + } + pipe := newBlobPipe(ctx, pipes) + pipes.Add(pipe) + _, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) { + fnVal := runtime.Get("__gopeed_blob_pipe_source") fn, ok := goja.AssertFunction(fnVal) if !ok { - return nil, fmt.Errorf("blob open helper is not callable") + return nil, fmt.Errorf("blob pipe helper is not callable") } request := map[string]any{ "offset": req.Offset, "end": req.End, } - return fn(nil, openValue, runtime.ToValue(request)) + return fn(nil, openValue, runtime.ToValue(request), runtime.ToValue(pipe.id)) }) if err != nil { + pipe.Close() return nil, err } - id := value.String() - if id == "" { - return nil, fmt.Errorf("blob open helper returned empty reader id") + return pipe, nil +} + +type blobPipeRegistry struct { + mu sync.Mutex + pipes map[string]*blobPipe + next atomic.Uint64 +} + +func newBlobPipeRegistry() *blobPipeRegistry { + return &blobPipeRegistry{ + pipes: map[string]*blobPipe{}, + } +} + +func (r *blobPipeRegistry) Add(pipe *blobPipe) { + r.mu.Lock() + r.pipes[pipe.id] = pipe + r.mu.Unlock() +} + +func (r *blobPipeRegistry) Remove(id string) { + r.mu.Lock() + delete(r.pipes, id) + r.mu.Unlock() +} + +func (r *blobPipeRegistry) Push(id string, chunk []byte) bool { + r.mu.Lock() + pipe := r.pipes[id] + r.mu.Unlock() + if pipe == nil { + return false + } + return pipe.Push(chunk) +} + +func (r *blobPipeRegistry) Close(id string, err error) { + r.mu.Lock() + pipe := r.pipes[id] + r.mu.Unlock() + if pipe != nil { + pipe.CloseWithError(err) + } +} + +func (r *blobPipeRegistry) NewID() string { + return fmt.Sprintf("pipe-%d", r.next.Add(1)) +} + +type blobPipeItem struct { + chunk []byte + err error + done bool +} + +type blobPipe struct { + id string + registry *blobPipeRegistry + ctx context.Context + cancel context.CancelFunc + ch chan blobPipeItem + + mu sync.Mutex + buf []byte + closed bool + once sync.Once +} + +func newBlobPipe(ctx context.Context, registry *blobPipeRegistry) *blobPipe { + pipeCtx, cancel := context.WithCancel(ctx) + return &blobPipe{ + id: registry.NewID(), + registry: registry, + ctx: pipeCtx, + cancel: cancel, + ch: make(chan blobPipeItem, 8), + } +} + +func (p *blobPipe) Push(chunk []byte) bool { + if len(chunk) == 0 { + return true + } + buf := append([]byte(nil), chunk...) + select { + case p.ch <- blobPipeItem{chunk: buf}: + return true + case <-p.ctx.Done(): + return false + } +} + +func (p *blobPipe) CloseWithError(err error) { + p.once.Do(func() { + p.registry.Remove(p.id) + select { + case p.ch <- blobPipeItem{done: true, err: err}: + case <-p.ctx.Done(): + } + }) +} + +func (p *blobPipe) Read(dst []byte) (int, error) { + p.mu.Lock() + if len(p.buf) > 0 { + n := copy(dst, p.buf) + p.buf = p.buf[n:] + p.mu.Unlock() + return n, nil + } + p.mu.Unlock() + + select { + case item := <-p.ch: + if item.done { + if item.err != nil { + return 0, item.err + } + return 0, io.EOF + } + n := copy(dst, item.chunk) + if n < len(item.chunk) { + p.mu.Lock() + p.buf = append(p.buf, item.chunk[n:]...) + p.mu.Unlock() + } + return n, nil + case <-p.ctx.Done(): + return 0, p.ctx.Err() + } +} + +func (p *blobPipe) Close() error { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return nil + } + p.closed = true + p.mu.Unlock() + p.registry.Remove(p.id) + p.cancel() + return nil +} + +func exportBytes(value goja.Value) []byte { + if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { + return nil + } + if ab, ok := value.Export().(goja.ArrayBuffer); ok { + return ab.Bytes() } - reader := &blobObjectURLReader{ - ctx: ctx, - loop: loop, - id: id, + if b, ok := value.Export().([]byte); ok { + return b } - return reader, nil + return nil } type blobObjectURLReader struct { @@ -340,6 +534,13 @@ type fetchStream struct { body io.ReadCloser cancel context.CancelFunc closeOnce sync.Once + ctx context.Context + ch chan fetchChunk +} + +type fetchChunk struct { + data []byte + err error } type fetchRequest struct { @@ -397,12 +598,12 @@ func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, return meta, nil } -func (r *fetchRegistry) Open(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) { +func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) { client := req.C() if proxyHandler != nil { client.SetProxy(proxyHandler) } - setFetchFingerprint(client, util.SafeGet[string](runtime, xhr.FingerprintMagicKey)) + setFetchFingerprint(client, fingerprint) contentType, body, err := buildFetchBody(reqMeta.Body) if err != nil { return nil, err @@ -460,9 +661,16 @@ func (r *fetchRegistry) Open(runtime *goja.Runtime, proxyHandler func(r *http.Re if resp.Response != nil && resp.Response.Body != nil { bodyCloser = resp.Response.Body } + stream := &fetchStream{ + body: bodyCloser, + cancel: cancel, + ctx: ctx, + ch: make(chan fetchChunk, 8), + } r.mu.Lock() - r.streams[id] = &fetchStream{body: bodyCloser, cancel: cancel} + r.streams[id] = stream r.mu.Unlock() + go stream.readLoop() return meta, nil } @@ -471,23 +679,55 @@ func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) { if stream == nil { return nil, true, nil } - if chunkSize <= 0 { - chunkSize = 64 * 1024 - } - buf := make([]byte, chunkSize) - n, err := stream.body.Read(buf) - if n > 0 { - return buf[:n], false, nil - } - if err == io.EOF { - r.Close(id) - return nil, true, nil + select { + case item, ok := <-stream.ch: + if !ok { + r.Close(id) + return nil, true, nil + } + if item.err != nil { + r.Close(id) + return nil, false, item.err + } + if chunkSize > 0 && len(item.data) > chunkSize { + head := append([]byte(nil), item.data[:chunkSize]...) + tail := append([]byte(nil), item.data[chunkSize:]...) + select { + case stream.ch <- fetchChunk{data: tail}: + case <-stream.ctx.Done(): + } + return head, false, nil + } + return item.data, false, nil + case <-stream.ctx.Done(): + return nil, false, stream.ctx.Err() } - if err != nil { - r.Close(id) - return nil, false, err +} + +func (s *fetchStream) readLoop() { + defer close(s.ch) + buf := make([]byte, 64*1024) + for { + n, err := s.body.Read(buf) + if n > 0 { + chunk := append([]byte(nil), buf[:n]...) + select { + case s.ch <- fetchChunk{data: chunk}: + case <-s.ctx.Done(): + return + } + } + if err == io.EOF { + return + } + if err != nil { + select { + case s.ch <- fetchChunk{err: err}: + case <-s.ctx.Done(): + } + return + } } - return nil, false, nil } func (r *fetchRegistry) Close(id string) { diff --git a/pkg/download/engine/inject/stream/stream.js b/pkg/download/engine/inject/stream/stream.js index 11107a1e7..22d5c0b5f 100644 --- a/pkg/download/engine/inject/stream/stream.js +++ b/pkg/download/engine/inject/stream/stream.js @@ -671,14 +671,57 @@ return { offset, end }; } - globalThis.__gopeed_blob_open_source = async function (openReadable, request) { + function yieldBlobPipeTask() { + return new Promise((resolve) => { + if (typeof setTimeout === "function") { + setTimeout(resolve, 0); + } else { + Promise.resolve().then(resolve); + } + }); + } + + function createPendingBlobReader(openReadable, request) { + let activeReader; + let ready; + function ensureReader() { + if (ready) { + return ready; + } + ready = (async function () { + const source = await openReadable(normalizeOpenRequest(request)); + activeReader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); + return activeReader; + })(); + return ready; + } + return { + async read() { + const reader = await ensureReader(); + return reader.read(); + }, + cancel(reason) { + return ensureReader().then(function (reader) { + return reader.cancel(reason); + }, function () {}); + }, + releaseLock() { + if (!ready) { + return; + } + return ready.then(function (reader) { + return reader.releaseLock(); + }, function () {}); + } + }; + } + + globalThis.__gopeed_blob_open_source = function (openReadable, request) { if (typeof openReadable !== "function") { throw new TypeError("blob opener must be callable"); } - const source = await openReadable(normalizeOpenRequest(request)); - const reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); const id = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random()); - blobReaders.set(id, reader); + blobReaders.set(id, createPendingBlobReader(openReadable, request)); return id; }; @@ -728,6 +771,40 @@ releaseReader(reader, "blob request closed"); }; + globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) { + if (typeof openReadable !== "function") { + throw new TypeError("blob opener must be callable"); + } + const startPipe = async function () { + let reader; + try { + const source = await openReadable(normalizeOpenRequest(request)); + reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); + while (true) { + const { done, value } = await reader.read(); + if (done) { + globalThis.__gopeed_blob_pipe_close(pipeId); + return; + } + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); + if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) { + releaseReader(reader, "blob pipe closed"); + return; + } + await yieldBlobPipeTask(); + } + } catch (error) { + globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error)); + } + }; + if (typeof setTimeout === "function") { + setTimeout(startPipe, 0); + } else { + Promise.resolve().then(startPipe); + } + }; + globalThis.__gopeed_blob_create_object_url = function (value, options) { const described = describeObjectURLValue(value); if (described.kind === "blob") { @@ -781,7 +858,7 @@ }); let meta; try { - meta = fetchOpen({ + meta = await fetchOpen({ url: request.url, method: request.method, headers, @@ -793,10 +870,10 @@ throw error instanceof Error ? error : new TypeError(String(error)); } const stream = new ReadableStream({ - pull(controller) { + async pull(controller) { let chunk; try { - chunk = fetchRead(meta.id, 64 * 1024); + chunk = await fetchRead(meta.id, 64 * 1024); } catch (error) { fetchClose(meta.id); controller.error(error instanceof Error ? error : new TypeError(String(error))); diff --git a/pkg/download/extension_test.go b/pkg/download/extension_test.go index cf0c94c8a..d229c8b5e 100644 --- a/pkg/download/extension_test.go +++ b/pkg/download/extension_test.go @@ -249,6 +249,285 @@ func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T }) } +func TestDownloader_Extension_BlobFetchDrainedStreamReportsEarlyProgress(t *testing.T) { + chunkDelay := 350 * time.Millisecond + chunks := []string{ + strings.Repeat("a", 4096), + strings.Repeat("b", 4096), + strings.Repeat("c", 4096), + strings.Repeat("d", 4096), + strings.Repeat("e", 4096), + } + payload := strings.Join(chunks, "") + upstreamDuration := time.Duration(len(chunks)) * chunkDelay + firstProgressTimeout := upstreamDuration * 3 / 5 + completionTimeout := upstreamDuration + 4*time.Second + requireEarlyProgressWindow(t, firstProgressTimeout, upstreamDuration) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/single-fetch-stream") || r.URL.Query().Get("source") != "1" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + flusher, _ := w.(http.Flusher) + for _, chunk := range chunks { + if _, err := w.Write([]byte(chunk)); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + time.Sleep(chunkDelay) + } + })) + defer server.Close() + + setupDownloader(func(downloader *Downloader) { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_stream_bridge", false); err != nil { + t.Fatal(err) + } + + rr, err := downloader.Resolve(&base.Request{ + URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)), + }, nil) + if err != nil { + t.Fatal(err) + } + if len(rr.Res.Files) != 1 { + t.Fatalf("unexpected resolve files: %#v", rr.Res.Files) + } + + dir := t.TempDir() + id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ + Path: dir, + Name: rr.Res.Files[0].Name, + }) + if err != nil { + t.Fatal(err) + } + + filePath := filepath.Join(dir, "single.bin") + waitForFileSizeAtLeast(t, filePath, 1, firstProgressTimeout) + waitForTaskTerminal(t, downloader, id, completionTimeout) + + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + if string(data) != payload { + t.Fatalf("unexpected single stream content length=%d want=%d", len(data), len(payload)) + } + }) +} + +func TestDownloader_Extension_BlobRangeFetchDrainedStreamReportsEarlyProgress(t *testing.T) { + chunkDelay := 350 * time.Millisecond + chunkSize := 4096 + chunks := []string{ + strings.Repeat("a", chunkSize), + strings.Repeat("b", chunkSize), + strings.Repeat("c", chunkSize), + strings.Repeat("d", chunkSize), + strings.Repeat("e", chunkSize), + strings.Repeat("f", chunkSize), + } + payload := strings.Join(chunks, "") + upstreamDuration := time.Duration(len(chunks)) * chunkDelay + firstProgressTimeout := upstreamDuration * 3 / 5 + completionTimeout := upstreamDuration + 4*time.Second + requireEarlyProgressWindow(t, firstProgressTimeout, upstreamDuration) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/range-fetch-stream") || r.URL.Query().Get("source") != "1" { + http.NotFound(w, r) + return + } + + start := 0 + end := len(payload) - 1 + status := http.StatusOK + if rangeHeader := r.Header.Get("Range"); strings.HasPrefix(rangeHeader, "bytes=") { + parts := strings.SplitN(strings.TrimPrefix(rangeHeader, "bytes="), "-", 2) + if len(parts) != 2 || parts[0] == "" { + http.Error(w, "bad range", http.StatusBadRequest) + return + } + parsedStart, err := strconv.Atoi(parts[0]) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + start = parsedStart + if parts[1] != "" { + parsedEnd, err := strconv.Atoi(parts[1]) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + end = parsedEnd + } + if start >= len(payload) || end < start { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + if end >= len(payload) { + end = len(payload) - 1 + } + status = http.StatusPartialContent + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload))) + } + + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) + w.WriteHeader(status) + + flusher, _ := w.(http.Flusher) + for offset := start; offset <= end; offset += chunkSize { + next := offset + chunkSize + if next > end+1 { + next = end + 1 + } + if _, err := w.Write([]byte(payload[offset:next])); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + time.Sleep(chunkDelay) + } + })) + defer server.Close() + + setupDownloader(func(downloader *Downloader) { + downloader.cfg.RefreshInterval = 50 + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_stream_bridge", false); err != nil { + t.Fatal(err) + } + + rr, err := downloader.Resolve(&base.Request{ + URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)), + }, nil) + if err != nil { + t.Fatal(err) + } + if len(rr.Res.Files) != 1 || !rr.Res.Range { + t.Fatalf("unexpected resolve result: %#v", rr.Res) + } + + dir := t.TempDir() + id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ + Path: dir, + Name: rr.Res.Files[0].Name, + Extra: map[string]any{ + "connections": 1, + }, + }) + if err != nil { + t.Fatal(err) + } + + waitForTaskDownloadedBetween(t, downloader, id, 1, int64(len(payload)-1), firstProgressTimeout) + waitForTaskTerminal(t, downloader, id, completionTimeout) + + data, err := os.ReadFile(filepath.Join(dir, "range.bin")) + if err != nil { + t.Fatal(err) + } + if string(data) != payload { + t.Fatalf("unexpected range stream content length=%d want=%d", len(data), len(payload)) + } + }) +} + +func TestDownloader_Extension_BlobMultiplexFetchStreamsReportEarlyProgress(t *testing.T) { + chunkDelay := 300 * time.Millisecond + videoChunks := []string{"video-0", "video-1", "video-2", "video-3"} + audioChunks := []string{"audio-0", "audio-1", "audio-2", "audio-3"} + videoPayload := strings.Join(videoChunks, "") + audioPayload := strings.Join(audioChunks, "") + upstreamDuration := time.Duration(len(videoChunks)+len(audioChunks)) * chunkDelay + firstProgressTimeout := upstreamDuration / 2 + completionTimeout := upstreamDuration + 4*time.Second + requireEarlyProgressWindow(t, firstProgressTimeout, upstreamDuration) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/multiplex-fetch-stream") || r.URL.Query().Get("source") != "1" { + http.NotFound(w, r) + return + } + flusher, _ := w.(http.Flusher) + for i := range videoChunks { + if _, err := fmt.Fprintf(w, "v:%s\n", videoChunks[i]); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + time.Sleep(chunkDelay) + if _, err := fmt.Fprintf(w, "a:%s\n", audioChunks[i]); err != nil { + return + } + if flusher != nil { + flusher.Flush() + } + time.Sleep(chunkDelay) + } + })) + defer server.Close() + + setupDownloader(func(downloader *Downloader) { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_stream_bridge", false); err != nil { + t.Fatal(err) + } + + rr, err := downloader.Resolve(&base.Request{ + URL: server.URL + "/multiplex-fetch-stream", + }, nil) + if err != nil { + t.Fatal(err) + } + if len(rr.Res.Files) != 2 { + t.Fatalf("unexpected resolve files: %#v", rr.Res.Files) + } + + dir := t.TempDir() + videoID, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ + Path: dir, + Name: rr.Res.Files[0].Name, + }) + if err != nil { + t.Fatal(err) + } + audioID, err := downloader.CreateDirect(rr.Res.Files[1].Req, &base.Options{ + Path: dir, + Name: rr.Res.Files[1].Name, + }) + if err != nil { + t.Fatal(err) + } + + videoPath := filepath.Join(dir, "video.bin") + audioPath := filepath.Join(dir, "audio.bin") + waitForFileSizeAtLeast(t, videoPath, 1, firstProgressTimeout) + waitForFileSizeAtLeast(t, audioPath, 1, firstProgressTimeout) + waitForTaskTerminal(t, downloader, videoID, completionTimeout) + waitForTaskTerminal(t, downloader, audioID, completionTimeout) + + videoData, err := os.ReadFile(videoPath) + if err != nil { + t.Fatal(err) + } + audioData, err := os.ReadFile(audioPath) + if err != nil { + t.Fatal(err) + } + if string(videoData) != videoPayload { + t.Fatalf("unexpected video content %q want %q", string(videoData), videoPayload) + } + if string(audioData) != audioPayload { + t.Fatalf("unexpected audio content %q want %q", string(audioData), audioPayload) + } + }) +} + func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) { setupDownloader(func(downloader *Downloader) { if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { @@ -1678,6 +1957,40 @@ func waitForFileSizeAtLeast(t *testing.T, path string, size int64, timeout time. t.Fatalf("timeout waiting for file %s size >= %d", path, size) } +func requireEarlyProgressWindow(t *testing.T, earlyTimeout, upstreamDuration time.Duration) { + t.Helper() + if earlyTimeout <= 0 { + t.Fatalf("early progress timeout must be positive, got %s", earlyTimeout) + } + if upstreamDuration <= 0 { + t.Fatalf("upstream duration must be positive, got %s", upstreamDuration) + } + if earlyTimeout >= upstreamDuration { + t.Fatalf("early progress timeout %s must be shorter than upstream duration %s", earlyTimeout, upstreamDuration) + } +} + +func waitForTaskDownloadedBetween(t *testing.T, downloader *Downloader, id string, min, max int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + task := downloader.GetTask(id) + if task != nil && task.Progress != nil && task.Progress.Downloaded >= min && task.Progress.Downloaded <= max { + return + } + time.Sleep(10 * time.Millisecond) + } + task := downloader.GetTask(id) + if task == nil { + t.Fatalf("timeout waiting for task %s downloaded between %d and %d: task not found", id, min, max) + } + downloaded := int64(0) + if task.Progress != nil { + downloaded = task.Progress.Downloaded + } + t.Fatalf("timeout waiting for task %s downloaded between %d and %d: got %d status=%s", id, min, max, downloaded, task.Status) +} + func waitForTaskTerminal(t *testing.T, downloader *Downloader, id string, timeout time.Duration) { t.Helper() doneCh := make(chan error, 1) diff --git a/pkg/download/testdata/extensions/blob_stream_bridge/index.js b/pkg/download/testdata/extensions/blob_stream_bridge/index.js new file mode 100644 index 000000000..f3c7e6164 --- /dev/null +++ b/pkg/download/testdata/extensions/blob_stream_bridge/index.js @@ -0,0 +1,198 @@ +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function byteLength(value) { + return encoder.encode(value).byteLength; +} + +function sourceURL(rawURL) { + const url = new URL(rawURL); + url.searchParams.set("source", "1"); + return url.toString(); +} + +function createFetchDrainedStream(url) { + return new ReadableStream({ + async start(controller) { + try { + const response = await fetch(url); + const reader = response.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + } + }, + }); +} + +function createFetchDrainedRangeStream(url, offset, end) { + return new ReadableStream({ + async start(controller) { + try { + const headers = {}; + if (offset > 0 || end >= 0) { + headers.Range = `bytes=${offset}-${end >= 0 ? end : ""}`; + } + const response = await fetch(url, { headers }); + const reader = response.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + } + }, + }); +} + +function createMultiplexStreams(url) { + let videoController; + let audioController; + let started = false; + + async function pump() { + if (started || !videoController || !audioController) { + return; + } + started = true; + + try { + const response = await fetch(url); + const reader = response.body.getReader(); + let pending = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + pending += decoder.decode(value, { stream: true }); + + let newlineIndex; + while ((newlineIndex = pending.indexOf("\n")) >= 0) { + const line = pending.slice(0, newlineIndex); + pending = pending.slice(newlineIndex + 1); + if (line.startsWith("v:")) { + videoController.enqueue(encoder.encode(line.slice(2))); + } else if (line.startsWith("a:")) { + audioController.enqueue(encoder.encode(line.slice(2))); + } + } + } + + if (pending.startsWith("v:")) { + videoController.enqueue(encoder.encode(pending.slice(2))); + } else if (pending.startsWith("a:")) { + audioController.enqueue(encoder.encode(pending.slice(2))); + } + videoController.close(); + audioController.close(); + } catch (error) { + videoController.error(error); + audioController.error(error); + } + } + + return { + video: new ReadableStream({ + start(controller) { + videoController = controller; + pump(); + }, + }), + audio: new ReadableStream({ + start(controller) { + audioController = controller; + pump(); + }, + }), + }; +} + +gopeed.events.onResolve(async function (ctx) { + if (ctx.req.url.includes("/single-fetch-stream")) { + const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0; + const url = gopeed.runtime.blob.createObjectURL(async () => createFetchDrainedStream(sourceURL(ctx.req.url)), { + size: payloadSize, + }); + ctx.res = { + name: "single-fetch-stream", + files: [ + { + name: "single.bin", + size: payloadSize, + req: { url }, + }, + ], + }; + return; + } + + if (ctx.req.url.includes("/range-fetch-stream")) { + const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0; + const url = gopeed.runtime.blob.createObjectURL( + async ({ offset = 0, end = -1 }) => createFetchDrainedRangeStream(sourceURL(ctx.req.url), offset, end), + { + size: payloadSize, + range: true, + } + ); + ctx.res = { + name: "range-fetch-stream", + range: true, + files: [ + { + name: "range.bin", + size: payloadSize, + req: { url }, + }, + ], + }; + return; + } + + if (ctx.req.url.includes("/multiplex-fetch-stream")) { + const videoChunks = ["video-0", "video-1", "video-2", "video-3"]; + const audioChunks = ["audio-0", "audio-1", "audio-2", "audio-3"]; + let streams; + const openStreams = () => { + if (!streams) { + streams = createMultiplexStreams(sourceURL(ctx.req.url)); + } + return streams; + }; + const videoUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().video, { + size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0), + }); + const audioUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().audio, { + size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0), + }); + ctx.res = { + name: "multiplex-fetch-stream", + files: [ + { + name: "video.bin", + size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0), + req: { url: videoUrl }, + }, + { + name: "audio.bin", + size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0), + req: { url: audioUrl }, + }, + ], + }; + } +}); diff --git a/pkg/download/testdata/extensions/blob_stream_bridge/manifest.json b/pkg/download/testdata/extensions/blob_stream_bridge/manifest.json new file mode 100644 index 000000000..ed17c6a14 --- /dev/null +++ b/pkg/download/testdata/extensions/blob_stream_bridge/manifest.json @@ -0,0 +1,18 @@ +{ + "name": "blob-stream-bridge", + "title": "gopeed extension blob stream bridge test", + "version": "0.0.1", + "scripts": [ + { + "event": "onResolve", + "match": { + "urls": [ + "*://*/single-fetch-stream*", + "*://*/range-fetch-stream*", + "*://*/multiplex-fetch-stream*" + ] + }, + "entry": "index.js" + } + ] +} From 02e79441f643243b8711fc9fbc06db9f2afe0740 Mon Sep 17 00:00:00 2001 From: Levi Date: Fri, 10 Jul 2026 14:05:30 +0800 Subject: [PATCH 5/7] update --- internal/blob/registry.go | 275 +++++++- internal/blob/registry_test.go | 617 ++++++++++++++++++ internal/protocol/http/fetcher.go | 97 ++- pkg/download/downloader.go | 492 +++++++++++--- pkg/download/downloader_test.go | 591 +++++++++++++++++ pkg/download/engine/inject/stream/module.go | 224 ++++++- .../inject/stream/module_integration_test.go | 278 ++++++++ .../engine/inject/stream/module_test.go | 191 ++++++ pkg/download/engine/inject/stream/stream.js | 135 ++-- pkg/download/extension.go | 33 +- pkg/download/extension_engine_session.go | 3 +- pkg/download/extension_test.go | 309 +++++---- pkg/download/model.go | 3 + .../extensions/blob_onstart_pause/index.js | 27 + .../blob_onstart_pause/manifest.json | 25 + .../testdata/extensions/blob_recover/index.js | 13 +- 16 files changed, 2915 insertions(+), 398 deletions(-) create mode 100644 pkg/download/engine/inject/stream/module_integration_test.go create mode 100644 pkg/download/engine/inject/stream/module_test.go create mode 100644 pkg/download/testdata/extensions/blob_onstart_pause/index.js create mode 100644 pkg/download/testdata/extensions/blob_onstart_pause/manifest.json diff --git a/internal/blob/registry.go b/internal/blob/registry.go index 1ac4d999f..0da060f96 100644 --- a/internal/blob/registry.go +++ b/internal/blob/registry.go @@ -15,10 +15,17 @@ import ( "strconv" "strings" "sync" + "time" ) const urlPathPrefix = "/__blob/" +const rangeSourceFailureLimit = 2 + +// unclaimedSourceTTL bounds how long a session-backed source may keep its +// engine alive without ever being claimed by a download task. +var unclaimedSourceTTL = 10 * time.Minute + var ( ErrInvalidURL = errors.New("invalid blob url") ErrInvalidOptions = errors.New("invalid blob options") @@ -58,12 +65,16 @@ type Source struct { URL string ContentType string - mu sync.Mutex - size int64 - rangeEnabled bool - revoked bool - session SessionRef - open OpenFunc + mu sync.Mutex + size int64 + rangeEnabled bool + revoked bool + taskRefs int + readErr error + rangeFailures int + session SessionRef + open OpenFunc + unclaimed *time.Timer } type Registry struct { @@ -114,7 +125,7 @@ func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) { }, &CreateOptions{ ContentType: contentType, Size: int64(len(buf)), - Range: true, + Range: len(buf) > 0, }) } @@ -134,12 +145,16 @@ func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, err if opts.Range && opts.Size <= 0 { return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions) } - baseURL, err := r.ensureServer() + // Keep server generation selection and source insertion atomic with Close. + r.serverMu.Lock() + baseURL, err := r.ensureServerLocked() if err != nil { + r.serverMu.Unlock() return "", err } id, err := randomID(18) if err != nil { + r.serverMu.Unlock() return "", err } if opts.Session != nil { @@ -158,6 +173,16 @@ func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, err r.mu.Lock() r.sources[id] = src r.mu.Unlock() + r.serverMu.Unlock() + if opts.Session != nil && unclaimedSourceTTL > 0 { + src.mu.Lock() + if !src.revoked { + src.unclaimed = time.AfterFunc(unclaimedSourceTTL, func() { + r.expireUnclaimed(src) + }) + } + src.mu.Unlock() + } return src.URL, nil } @@ -175,6 +200,50 @@ func (r *Registry) Metadata(raw string) (Metadata, error) { }, nil } +// Acquire retains a source for a task. A source remains available until every +// task that acquired it calls Release, unless it is explicitly revoked. +func (r *Registry) Acquire(raw string) error { + src, err := r.get(raw) + if err != nil { + return err + } + return src.acquireTask() +} + +// Release releases a task's reference to a source. Releasing the final task +// reference revokes the source. Sources that have never been acquired remain +// registered until Revoke or Registry.Close is called; session-backed sources +// are also bounded by the unclaimed-source TTL. +func (r *Registry) Release(raw string) error { + src, err := r.get(raw) + if err != nil { + return err + } + session, last, err := src.releaseTask() + if err != nil { + return err + } + if !last { + return nil + } + r.deleteSource(src) + if session != nil { + session.Release() + } + return nil +} + +// SourceError returns the first error encountered while opening or reading a +// source. Invalid and missing sources have no source error. HTTP response write +// errors are deliberately not recorded here. +func (r *Registry) SourceError(raw string) error { + src, err := r.get(raw) + if err != nil { + return nil + } + return src.sourceError() +} + func (r *Registry) Revoke(raw string) error { src, err := r.get(raw) if err != nil { @@ -194,13 +263,6 @@ func (r *Registry) Close() error { r.server = nil r.listener = nil r.baseURL = "" - r.serverMu.Unlock() - if server != nil { - _ = server.Close() - } - if listener != nil { - _ = listener.Close() - } r.mu.Lock() sources := make([]*Source, 0, len(r.sources)) for _, src := range r.sources { @@ -208,15 +270,22 @@ func (r *Registry) Close() error { } r.sources = make(map[string]*Source) r.mu.Unlock() + r.serverMu.Unlock() + if server != nil { + _ = server.Close() + } + if listener != nil { + _ = listener.Close() + } for _, src := range sources { src.close() } return nil } -func (r *Registry) ensureServer() (string, error) { - r.serverMu.Lock() - defer r.serverMu.Unlock() +// ensureServerLocked returns the current server URL or starts a new server. +// The caller must hold serverMu. +func (r *Registry) ensureServerLocked() (string, error) { if r.baseURL != "" { return r.baseURL, nil } @@ -252,11 +321,18 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.NotFound(w, req) return } - meta, open := src.snapshot() + meta, open, session := src.acquireOpen() if open == nil { http.NotFound(w, req) return } + if session != nil { + defer session.Release() + } + if src.sourceError() != nil { + http.Error(w, "blob source unavailable", http.StatusGone) + return + } start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range) if err != nil { http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) @@ -270,7 +346,16 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { End: end, }) if err != nil { - http.NotFound(w, req) + if req.Context().Err() == nil || !errors.Is(err, req.Context().Err()) { + src.recordSourceError(err, meta.Range) + } + http.Error(w, "blob source unavailable", http.StatusGone) + return + } + if reader == nil { + err = fmt.Errorf("%w: opener returned a nil reader", ErrSourceClosed) + src.recordSourceError(err, meta.Range) + http.Error(w, "blob source unavailable", http.StatusGone) return } defer reader.Close() @@ -279,24 +364,109 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { if ranged { w.WriteHeader(http.StatusPartialContent) } + limit := int64(-1) if end >= start { - _, _ = copyWithFlush(w, reader, end-start+1) - return + limit = end - start + 1 + } else if meta.Size > 0 { + limit = meta.Size + } + _, readErr, writeErr := copyWithFlush(w, reader, limit) + if readErr != nil { + if ctxErr := req.Context().Err(); ctxErr == nil || !errors.Is(readErr, ctxErr) { + src.recordSourceError(readErr, meta.Range) + } + } else if writeErr == nil && req.Context().Err() == nil { + src.recordSourceSuccess(meta.Range) } - _, _ = copyWithFlush(w, reader, -1) } -func (s *Source) snapshot() (Metadata, OpenFunc) { +func (s *Source) acquireOpen() (Metadata, OpenFunc, SessionRef) { s.mu.Lock() defer s.mu.Unlock() if s.revoked { - return Metadata{}, nil + return Metadata{}, nil, nil + } + if s.session != nil { + // The source's own retain guarantees that the session cannot become + // closed while this additional active-reader retain is acquired. + s.session.Retain() } return Metadata{ ContentType: s.ContentType, Size: s.size, Range: s.rangeEnabled, - }, s.open + }, s.open, s.session +} + +func (s *Source) acquireTask() error { + s.mu.Lock() + if s.revoked { + s.mu.Unlock() + return ErrSourceRevoked + } + s.taskRefs++ + timer := s.unclaimed + s.unclaimed = nil + s.mu.Unlock() + if timer != nil { + timer.Stop() + } + return nil +} + +func (s *Source) releaseTask() (session SessionRef, last bool, err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.revoked { + return nil, false, ErrSourceRevoked + } + if s.taskRefs == 0 { + return nil, false, ErrSourceClosed + } + s.taskRefs-- + if s.taskRefs > 0 { + return nil, false, nil + } + s.revoked = true + s.open = nil + session = s.session + s.session = nil + return session, true, nil +} + +func (s *Source) sourceError() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.readErr +} + +func (s *Source) recordSourceError(err error, ranged bool) { + if err == nil { + return + } + s.mu.Lock() + if s.readErr == nil && ranged { + s.rangeFailures++ + if s.rangeFailures < rangeSourceFailureLimit { + s.mu.Unlock() + return + } + } + if s.readErr == nil { + s.readErr = err + } + s.mu.Unlock() +} + +func (s *Source) recordSourceSuccess(ranged bool) { + if !ranged { + return + } + s.mu.Lock() + if s.readErr == nil { + s.rangeFailures = 0 + } + s.mu.Unlock() } func (s *Source) close() { @@ -306,19 +476,53 @@ func (s *Source) close() { s.open = nil session := s.session s.session = nil + timer := s.unclaimed + s.unclaimed = nil s.mu.Unlock() + if timer != nil { + timer.Stop() + } if !alreadyRevoked && session != nil { session.Release() } } +func (s *Source) expireUnclaimed() (SessionRef, bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.unclaimed = nil + if s.revoked || s.taskRefs > 0 { + return nil, false + } + s.revoked = true + s.open = nil + session := s.session + s.session = nil + return session, true +} + +func (r *Registry) expireUnclaimed(src *Source) { + session, expired := src.expireUnclaimed() + if !expired { + return + } + r.deleteSource(src) + if session != nil { + session.Release() + } +} + func (r *Registry) removeSource(src *Source) { + src.close() + r.deleteSource(src) +} + +func (r *Registry) deleteSource(src *Source) { r.mu.Lock() if current := r.sources[src.ID]; current == src { delete(r.sources, src.ID) } r.mu.Unlock() - src.close() } func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) { @@ -338,15 +542,17 @@ func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged } } -func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (int64, error) { +func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (written int64, readErr error, writeErr error) { if limit >= 0 { reader = io.LimitReader(reader, limit) } buf := make([]byte, 32*1024) - var written int64 flusher, canFlush := w.(http.Flusher) for { nr, er := reader.Read(buf) + if er != nil && !errors.Is(er, io.EOF) { + readErr = er + } if nr > 0 { nw, ew := w.Write(buf[:nr]) if nw > 0 { @@ -356,17 +562,20 @@ func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (int64, } } if ew != nil { - return written, ew + return written, readErr, ew } if nr != nw { - return written, io.ErrShortWrite + return written, readErr, io.ErrShortWrite } } if er != nil { if errors.Is(er, io.EOF) { - return written, nil + if limit >= 0 && written < limit { + return written, io.ErrUnexpectedEOF, nil + } + return written, nil, nil } - return written, er + return written, readErr, nil } } } diff --git a/internal/blob/registry_test.go b/internal/blob/registry_test.go index db23aa7e8..912daa626 100644 --- a/internal/blob/registry_test.go +++ b/internal/blob/registry_test.go @@ -9,8 +9,85 @@ import ( "strings" "sync" "testing" + "time" ) +type terminalErrorReader struct { + err error +} + +func (r *terminalErrorReader) Read([]byte) (int, error) { + return 0, r.err +} + +type contextErrorReader struct { + ctx context.Context +} + +func (r *contextErrorReader) Read([]byte) (int, error) { + <-r.ctx.Done() + return 0, r.ctx.Err() +} + +func (r *contextErrorReader) Close() error { return nil } + +type failingResponseWriter struct { + header http.Header + err error +} + +func (w *failingResponseWriter) Header() http.Header { + return w.header +} + +func (w *failingResponseWriter) Write([]byte) (int, error) { + return 0, w.err +} + +func (w *failingResponseWriter) WriteHeader(int) {} + +type countingSession struct { + mu sync.Mutex + refs int + zero chan struct{} + zeroOnce sync.Once +} + +func newCountingSession() *countingSession { + return &countingSession{zero: make(chan struct{})} +} + +func (s *countingSession) Retain() { + s.mu.Lock() + s.refs++ + s.mu.Unlock() +} + +func (s *countingSession) Release() { + s.mu.Lock() + s.refs-- + refs := s.refs + s.mu.Unlock() + if refs == 0 { + s.zeroOnce.Do(func() { close(s.zero) }) + } +} + +func (s *countingSession) RefCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.refs +} + +func setUnclaimedSourceTTLForTest(t *testing.T, ttl time.Duration) { + t.Helper() + previous := unclaimedSourceTTL + unclaimedSourceTTL = ttl + t.Cleanup(func() { + unclaimedSourceTTL = previous + }) +} + func TestRegistryBlobHTTPGetAndRange(t *testing.T) { registry := NewRegistry(t.TempDir()) defer registry.Close() @@ -73,6 +150,42 @@ func TestRegistryBlobHTTPGetAndRange(t *testing.T) { } } +func TestRegistryEmptyBlob(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateBlob(nil, "application/octet-stream") + if err != nil { + t.Fatal(err) + } + meta, err := registry.Metadata(url) + if err != nil { + t.Fatal(err) + } + if meta.Size != 0 || meta.Range { + t.Fatalf("unexpected empty blob metadata: %#v", meta) + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil { + t.Fatal(readErr) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %d", resp.StatusCode) + } + if len(body) != 0 { + t.Fatalf("unexpected body: %q", body) + } + if got := resp.Header.Get("Content-Length"); got != "0" { + t.Fatalf("unexpected content length: %q", got) + } +} + func TestRegistryOpenerSizeWithoutRange(t *testing.T) { registry := NewRegistry(t.TempDir()) defer registry.Close() @@ -218,6 +331,510 @@ func TestRegistryRejectsRangeWithoutSize(t *testing.T) { } } +func TestRegistryTracksSourceErrors(t *testing.T) { + t.Run("reader error", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + boom := errors.New("reader failed") + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil + }, &CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil { + t.Fatalf("the HTTP transport should end normally, got %v", readErr) + } + if string(body) != "partial" { + t.Fatalf("unexpected partial body: %q", body) + } + if got := registry.SourceError(url); !errors.Is(got, boom) { + t.Fatalf("expected source error %v, got %v", boom, got) + } + }) + + t.Run("short known-size reader", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("short")), nil + }, &CreateOptions{Size: 10}) + if err != nil { + t.Fatal(err) + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) { + t.Fatalf("expected unexpected EOF, got %v", got) + } + }) + + t.Run("range reader error becomes terminal", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + boom := errors.New("range reader failed") + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil + }, &CreateOptions{Size: 10, Range: true}) + if err != nil { + t.Fatal(err) + } + + for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-9") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + } + if got := registry.SourceError(url); !errors.Is(got, boom) { + t.Fatalf("expected range source error %v, got %v", boom, got) + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusGone { + t.Fatalf("expected terminal source status %d, got %d", http.StatusGone, resp.StatusCode) + } + }) + + t.Run("short range reader", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("short")), nil + }, &CreateOptions{Size: 10, Range: true}) + if err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-9") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + } + if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) { + t.Fatalf("expected range unexpected EOF, got %v", got) + } + }) + + t.Run("successful range retry clears transient failure", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + attempt := 0 + boom := errors.New("transient range failure") + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + attempt++ + if attempt == 1 { + return io.NopCloser(&terminalErrorReader{err: boom}), nil + } + return io.NopCloser(strings.NewReader("0123456789")), nil + }, &CreateOptions{Size: 10, Range: true}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-9") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + _ = resp.Body.Close() + } + if got := registry.SourceError(url); got != nil { + t.Fatalf("successful retry left transient source error: %v", got) + } + }) + + t.Run("request cancellation is not a source error", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + opened := make(chan struct{}) + url, err := registry.CreateOpener(func(ctx context.Context, _ OpenRequest) (io.ReadCloser, error) { + close(opened) + return &contextErrorReader{ctx: ctx}, nil + }, &CreateOptions{}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + resp, _ := http.DefaultClient.Do(req) + if resp != nil { + _ = resp.Body.Close() + } + close(done) + }() + <-opened + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("cancelled request did not finish") + } + if got := registry.SourceError(url); got != nil { + t.Fatalf("request cancellation was recorded as source error: %v", got) + } + }) + + t.Run("nil reader", func(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return nil, nil + }, &CreateOptions{}) + if err != nil { + t.Fatal(err) + } + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusGone { + t.Fatalf("expected nil reader status %d, got %d", http.StatusGone, resp.StatusCode) + } + if got := registry.SourceError(url); !errors.Is(got, ErrSourceClosed) { + t.Fatalf("expected closed source error, got %v", got) + } + }) +} + +func TestRegistryDoesNotRecordResponseWriteError(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("payload")), nil + }, &CreateOptions{Size: 7}) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + writerErr := errors.New("client connection closed") + registry.ServeHTTP(&failingResponseWriter{header: make(http.Header), err: writerErr}, req) + if got := registry.SourceError(url); got != nil { + t.Fatalf("response write error was recorded as a source error: %v", got) + } +} + +func TestRegistryTaskReferencesShareSource(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + url, err := registry.CreateBlob([]byte("shared"), "text/plain") + if err != nil { + t.Fatal(err) + } + if err := registry.Acquire(url); err != nil { + t.Fatal(err) + } + if err := registry.Acquire(url); err != nil { + t.Fatal(err) + } + if err := registry.Release(url); err != nil { + t.Fatal(err) + } + if !registry.IsURL(url) { + t.Fatal("the first task release revoked a source still used by another task") + } + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil || string(body) != "shared" { + t.Fatalf("unexpected shared response body %q, error %v", body, readErr) + } + + if err := registry.Release(url); err != nil { + t.Fatal(err) + } + if registry.IsURL(url) { + t.Fatal("the final task release did not revoke the source") + } +} + +func TestRegistryUnclaimedSessionSourceExpires(t *testing.T) { + setUnclaimedSourceTTLForTest(t, 25*time.Millisecond) + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + session := newCountingSession() + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("unused")), nil + }, &CreateOptions{Size: 6, Session: session}) + if err != nil { + t.Fatal(err) + } + if got := session.RefCount(); got != 1 { + t.Fatalf("unexpected initial session ref count: %d", got) + } + + select { + case <-session.zero: + case <-time.After(2 * time.Second): + t.Fatal("unclaimed source did not expire") + } + if registry.IsURL(url) { + t.Fatal("expired source is still registered") + } + if got := session.RefCount(); got != 0 { + t.Fatalf("expired source retained its session: %d", got) + } +} + +func TestRegistryAcquireStopsUnclaimedSourceExpiration(t *testing.T) { + const ttl = 25 * time.Millisecond + setUnclaimedSourceTTLForTest(t, ttl) + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + session := newCountingSession() + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("claimed")), nil + }, &CreateOptions{Size: 7, Session: session}) + if err != nil { + t.Fatal(err) + } + if err := registry.Acquire(url); err != nil { + t.Fatal(err) + } + + select { + case <-session.zero: + t.Fatal("claimed source expired") + case <-time.After(4 * ttl): + } + if !registry.IsURL(url) { + t.Fatal("claimed source was removed") + } + if err := registry.Release(url); err != nil { + t.Fatal(err) + } + select { + case <-session.zero: + case <-time.After(2 * time.Second): + t.Fatal("final task release did not release the session") + } +} + +func TestRegistryUnclaimedExpirationKeepsActiveReaderSession(t *testing.T) { + setUnclaimedSourceTTLForTest(t, 100*time.Millisecond) + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + session := newCountingSession() + reader, writer := io.Pipe() + opened := make(chan struct{}) + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + close(opened) + return reader, nil + }, &CreateOptions{Size: 7, Session: session}) + if err != nil { + t.Fatal(err) + } + + type result struct { + body []byte + err error + } + resultCh := make(chan result, 1) + go func() { + resp, err := http.Get(url) + if err != nil { + resultCh <- result{err: err} + return + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + resultCh <- result{body: body, err: err} + }() + + select { + case <-opened: + case got := <-resultCh: + t.Fatalf("request ended before the opener ran: body %q, error %v", got.body, got.err) + case <-time.After(2 * time.Second): + t.Fatal("active request did not open") + } + deadline := time.NewTimer(2 * time.Second) + ticker := time.NewTicker(5 * time.Millisecond) + defer deadline.Stop() + defer ticker.Stop() + for registry.IsURL(url) { + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("unclaimed source did not expire while its reader was active") + } + } + if got := session.RefCount(); got != 1 { + t.Fatalf("expiration released the active-reader session ref: %d", got) + } + if _, err := writer.Write([]byte("payload")); err != nil { + t.Fatal(err) + } + _ = writer.Close() + + select { + case got := <-resultCh: + if got.err != nil || string(got.body) != "payload" { + t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err) + } + case <-time.After(2 * time.Second): + t.Fatal("active response did not finish after source expiration") + } + select { + case <-session.zero: + if got := session.RefCount(); got != 0 { + t.Fatalf("unexpected final session ref count: %d", got) + } + case <-time.After(2 * time.Second): + t.Fatal("active-reader session ref was not released") + } +} + +func TestRegistryCloseCancelsUnclaimedExpiration(t *testing.T) { + const ttl = 25 * time.Millisecond + setUnclaimedSourceTTLForTest(t, ttl) + registry := NewRegistry(t.TempDir()) + + session := newCountingSession() + _, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("unused")), nil + }, &CreateOptions{Size: 6, Session: session}) + if err != nil { + t.Fatal(err) + } + if err := registry.Close(); err != nil { + t.Fatal(err) + } + select { + case <-session.zero: + case <-time.After(2 * time.Second): + t.Fatal("registry close did not release the session") + } + <-time.After(4 * ttl) + if got := session.RefCount(); got != 0 { + t.Fatalf("expiration released the session again after close: %d", got) + } +} + +func TestRegistryRevokeKeepsActiveReaderSession(t *testing.T) { + registry := NewRegistry(t.TempDir()) + defer registry.Close() + + session := newCountingSession() + reader, writer := io.Pipe() + opened := make(chan struct{}) + url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) { + close(opened) + return reader, nil + }, &CreateOptions{Size: 7, Session: session}) + if err != nil { + t.Fatal(err) + } + + type result struct { + body []byte + err error + } + resultCh := make(chan result, 1) + go func() { + resp, err := http.Get(url) + if err != nil { + resultCh <- result{err: err} + return + } + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + resultCh <- result{body: body, err: err} + }() + + <-opened + if got := session.RefCount(); got != 2 { + t.Fatalf("expected source and active-reader session refs, got %d", got) + } + if err := registry.Revoke(url); err != nil { + t.Fatal(err) + } + if got := session.RefCount(); got != 1 { + t.Fatalf("revoke released the active-reader session ref: %d", got) + } + if _, err := writer.Write([]byte("payload")); err != nil { + t.Fatal(err) + } + _ = writer.Close() + + select { + case got := <-resultCh: + if got.err != nil || string(got.body) != "payload" { + t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err) + } + case <-time.After(2 * time.Second): + t.Fatal("active response did not finish after revoke") + } + select { + case <-session.zero: + if got := session.RefCount(); got != 0 { + t.Fatalf("unexpected final session ref count: %d", got) + } + case <-time.After(2 * time.Second): + t.Fatal("active-reader session ref was not released") + } +} + func TestRegistryHTTPErrorStatuses(t *testing.T) { registry := NewRegistry(t.TempDir()) defer registry.Close() diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index bf43cfcfd..a9ed14ad6 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -354,57 +354,26 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { // Record connection time as baseline for fast-fail timeout f.updateMaxConnTime(time.Since(connStartTime)) + // Parse response to get resource info + res := &base.Resource{ + Range: false, + Files: []*base.FileInfo{}, + } + if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { resp.Body.Close() f.setState(stateError) return NewRequestError(resp.StatusCode) } - res := parseResourceFromResponse(httpReq, resp) - f.meta.Res = res - - // Save redirect URL for later connections - f.redirectURL = resp.Request.URL.String() - - // IMPORTANT: Keep the response body open for downloading in Start phase - // This is crucial for one-time URLs that can only be accessed once - f.resolveRespLock.Lock() - f.resolveResp = resp - f.resolveRespLock.Unlock() - - f.setState(stateResolved) - - // Signal that resolve is complete - f.resolvedOnce.Do(func() { - close(f.resolvedCh) - }) - - // Start async prefetch in background (only for range-supported resources) - // For non-range resources, the response will be used directly in Start - if res.Range && res.Size > 0 { - f.prefetchStopCh = make(chan struct{}) - go f.asyncPrefetch() - } - - // If start was called before resolve completed, auto-start - if f.startPending.Load() { - go f.doStart() - } - - return nil -} - -func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base.Resource { - res := &base.Resource{ - Range: false, - Files: []*base.FileInfo{}, - } + // Check if server supports range requests acceptRanges := resp.Header.Get(base.HttpHeaderAcceptRanges) contentRange := resp.Header.Get(base.HttpHeaderContentRange) if acceptRanges == base.HttpHeaderBytes || strings.HasPrefix(contentRange, base.HttpHeaderBytes) { res.Range = true } + // Get content length from Content-Length header contentLength := resp.Header.Get(base.HttpHeaderContentLength) if contentLength != "" { parse, err := strconv.ParseInt(contentLength, 10, 64) @@ -413,6 +382,7 @@ func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base } } + // Parse last modified time var lastModifiedTime *time.Time lastModified := resp.Header.Get(base.HttpHeaderLastModified) if lastModified != "" { @@ -425,6 +395,7 @@ func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base Ctime: lastModifiedTime, } + // Parse filename contentDisposition := resp.Header.Get(base.HttpHeaderContentDisposition) if contentDisposition != "" { file.Name = parseFilename(contentDisposition) @@ -432,6 +403,7 @@ func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base if file.Name == "" { file.Name = path.Base(httpReq.URL.Path) if file.Name != "" { + // Use PathUnescape instead of QueryUnescape to correctly handle %2B (should decode to +, not space) file.Name, _ = url.PathUnescape(file.Name) } } @@ -440,7 +412,37 @@ func parseResourceFromResponse(httpReq *http.Request, resp *http.Response) *base } res.Files = append(res.Files, file) - return res + f.meta.Res = res + + // Save redirect URL for later connections + f.redirectURL = resp.Request.URL.String() + + // IMPORTANT: Keep the response body open for downloading in Start phase + // This is crucial for one-time URLs that can only be accessed once + f.resolveRespLock.Lock() + f.resolveResp = resp + f.resolveRespLock.Unlock() + + f.setState(stateResolved) + + // Signal that resolve is complete + f.resolvedOnce.Do(func() { + close(f.resolvedCh) + }) + + // Start async prefetch in background (only for range-supported resources) + // For non-range resources, the response will be used directly in Start + if res.Range && res.Size > 0 { + f.prefetchStopCh = make(chan struct{}) + go f.asyncPrefetch() + } + + // If start was called before resolve completed, auto-start + if f.startPending.Load() { + go f.doStart() + } + + return nil } // asyncPrefetch downloads data in background during resolve phase @@ -574,12 +576,6 @@ func (f *Fetcher) Start() error { switch state { case stateResolved, statePaused: - if state == statePaused { - select { - case <-f.doneCh: - default: - } - } // Normal case: resolved or resuming from pause return f.doStart() @@ -674,11 +670,6 @@ func (f *Fetcher) doStart() error { if err = base.ParseReqExtra[fhttp.ReqExtra](f.meta.Req); err != nil { return err } - f.redirectLock.Lock() - if strings.Contains(f.redirectURL, "/__blob/") && f.redirectURL != f.meta.Req.URL { - f.redirectURL = "" - } - f.redirectLock.Unlock() // Initialize slow start controller maxConns := f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections @@ -1718,10 +1709,6 @@ func (f *Fetcher) Pause() error { f.fileMu.Unlock() f.setState(statePaused) - select { - case f.doneCh <- context.Canceled: - default: - } return nil } diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 4059a3575..00a1e1183 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -100,7 +100,6 @@ type Downloader struct { tasks []*Task waitTasks []*Task watchedTasks sync.Map - watchWG sync.WaitGroup listener Listener lock *sync.Mutex @@ -195,9 +194,6 @@ func (d *Downloader) Setup() error { tasks = append(tasks[:i], tasks[i+1:]...) continue } - if task.Meta.Req != nil && isInternalBlobURLString(task.Meta.Req.URL) && task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError { - task.Status = base.DownloadStatusError - } d.assignFetcherManager(task) initTask(task) if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError { @@ -246,13 +242,18 @@ func (d *Downloader) Setup() error { if len(d.tasks) > 0 { for _, task := range d.tasks { func() { + // Do not acquire d.lock (via GetTask) while holding + // statusLock; scheduling uses the opposite lock order. + if d.GetTask(task.ID) == nil { + return + } task.statusLock.Lock() - defer task.statusLock.Unlock() if task.Status != base.DownloadStatusRunning && !task.Uploading { + task.statusLock.Unlock() return } - // check if task is deleted - if d.GetTask(task.ID) == nil || task.fetcher == nil { + if task.fetcher == nil { + task.statusLock.Unlock() return } @@ -274,6 +275,8 @@ func (d *Downloader) Setup() error { task.Progress.UploadSpeed = task.updateUploadSpeed(currentUploaded-task.Progress.Uploaded, tick) task.Progress.Uploaded = currentUploaded } + task.statusLock.Unlock() + // Listener callbacks may Pause/Continue and acquire statusLock. d.emit(EventKeyProgress, task) // store fetcher progress when download/upload data changed @@ -353,6 +356,12 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet } // Get proxy config, task request proxy config has higher priority, then use global proxy config ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) { + // Registry-backed blob URLs are loopback capability URLs and must never be + // sent through a configured proxy. Only exact, currently registered URLs + // take this path; everything else keeps normal HTTP proxy behavior. + if d.blob != nil && fetcher != nil && fetcher.Meta() != nil && fetcher.Meta().Req != nil && d.blob.IsURL(fetcher.Meta().Req.URL) { + return nil + } if requestProxy == nil { return d.cfg.Proxy.ToHandler() } @@ -537,34 +546,40 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err if task == nil { return ErrTaskNotFound } + if err := func() error { + task.lock.Lock() + defer task.lock.Unlock() - // Restore fetcher if not loaded - if task.fetcher == nil { - err := func() error { - task.statusLock.Lock() - defer task.statusLock.Unlock() + // Restore fetcher if not loaded + if task.fetcher == nil { + err := func() error { + task.statusLock.Lock() + defer task.statusLock.Unlock() - return d.restoreFetcher(task) - }() - if err != nil { - return err + return d.restoreFetcher(task) + }() + if err != nil { + return err + } } - } - // Call the fetcher's Patch method - if err := task.fetcher.Patch(req, opts); err != nil { - return err - } + // Call the fetcher's Patch method + if err := task.fetcher.Patch(req, opts); err != nil { + return err + } - // Update task meta from fetcher - task.Meta = task.fetcher.Meta() + // Update task meta from fetcher and transfer any live Blob lease. + task.Meta = task.fetcher.Meta() + if err := d.syncBlobTaskLease(task); err != nil { + return err + } - // Save task to storage - if err := d.saveTask(task); err != nil { + return d.saveTask(task) + }(); err != nil { return err } - // Emit progress event to notify listeners + // Emit outside task.lock so listeners may safely call back into Patch/Pause. d.emit(EventKeyProgress, task) return nil @@ -622,6 +637,7 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) { } realContinueTasks := make([]*Task, 0) + pausedTasks := make([]*Task, 0) func() { d.lock.Lock() defer d.lock.Unlock() @@ -634,12 +650,16 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) { pausedCount := 0 for _, task := range d.tasks { if task.Status == base.DownloadStatusRunning { - if err = d.doPause(task); err != nil { + var queued bool + queued, err = d.doPauseForScheduling(task) + if err != nil { return } - task.Status = base.DownloadStatusWait - d.waitTasks = append(d.waitTasks, task) - pausedCount++ + if queued { + d.waitTasks = append(d.waitTasks, task) + pausedTasks = append(pausedTasks, task) + pausedCount++ + } } if pausedCount == needPauseCount { break @@ -656,6 +676,14 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) { } } }() + if err != nil { + return + } + // Scheduling pauses run while d.lock is held, but events must not: listeners + // are allowed to call back into Downloader methods that acquire d.lock. + for _, task := range pausedTasks { + d.emit(EventKeyPause, task) + } for _, task := range realContinueTasks { if err = d.doStart(task); err != nil { @@ -1012,11 +1040,8 @@ func (d *Downloader) watch(task *Task) { if _, loaded := d.watchedTasks.LoadOrStore(task.ID, true); loaded { return } - d.watchWG.Add(1) - defer func() { d.watchedTasks.Delete(task.ID) - d.watchWG.Done() }() // wait task upload done @@ -1037,18 +1062,27 @@ func (d *Downloader) watch(task *Task) { } } - if task.Status == base.DownloadStatusDone { + if d.taskStatus(task) == base.DownloadStatusDone { return } for { err := task.fetcher.Wait() if err != nil { - if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait { + status := d.taskStatus(task) + if status == base.DownloadStatusPause || status == base.DownloadStatusWait { return } - d.doOnError(task, err) - if task.Status == base.DownloadStatusRunning { + var sourceErr error + if d.blob != nil && task.Meta != nil && task.Meta.Req != nil { + sourceErr = d.blob.SourceError(task.Meta.Req.URL) + } + if sourceErr != nil { + d.doOnBlobSourceError(task, sourceErr) + } else { + d.doOnError(task, err) + } + if d.taskStatus(task) == base.DownloadStatusRunning { continue } return @@ -1058,6 +1092,15 @@ func (d *Downloader) watch(task *Task) { if task.Meta.Res == nil || d.GetTask(task.ID) == nil { return } + if d.blob != nil && task.Meta.Req != nil { + if err := d.blob.SourceError(task.Meta.Req.URL); err != nil { + d.doOnBlobSourceError(task, err) + if d.taskStatus(task) == base.DownloadStatusRunning { + continue + } + return + } + } task.Progress.Used = task.timer.Used() if task.Meta.Res.Size == 0 { @@ -1070,7 +1113,9 @@ func (d *Downloader) watch(task *Task) { totalSize := task.Meta.Res.Size task.Progress.Speed = totalSize / used task.Progress.Downloaded = totalSize - task.updateStatus(base.DownloadStatusDone) + if !d.markTaskDone(task) { + return + } d.storage.Put(bucketTask, task.ID, task.clone()) d.emit(EventKeyDone, task) d.emit(EventKeyFinally, task, err) @@ -1141,10 +1186,61 @@ func (d *Downloader) watch(task *Task) { } func (d *Downloader) doOnError(task *Task, err error) { + d.handleOnError(task, err, false) +} + +func (d *Downloader) doOnBlobSourceError(task *Task, err error) { + d.handleOnError(task, err, true) +} + +func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) { + if task == nil || task.lock == nil { + return + } + task.lock.Lock() + if !d.markTaskError(task) { + task.lock.Unlock() + return + } d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID) - task.updateStatus(base.DownloadStatusError) - d.triggerOnError(task, err) - if task.Status == base.DownloadStatusError { + oldURL := "" + if task.Meta != nil && task.Meta.Req != nil { + oldURL = task.Meta.Req.URL + } + extTask := d.triggerOnError(task, err) + newURL := "" + if task.Meta != nil && task.Meta.Req != nil { + newURL = task.Meta.Req.URL + } + if oldURL != "" && newURL != oldURL { + // Extensions are allowed to replace an expired URL from onError. Patch the + // existing fetcher through its public API so protocol-owned redirect state + // is reset without teaching the HTTP fetcher about blob URLs. + if task.fetcher != nil { + if patchErr := task.fetcher.Patch(&base.Request{URL: newURL}, nil); patchErr != nil { + d.Logger.Warn().Err(patchErr).Msgf("patch recovered task url failed, task id: %s", task.ID) + extTask.continueRequested = false + } + } + d.releaseBlobTaskURL(task, oldURL) + } + if resetFetcher && extTask.continueRequested { + if resetErr := d.resetTaskFetcher(task); resetErr != nil { + d.Logger.Warn().Err(resetErr).Msgf("reset recovered task fetcher failed, task id: %s", task.ID) + extTask.continueRequested = false + } + } + task.lock.Unlock() + if extTask.pauseRequested { + if pauseErr := d.Pause(&TaskFilter{IDs: []string{task.ID}}); pauseErr != nil { + d.Logger.Warn().Err(pauseErr).Msgf("extension pause task failed, task id: %s", task.ID) + } + } else if extTask.continueRequested { + if continueErr := d.Continue(&TaskFilter{IDs: []string{task.ID}}); continueErr != nil { + d.Logger.Warn().Err(continueErr).Msgf("extension continue task failed, task id: %s", task.ID) + } + } + if d.taskStatus(task) == base.DownloadStatusError { d.releaseBlobTask(task) d.emit(EventKeyError, task, err) d.emit(EventKeyFinally, task, err) @@ -1153,6 +1249,37 @@ func (d *Downloader) doOnError(task *Task, err error) { } } +func (d *Downloader) resetTaskFetcher(task *Task) error { + if task == nil || task.Meta == nil || task.Meta.Req == nil { + return errors.New("task request is unavailable") + } + fm, err := d.parseFm(task.Meta.Req.URL) + if err != nil { + return err + } + opts := task.Meta.Opts + f := fm.Build() + d.setupFetcher(fm, f) + // A fresh source must never inherit bytes or preallocation from the failed + // source. HTTP opens existing targets without truncating them, so remove the + // old single-file target before installing a fresh fetcher. + if task.Meta.Res != nil && task.Meta.Opts != nil && len(task.Meta.Res.Files) > 0 { + if err := util.SafeRemove(task.Meta.SingleFilepath()); err != nil { + return err + } + } + f.Meta().Req = task.Meta.Req + f.Meta().Opts = opts + task.fetcherManager = fm + task.fetcher = f + task.Meta = f.Meta() + if task.Progress != nil { + task.Progress.Downloaded = 0 + task.Progress.Speed = 0 + } + return d.storage.Delete(bucketSave, task.ID) +} + func (d *Downloader) restoreTask(task *Task) error { if task.fetcher == nil { if err := d.restoreFetcher(task); err != nil { @@ -1205,6 +1332,14 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str task.Progress = &Progress{} _, task.Uploading = f.(fetcher.Uploader) initTask(task) + if err = d.syncBlobTaskLease(task); err != nil { + return "", err + } + defer func() { + if err != nil { + d.releaseBlobTask(task) + } + }() if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil { return } @@ -1271,6 +1406,7 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error func (d *Downloader) doStart(task *Task) (err error) { var isCreate bool + var generation uint64 isReturn, err := d.statusMut(task, func() (isReturn bool, err error) { if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone { isReturn = true @@ -1284,6 +1420,8 @@ func (d *Downloader) doStart(task *Task) (err error) { } isCreate = task.Status == base.DownloadStatusReady task.updateStatus(base.DownloadStatusRunning) + task.runGeneration++ + generation = task.runGeneration return }) @@ -1295,24 +1433,37 @@ func (d *Downloader) doStart(task *Task) (err error) { return } + started := false + pauseBeforeStart := false handler := func() error { task.lock.Lock() defer task.lock.Unlock() + if !d.taskIsRunningGeneration(task, generation) { + return nil + } if task.fetcher == nil { if err := d.restoreFetcher(task); err != nil { return err } } - d.triggerOnStart(task) + extTask := d.triggerOnStart(task) + // onStart may synchronously request a pause through ExtensionTask. + // Do not resolve or start a task after that request changed its state. + if !d.taskIsRunningGeneration(task, generation) { + return nil + } if task.fetcher != nil { task.fetcher.Meta().Req = task.Meta.Req task.fetcher.Meta().Res = task.Meta.Res task.fetcher.Meta().Opts = task.Meta.Opts } - if task.Meta.Req != nil && isInternalBlobURLString(task.Meta.Req.URL) && (d.blob == nil || !d.blob.IsURL(task.Meta.Req.URL)) { - task.fetcher = nil - return internalblob.ErrSourceNotFound + if err := d.syncBlobTaskLease(task); err != nil { + return err + } + if extTask.pauseRequested { + pauseBeforeStart = true + return nil } if task.Meta.Res == nil { err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts) @@ -1351,20 +1502,38 @@ func (d *Downloader) doStart(task *Task) (err error) { } task.Progress.Speed = 0 + if !d.taskIsRunningGeneration(task, generation) { + return nil + } task.timer.Start() if err := task.fetcher.Start(); err != nil { return err } + started = true if err := d.saveTask(task); err != nil { return err } - go d.watch(task) - d.emit(EventKeyStart, task) return nil } go func() { if err := handler(); err != nil { - d.doOnError(task, err) + if d.taskIsRunningGeneration(task, generation) { + d.doOnError(task, err) + } + return + } + if pauseBeforeStart { + if err := d.doPauseBeforeStart(task); err != nil { + d.Logger.Error().Stack().Err(err).Msgf("onStart pause task failed, task id: %s", task.ID) + } + d.notifyRunning() + return + } + if started { + if d.taskIsRunningGeneration(task, generation) { + d.emit(EventKeyStart, task) + } + go d.watch(task) } }() @@ -1372,52 +1541,152 @@ func (d *Downloader) doStart(task *Task) (err error) { } func (d *Downloader) doPause(task *Task) (err error) { - isReturn, err := d.statusMut(task, func() (isReturn bool, err error) { + generation, isReturn, err := d.preparePause(task) + if err != nil || isReturn { + return err + } + handled, err := d.runPauseHandler(task, generation, true) + if err != nil { + d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID) + return err + } + if handled { + d.emit(EventKeyPause, task) + } + return nil +} + +func (d *Downloader) doPauseBeforeStart(task *Task) (err error) { + generation, isReturn, err := d.preparePause(task) + if err != nil || isReturn { + return err + } + handled, err := d.runPauseHandler(task, generation, false) + if err != nil { + return err + } + if handled { + d.emit(EventKeyPause, task) + } + return nil +} + +// doPauseForScheduling physically pauses a running task without emitting while +// the caller holds d.lock, then atomically moves it to the wait queue state. +func (d *Downloader) doPauseForScheduling(task *Task) (bool, error) { + generation, isReturn, err := d.preparePause(task) + if err != nil || isReturn { + return false, err + } + handled, err := d.runPauseHandler(task, generation, true) + if err != nil || !handled { + return false, err + } + task.statusLock.Lock() + queued := task.Status == base.DownloadStatusPause && task.runGeneration == generation + if queued { + task.Status = base.DownloadStatusWait + } + task.statusLock.Unlock() + return queued, nil +} + +func (d *Downloader) preparePause(task *Task) (generation uint64, isReturn bool, err error) { + isReturn, err = d.statusMut(task, func() (isReturn bool, err error) { if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusDone { isReturn = true return } task.updateStatus(base.DownloadStatusPause) + task.runGeneration++ + generation = task.runGeneration task.timer.Pause() return }) if err != nil { d.Logger.Error().Stack().Err(err).Msgf("pause task failed, task id: %s", task.ID) - return - } - if isReturn { - return } + return generation, isReturn, err +} - handler := func() error { - task.lock.Lock() - defer task.lock.Unlock() +func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher bool) (bool, error) { + task.lock.Lock() + defer task.lock.Unlock() + if !d.taskIsPausedGeneration(task, generation) { + return false, nil + } - if task.fetcher != nil { - if err := task.fetcher.Pause(); err != nil { - return err - } + if pauseFetcher && task.fetcher != nil { + if err := task.fetcher.Pause(); err != nil { + return false, err } - d.releaseBlobTask(task) - if task.fetcherManager != nil && task.fetcher != nil { - if err := d.saveTask(task); err != nil { - return err - } - } else { - if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil { - return err - } - } - d.emit(EventKeyPause, task) - return nil } - go func() { - if err := handler(); err != nil { - d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID) + if task.fetcherManager != nil && task.fetcher != nil { + if err := d.saveTask(task); err != nil { + return false, err } - }() - return + } else { + if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil { + return false, err + } + } + return true, nil +} + +func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool { + if task == nil || task.statusLock == nil { + return false + } + task.statusLock.Lock() + defer task.statusLock.Unlock() + return task.Status == base.DownloadStatusRunning && task.runGeneration == generation +} + +func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool { + if task == nil || task.statusLock == nil { + return false + } + task.statusLock.Lock() + defer task.statusLock.Unlock() + return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation +} + +func (d *Downloader) taskStatus(task *Task) base.Status { + if task == nil || task.statusLock == nil { + return "" + } + task.statusLock.Lock() + defer task.statusLock.Unlock() + return task.Status +} + +func (d *Downloader) markTaskDone(task *Task) bool { + if task == nil || task.statusLock == nil { + return false + } + task.statusLock.Lock() + defer task.statusLock.Unlock() + if task.Status != base.DownloadStatusRunning { + return false + } + task.updateStatus(base.DownloadStatusDone) + task.runGeneration++ + return true +} + +func (d *Downloader) markTaskError(task *Task) bool { + if task == nil || task.statusLock == nil { + return false + } + task.statusLock.Lock() + defer task.statusLock.Unlock() + if task.Status != base.DownloadStatusRunning { + return false + } + task.updateStatus(base.DownloadStatusError) + task.runGeneration++ + return true } // redirect stderr to log file, when panic happened log it @@ -1834,26 +2103,77 @@ func initTask(task *Task) { task.statusLock = &sync.Mutex{} task.lock = &sync.Mutex{} + task.blobRefLock = &sync.Mutex{} task.speedArr = make([]int64, 0) task.uploadSpeedArr = make([]int64, 0) } -func (d *Downloader) revokeBlobTask(task *Task) { - if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" { +func (d *Downloader) releaseBlobTask(task *Task) { + if d.blob == nil || task == nil { return } - if !d.blob.IsURL(task.Meta.Req.URL) { - return + if task.blobRefLock != nil { + task.blobRefLock.Lock() + raw := task.blobURL + task.blobURL = "" + task.blobRefLock.Unlock() + if raw != "" { + _ = d.blob.Release(raw) + } } - _ = d.blob.Revoke(task.Meta.Req.URL) } -func (d *Downloader) releaseBlobTask(task *Task) { - d.revokeBlobTask(task) +func (d *Downloader) syncBlobTaskLease(task *Task) error { + if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil { + return nil + } + if task.blobRefLock == nil { + task.blobRefLock = &sync.Mutex{} + } + task.blobRefLock.Lock() + defer task.blobRefLock.Unlock() + terminal := false + if task.statusLock != nil { + task.statusLock.Lock() + terminal = task.Status == base.DownloadStatusDone || task.Status == base.DownloadStatusError + task.statusLock.Unlock() + } + newURL := "" + if !terminal && d.blob.IsURL(task.Meta.Req.URL) { + newURL = task.Meta.Req.URL + } + if task.blobURL == newURL { + return nil + } + if newURL != "" { + if err := d.blob.Acquire(newURL); err != nil { + return err + } + } + oldURL := task.blobURL + task.blobURL = newURL + if oldURL != "" { + _ = d.blob.Release(oldURL) + } + return nil } -func isInternalBlobURLString(raw string) bool { - return strings.Contains(raw, "/__blob/") +func (d *Downloader) releaseBlobTaskURL(task *Task, raw string) { + if d.blob == nil || task == nil || raw == "" { + return + } + if task.blobRefLock == nil { + task.blobRefLock = &sync.Mutex{} + } + task.blobRefLock.Lock() + claimed := task.blobURL == raw + if claimed { + task.blobURL = "" + } + task.blobRefLock.Unlock() + if claimed { + _ = d.blob.Release(raw) + } } var defaultDownloader = NewDownloader(nil) diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index ba30c59f4..a5dec36ba 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -2,18 +2,25 @@ package download import ( "archive/zip" + "bytes" + "context" + "errors" "fmt" "io" "net" gohttp "net/http" + "net/http/httptest" "os" "path/filepath" "strconv" "strings" "sync" + "sync/atomic" "testing" "time" + internalblob "github.com/GopeedLab/gopeed/internal/blob" + "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/test" "github.com/GopeedLab/gopeed/pkg/base" @@ -21,6 +28,95 @@ import ( "github.com/GopeedLab/gopeed/pkg/util" ) +type generationTestManager struct { + starts atomic.Int32 + pauses atomic.Int32 + holdOpen bool + resolveStarted chan struct{} + resolveRelease <-chan struct{} + resolveErr error + resolveOnce sync.Once + pauseStarted chan struct{} + pauseRelease <-chan struct{} + pauseOnce sync.Once +} + +func (m *generationTestManager) Name() string { return "generation" } +func (m *generationTestManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{{Type: fetcher.FilterTypeUrl, Pattern: "generation"}} +} +func (m *generationTestManager) Build() fetcher.Fetcher { + return &generationTestFetcher{manager: m, meta: &fetcher.FetcherMeta{}} +} +func (m *generationTestManager) ParseName(string) string { return "generation.bin" } +func (m *generationTestManager) AutoRename() bool { return false } +func (m *generationTestManager) DefaultConfig() any { return map[string]any{} } +func (m *generationTestManager) Store(fetcher.Fetcher) (any, error) { return nil, nil } +func (m *generationTestManager) Restore() (any, func(*fetcher.FetcherMeta, any) fetcher.Fetcher) { + return nil, func(meta *fetcher.FetcherMeta, _ any) fetcher.Fetcher { + return &generationTestFetcher{manager: m, meta: meta} + } +} +func (m *generationTestManager) Close() error { return nil } + +type generationTestFetcher struct { + manager *generationTestManager + meta *fetcher.FetcherMeta + done chan error +} + +func (f *generationTestFetcher) Setup(*controller.Controller) { + if f.done == nil { + f.done = make(chan error, 2) + } +} +func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error { + if f.manager.resolveStarted != nil { + f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) }) + } + if f.manager.resolveRelease != nil { + <-f.manager.resolveRelease + } + if f.manager.resolveErr != nil { + return f.manager.resolveErr + } + f.meta.Req = req + f.meta.Opts = opts + f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}} + return nil +} +func (f *generationTestFetcher) Start() error { + f.manager.starts.Add(1) + if !f.manager.holdOpen { + f.done <- nil + } + return nil +} +func (f *generationTestFetcher) Patch(req *base.Request, opts *base.Options) error { + if req != nil && req.URL != "" { + f.meta.Req.URL = req.URL + } + if opts != nil { + f.meta.Opts = opts + } + return nil +} +func (f *generationTestFetcher) Pause() error { + f.manager.pauses.Add(1) + if f.manager.pauseStarted != nil { + f.manager.pauseOnce.Do(func() { close(f.manager.pauseStarted) }) + } + if f.manager.pauseRelease != nil { + <-f.manager.pauseRelease + } + return nil +} +func (f *generationTestFetcher) Close() error { return nil } +func (f *generationTestFetcher) Stats() any { return nil } +func (f *generationTestFetcher) Meta() *fetcher.FetcherMeta { return f.meta } +func (f *generationTestFetcher) Progress() fetcher.Progress { return fetcher.Progress{1} } +func (f *generationTestFetcher) Wait() error { return <-f.done } + func newTestDownloadOpt() *base.Options { return &base.Options{ Path: test.Dir, @@ -124,6 +220,501 @@ func TestDownloader_BlobTaskKeepsConfiguredHTTPConnections(t *testing.T) { waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) } +func TestDownloader_ExternalBlobLikePathUsesNormalHTTP(t *testing.T) { + payload := []byte("external blob-like path") + server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) { + if r.URL.Path != "/__blob/file.bin" { + gohttp.NotFound(w, r) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + _, _ = w.Write(payload) + })) + defer server.Close() + + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + + dir := t.TempDir() + id, err := downloader.CreateDirect(&base.Request{URL: server.URL + "/__blob/file.bin"}, &base.Options{ + Path: dir, + Name: "external.bin", + }) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + got, err := os.ReadFile(filepath.Join(dir, "external.bin")) + if err != nil { + t.Fatal(err) + } + if string(got) != string(payload) { + t.Fatalf("unexpected external payload %q", string(got)) + } +} + +func TestDownloader_InternalBlobBypassesGlobalProxy(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloader.cfg.DownloaderStoreConfig.Proxy = &base.DownloaderProxyConfig{ + Enable: true, + Scheme: "http", + Host: "127.0.0.1:1", + } + + payload := []byte("direct blob") + blobURL, err := downloader.blob.CreateBlob(payload, "application/octet-stream") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{ + Path: dir, + Name: "direct.bin", + }) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + got, err := os.ReadFile(filepath.Join(dir, "direct.bin")) + if err != nil { + t.Fatal(err) + } + if string(got) != string(payload) { + t.Fatalf("unexpected blob payload %q", string(got)) + } +} + +func TestDownloader_RegistryMissUsesOrdinaryHTTPError(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + + liveURL, err := downloader.blob.CreateBlob([]byte("unused"), "text/plain") + if err != nil { + t.Fatal(err) + } + missingURL := liveURL[:len(liveURL)-1] + "x" + if downloader.blob.IsURL(missingURL) { + t.Fatal("test URL unexpectedly matched the Registry") + } + + errorCh := make(chan error, 1) + downloader.Listener(func(event *Event) { + if event.Key == EventKeyError { + select { + case errorCh <- event.Err: + default: + } + } + }) + _, err = downloader.CreateDirect(&base.Request{URL: missingURL}, &base.Options{ + Path: t.TempDir(), + Name: "missing.bin", + }) + if err != nil { + t.Fatalf("Registry miss was rejected before ordinary HTTP handling: %v", err) + } + select { + case got := <-errorCh: + if errors.Is(got, internalblob.ErrSourceNotFound) { + t.Fatalf("Registry miss surfaced Blob error instead of HTTP failure: %v", got) + } + if got == nil || !strings.Contains(got.Error(), "404") { + t.Fatalf("expected ordinary HTTP 404 failure, got %v", got) + } + case <-time.After(10 * time.Second): + t.Fatal("Registry miss did not reach the ordinary HTTP error path") + } +} + +func TestDownloader_TasksShareBlobLease(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloader.cfg.MaxRunning = 1 + + payload := bytes.Repeat([]byte("shared-blob-"), 8192) + allowLaterOpens := make(chan struct{}) + var opens atomic.Int32 + blobURL, err := downloader.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) { + if opens.Add(1) > 1 { + select { + case <-allowLaterOpens: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + start := req.Offset + end := int64(len(payload)) + if req.End >= start && req.End+1 < end { + end = req.End + 1 + } + return io.NopCloser(bytes.NewReader(payload[start:end])), nil + }, &internalblob.CreateOptions{Size: int64(len(payload))}) + if err != nil { + t.Fatal(err) + } + + dirA, dirB := t.TempDir(), t.TempDir() + idA, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirA, Name: "a.bin"}) + if err != nil { + t.Fatal(err) + } + idB, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirB, Name: "b.bin"}) + if err != nil { + t.Fatal(err) + } + + waitForTaskStatus(t, downloader, idA, base.DownloadStatusDone, 5*time.Second) + if err := downloader.Delete(&TaskFilter{IDs: []string{idA}}, false); err != nil { + t.Fatal(err) + } + if !downloader.blob.IsURL(blobURL) { + t.Fatal("deleting the first task revoked a Blob still leased by the second task") + } + close(allowLaterOpens) + waitForTaskStatus(t, downloader, idB, base.DownloadStatusDone, 5*time.Second) + if downloader.blob.IsURL(blobURL) { + t.Fatal("final task completion did not release the shared Blob") + } + for _, file := range []string{filepath.Join(dirA, "a.bin"), filepath.Join(dirB, "b.bin")} { + got, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("unexpected shared Blob content in %s", file) + } + } +} + +func TestDownloader_EmptyBlob(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + + blobURL, err := downloader.blob.CreateBlob(nil, "application/octet-stream") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dir, Name: "empty.bin"}) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + info, err := os.Stat(filepath.Join(dir, "empty.bin")) + if err != nil { + t.Fatal(err) + } + if info.Size() != 0 { + t.Fatalf("unexpected empty Blob size: %d", info.Size()) + } +} + +func TestDownloader_PatchTransfersBlobLease(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + // Keep the task waiting so Patch exercises only metadata and lease transfer. + downloader.cfg.MaxRunning = 0 + + urlA, err := downloader.blob.CreateBlob([]byte("a"), "text/plain") + if err != nil { + t.Fatal(err) + } + urlB, err := downloader.blob.CreateBlob([]byte("b"), "text/plain") + if err != nil { + t.Fatal(err) + } + id, err := downloader.CreateDirect(&base.Request{URL: urlA}, &base.Options{Path: t.TempDir(), Name: "patched.bin"}) + if err != nil { + t.Fatal(err) + } + if err := downloader.Patch(id, &base.Request{URL: urlB}, nil); err != nil { + t.Fatal(err) + } + if downloader.blob.IsURL(urlA) { + t.Fatal("Patch retained the old Blob lease") + } + if !downloader.blob.IsURL(urlB) { + t.Fatal("Patch did not acquire the new Blob lease") + } + if err := downloader.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil { + t.Fatal(err) + } + if downloader.blob.IsURL(urlB) { + t.Fatal("deleting patched task did not release the new Blob lease") + } +} + +func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) { + manager := &generationTestManager{} + downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloader.cfg.MaxRunning = 0 + + id, err := downloader.CreateDirect(&base.Request{URL: "generation://test"}, &base.Options{ + Path: t.TempDir(), + Name: "generation.bin", + }) + if err != nil { + t.Fatal(err) + } + task := downloader.GetTask(id) + if task == nil || task.Status != base.DownloadStatusWait { + t.Fatalf("expected queued task, got %#v", task) + } + + // Hold task.lock so the first Start, its Pause handler, and the second Start + // are all queued. Only the newest start generation may reach Fetcher.Start. + task.lock.Lock() + downloader.cfg.MaxRunning = 1 + if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + pauseDone := make(chan error, 1) + go func() { + pauseDone <- downloader.doPauseBeforeStart(task) + }() + deadline := time.Now().Add(2 * time.Second) + for downloader.taskStatus(task) != base.DownloadStatusPause && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if downloader.taskStatus(task) != base.DownloadStatusPause { + t.Fatal("deferred pre-start Pause did not update task status") + } + if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + task.lock.Unlock() + select { + case err := <-pauseDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("deferred pre-start Pause did not finish") + } + + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 3*time.Second) + if got := manager.starts.Load(); got != 1 { + t.Fatalf("stale start generation reached Fetcher.Start: got %d starts, want 1", got) + } +} + +func TestExtensionTaskDefersOnStartPauseUntilTaskUnlock(t *testing.T) { + downloader := NewDownloader(nil) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloader.cfg.MaxRunning = 0 + + blobURL, err := downloader.blob.CreateBlob([]byte("pause"), "text/plain") + if err != nil { + t.Fatal(err) + } + id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{ + Path: t.TempDir(), + Name: "pause.bin", + }) + if err != nil { + t.Fatal(err) + } + liveTask := downloader.GetTask(id) + if liveTask == nil { + t.Fatal("task not found") + } + patched := make(chan error, 1) + downloader.Listener(func(event *Event) { + if event.Key == EventKeyPause && event.Task.ID == id { + patched <- downloader.Patch(id, &base.Request{Labels: map[string]string{"paused": "true"}}, nil) + } + }) + + // doStart invokes onStart while holding this lock. Record the action on the + // extension wrapper and apply it only after the handler unlocks. + extTask := NewExtensionTask(downloader, liveTask) + extTask.deferActions = true + liveTask.lock.Lock() + err = extTask.Pause() + liveTask.lock.Unlock() + if err != nil { + t.Fatal(err) + } + if !extTask.pauseRequested { + t.Fatal("onStart Pause was not deferred") + } + if err := downloader.doPauseBeforeStart(liveTask); err != nil { + t.Fatal(err) + } + liveTask.statusLock.Lock() + status := liveTask.Status + liveTask.statusLock.Unlock() + if status != base.DownloadStatusPause { + t.Fatalf("extension pause did not update live task: %s", status) + } + select { + case err := <-patched: + if err != nil { + t.Fatalf("Pause listener could not re-enter Patch: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Pause listener deadlocked while re-entering Patch") + } +} + +func TestDownloader_PauseWinsAgainstStaleResolveError(t *testing.T) { + releaseResolve := make(chan struct{}) + manager := &generationTestManager{ + resolveStarted: make(chan struct{}), + resolveRelease: releaseResolve, + resolveErr: errors.New("stale resolve failed"), + } + downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + + errorEvent := make(chan error, 1) + downloader.Listener(func(event *Event) { + if event.Key == EventKeyError { + errorEvent <- event.Err + } + }) + id, err := downloader.CreateDirect(&base.Request{URL: "generation://stale-resolve"}, &base.Options{ + Path: t.TempDir(), + Name: "stale.bin", + }) + if err != nil { + t.Fatal(err) + } + select { + case <-manager.resolveStarted: + case <-time.After(2 * time.Second): + t.Fatal("Resolve did not start") + } + + pauseDone := make(chan error, 1) + go func() { + pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}}) + }() + deadline := time.Now().Add(2 * time.Second) + for downloader.taskStatus(downloader.GetTask(id)) != base.DownloadStatusPause && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause { + t.Fatalf("Pause did not win the status transition: %s", got) + } + close(releaseResolve) + select { + case err := <-pauseDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("Pause did not finish after stale Resolve returned") + } + if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause { + t.Fatalf("stale Resolve error overwrote Pause: %s", got) + } + select { + case err := <-errorEvent: + t.Fatalf("stale Resolve emitted onError after Pause: %v", err) + case <-time.After(100 * time.Millisecond): + } +} + +func TestDownloader_SchedulingPauseCompletesBeforeReplacementStart(t *testing.T) { + releasePause := make(chan struct{}) + manager := &generationTestManager{ + holdOpen: true, + pauseStarted: make(chan struct{}), + pauseRelease: releasePause, + } + downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloader.cfg.MaxRunning = 1 + + _, err := downloader.CreateDirect(&base.Request{URL: "generation://a"}, &base.Options{ + Path: t.TempDir(), Name: "a.bin", + }) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for manager.starts.Load() != 1 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if manager.starts.Load() != 1 { + t.Fatal("first task did not physically start") + } + idB, err := downloader.CreateDirect(&base.Request{URL: "generation://b"}, &base.Options{ + Path: t.TempDir(), Name: "b.bin", + }) + if err != nil { + t.Fatal(err) + } + if downloader.taskStatus(downloader.GetTask(idB)) != base.DownloadStatusWait { + t.Fatal("second task was not queued") + } + + continueDone := make(chan error, 1) + go func() { + continueDone <- downloader.Continue(&TaskFilter{IDs: []string{idB}}) + }() + select { + case <-manager.pauseStarted: + case <-time.After(2 * time.Second): + t.Fatal("running task was not physically paused") + } + if got := manager.starts.Load(); got != 1 { + t.Fatalf("replacement started before Pause completed: %d starts", got) + } + close(releasePause) + select { + case err := <-continueDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("Continue did not return after Pause completed") + } + deadline = time.Now().Add(2 * time.Second) + for manager.starts.Load() != 2 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := manager.starts.Load(); got != 2 { + t.Fatalf("replacement task did not start: %d starts", got) + } + if got := manager.pauses.Load(); got != 1 { + t.Fatalf("unexpected physical Pause count: %d", got) + } +} + func TestDownloader_Create(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() diff --git a/pkg/download/engine/inject/stream/module.go b/pkg/download/engine/inject/stream/module.go index 6fff322a9..65af2d6b0 100644 --- a/pkg/download/engine/inject/stream/module.go +++ b/pkg/download/engine/inject/stream/module.go @@ -30,6 +30,7 @@ type Config struct { CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error) RevokeObjectURL func(url string) error ProxyHandler func(r *http.Request) (*url.URL, error) + RegisterCleanup func(cleanup func()) } type ObjectURLOptions struct { @@ -96,6 +97,9 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error return err } fetchRegistry := newFetchRegistry() + if cfg.RegisterCleanup != nil { + cfg.RegisterCleanup(fetchRegistry.CloseAll) + } if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value { reqMeta, err := exportFetchRequest(runtime, call.Argument(0)) if err != nil { @@ -180,7 +184,7 @@ func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, pip if pipes == nil { return nil, fmt.Errorf("blob pipe registry not configured") } - pipe := newBlobPipe(ctx, pipes) + pipe := newBlobPipe(ctx, loop, pipes) pipes.Add(pipe) _, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) { fnVal := runtime.Get("__gopeed_blob_pipe_source") @@ -256,6 +260,7 @@ type blobPipeItem struct { type blobPipe struct { id string + loop *eventloop.EventLoop registry *blobPipeRegistry ctx context.Context cancel context.CancelFunc @@ -267,10 +272,11 @@ type blobPipe struct { once sync.Once } -func newBlobPipe(ctx context.Context, registry *blobPipeRegistry) *blobPipe { +func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe { pipeCtx, cancel := context.WithCancel(ctx) return &blobPipe{ id: registry.NewID(), + loop: loop, registry: registry, ctx: pipeCtx, cancel: cancel, @@ -341,9 +347,29 @@ func (p *blobPipe) Close() error { p.mu.Unlock() p.registry.Remove(p.id) p.cancel() + p.cancelSourceReader() return nil } +func (p *blobPipe) cancelSourceReader() { + if p.loop == nil { + return + } + // Invoke the JavaScript cancellation before returning from Close. The + // source session may be released immediately after the HTTP handler closes + // this reader, so merely queueing the callback could let the engine stop + // before reader.cancel() is called. + _, _ = runOnLoop(p.loop, func(runtime *goja.Runtime) (goja.Value, error) { + fnVal := runtime.Get("__gopeed_blob_cancel_pipe_source") + fn, ok := goja.AssertFunction(fnVal) + if !ok { + return goja.Undefined(), nil + } + _, err := fn(nil, runtime.ToValue(p.id), runtime.ToValue("blob request closed")) + return goja.Undefined(), err + }) +} + func exportBytes(value goja.Value) []byte { if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { return nil @@ -528,6 +554,9 @@ func exportJSError(value goja.Value) error { type fetchRegistry struct { mu sync.Mutex streams map[string]*fetchStream + ctx context.Context + cancel context.CancelFunc + closed bool } type fetchStream struct { @@ -536,6 +565,8 @@ type fetchStream struct { closeOnce sync.Once ctx context.Context ch chan fetchChunk + readMu sync.Mutex + pending []byte } type fetchChunk struct { @@ -552,6 +583,15 @@ type fetchRequest struct { Credentials string } +type fetchFormDataEntry struct { + name string + value any +} + +type fetchFormDataSnapshot struct { + entries []fetchFormDataEntry +} + type fetchOpenMeta struct { ID string `json:"id"` Status int `json:"status"` @@ -561,8 +601,11 @@ type fetchOpenMeta struct { } func newFetchRegistry() *fetchRegistry { + ctx, cancel := context.WithCancel(context.Background()) return &fetchRegistry{ streams: make(map[string]*fetchStream), + ctx: ctx, + cancel: cancel, } } @@ -593,12 +636,82 @@ func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, } bodyVal := obj.Get("body") if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) { - meta.Body = bodyVal.Export() + body, err := snapshotFetchBody(bodyVal.Export()) + if err != nil { + return nil, err + } + meta.Body = body } return meta, nil } +// snapshotFetchBody runs on the JavaScript event loop before fetch work is +// handed to a background goroutine. Values exported by goja may still share +// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the +// background request must only retain immutable Go-owned data. +func snapshotFetchBody(body any) (any, error) { + switch v := body.(type) { + case nil, string: + return v, nil + case []byte: + return append([]byte(nil), v...), nil + case goja.ArrayBuffer: + return append([]byte(nil), v.Bytes()...), nil + case *file.File: + return snapshotFetchFile(v), nil + case *formdata.FormData: + entries := v.Entries() + snapshot := &fetchFormDataSnapshot{ + entries: make([]fetchFormDataEntry, 0, len(entries)), + } + for _, entry := range entries { + pair, ok := entry.([]any) + if !ok || len(pair) != 2 { + return nil, fmt.Errorf("invalid FormData entry %T", entry) + } + name, ok := pair[0].(string) + if !ok { + return nil, fmt.Errorf("invalid FormData field name %T", pair[0]) + } + value := pair[1] + if formFile, ok := value.(*file.File); ok { + value = snapshotFetchFile(formFile) + } + snapshot.entries = append(snapshot.entries, fetchFormDataEntry{ + name: name, + value: value, + }) + } + return snapshot, nil + default: + if typed, ok := v.(interface{ Bytes() []byte }); ok { + return append([]byte(nil), typed.Bytes()...), nil + } + return v, nil + } +} + +func snapshotFetchFile(src *file.File) *file.File { + if src == nil { + return nil + } + return &file.File{ + Reader: src.Reader, + Closer: src.Closer, + Name: src.Name, + Size: src.Size, + } +} + func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return nil, context.Canceled + } + registryCtx := r.ctx + r.mu.Unlock() + client := req.C() if proxyHandler != nil { client.SetProxy(proxyHandler) @@ -608,7 +721,13 @@ func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Reque if err != nil { return nil, err } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(registryCtx) + keepCancel := false + defer func() { + if !keepCancel { + cancel() + } + }() reqBuilder := client.R() reqBuilder.SetContext(ctx) reqBuilder.DisableAutoReadResponse() @@ -668,8 +787,14 @@ func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Reque ch: make(chan fetchChunk, 8), } r.mu.Lock() + if r.closed { + r.mu.Unlock() + closeFetchStream(stream) + return nil, context.Canceled + } r.streams[id] = stream r.mu.Unlock() + keepCancel = true go stream.readLoop() return meta, nil } @@ -679,6 +804,17 @@ func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) { if stream == nil { return nil, true, nil } + stream.readMu.Lock() + defer stream.readMu.Unlock() + if len(stream.pending) > 0 { + n := len(stream.pending) + if chunkSize > 0 && n > chunkSize { + n = chunkSize + } + chunk := append([]byte(nil), stream.pending[:n]...) + stream.pending = stream.pending[n:] + return chunk, false, nil + } select { case item, ok := <-stream.ch: if !ok { @@ -691,11 +827,7 @@ func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) { } if chunkSize > 0 && len(item.data) > chunkSize { head := append([]byte(nil), item.data[:chunkSize]...) - tail := append([]byte(nil), item.data[chunkSize:]...) - select { - case stream.ch <- fetchChunk{data: tail}: - case <-stream.ctx.Done(): - } + stream.pending = append(stream.pending[:0], item.data[chunkSize:]...) return head, false, nil } return item.data, false, nil @@ -735,16 +867,45 @@ func (r *fetchRegistry) Close(id string) { stream := r.streams[id] delete(r.streams, id) r.mu.Unlock() - if stream != nil { - stream.closeOnce.Do(func() { - if stream.cancel != nil { - stream.cancel() - } - _ = stream.body.Close() - }) + closeFetchStream(stream) +} + +// CloseAll permanently closes the registry and cancels both active and +// in-flight fetches. It is safe to call more than once. +func (r *fetchRegistry) CloseAll() { + if r == nil { + return + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + r.closed = true + if r.cancel != nil { + r.cancel() + } + streams := r.streams + r.streams = make(map[string]*fetchStream) + r.mu.Unlock() + + for _, stream := range streams { + closeFetchStream(stream) } } +func closeFetchStream(stream *fetchStream) { + if stream == nil { + return + } + stream.closeOnce.Do(func() { + if stream.cancel != nil { + stream.cancel() + } + _ = stream.body.Close() + }) +} + func (r *fetchRegistry) Abort(id string, _ string) { r.Close(id) } @@ -767,24 +928,33 @@ func buildFetchBody(body any) (string, any, error) { return "application/octet-stream", v.Bytes(), nil case *file.File: return "application/octet-stream", v.Reader, nil - case *formdata.FormData: + case *fetchFormDataSnapshot: pr, pw := io.Pipe() mw := xhr.NewMultipart(pw) - for _, e := range v.Entries() { - arr := e.([]any) - key := arr[0].(string) - val := arr[1] - switch vv := val.(type) { + for _, entry := range v.entries { + switch vv := entry.value.(type) { case string: - mw.WriteField(key, vv) + if err := mw.WriteField(entry.name, vv); err != nil { + _ = pw.CloseWithError(err) + return "", nil, err + } case *file.File: - mw.WriteFile(key, vv) + if err := mw.WriteFile(entry.name, vv); err != nil { + _ = pw.CloseWithError(err) + return "", nil, err + } } } go func() { - defer pw.Close() - defer mw.Close() - mw.Send() + if err := mw.Send(); err != nil { + _ = pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + _ = pw.CloseWithError(err) + return + } + _ = pw.Close() }() return mw.FormDataContentType(), pr, nil default: diff --git a/pkg/download/engine/inject/stream/module_integration_test.go b/pkg/download/engine/inject/stream/module_integration_test.go new file mode 100644 index 000000000..6a7151493 --- /dev/null +++ b/pkg/download/engine/inject/stream/module_integration_test.go @@ -0,0 +1,278 @@ +package stream_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/download/engine" + "github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream" +) + +type createdObjectURL struct { + opts *stream.ObjectURLOptions + open stream.ObjectURLOpener +} + +func TestResponseBlobObjectURLReadsResponseStream(t *testing.T) { + const payload = "response blob payload" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = io.WriteString(w, payload[:8]) + w.(http.Flusher).Flush() // force an unknown-length/chunked response + _, _ = io.WriteString(w, payload[8:]) + })) + defer server.Close() + + created := make(chan createdObjectURL, 1) + engine, cleanup := newStreamTestEngine(t, created) + defer func() { + cleanup() + engine.Close() + }() + + value, err := engine.RunString(fmt.Sprintf(` + (async () => { + const response = await fetch(%q); + const blob = await response.blob(); + return __gopeed_blob_create_object_url(blob); + })() + `, server.URL)) + if err != nil { + t.Fatal(err) + } + if value != "blob:test-1" { + t.Fatalf("unexpected object URL: %#v", value) + } + + object := <-created + if object.opts.Size != int64(len(payload)) || object.opts.ContentType != "text/plain" { + t.Fatalf("unexpected object URL options: %#v", object.opts) + } + if !object.opts.Range { + t.Fatal("materialized response Blob should advertise range support") + } + for i := 0; i < 2; i++ { + reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1}) + if err != nil { + t.Fatal(err) + } + data, readErr := io.ReadAll(reader) + _ = reader.Close() + if readErr != nil { + t.Fatal(readErr) + } + if got := string(data); got != payload { + t.Fatalf("unexpected response Blob data on open %d: %q", i+1, got) + } + } + rangeReader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 9, End: 12}) + if err != nil { + t.Fatal(err) + } + rangeData, readErr := io.ReadAll(rangeReader) + _ = rangeReader.Close() + if readErr != nil { + t.Fatal(readErr) + } + if got, want := string(rangeData), payload[9:13]; got != want { + t.Fatalf("unexpected materialized Blob range: got %q want %q", got, want) + } +} + +func TestResponseBlobRejectsTruncatedResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "64") + _, _ = io.WriteString(w, "partial") + })) + defer server.Close() + + created := make(chan createdObjectURL, 1) + engine, cleanup := newStreamTestEngine(t, created) + defer func() { + cleanup() + engine.Close() + }() + + value, err := engine.RunString(fmt.Sprintf(` + (async () => { + const response = await fetch(%q); + try { + await response.blob(); + return "resolved"; + } catch (_) { + return "rejected"; + } + })() + `, server.URL)) + if err != nil { + t.Fatal(err) + } + if value != "rejected" { + t.Fatalf("truncated response unexpectedly materialized a Blob: %#v", value) + } +} + +func TestEmptyBlobObjectURL(t *testing.T) { + created := make(chan createdObjectURL, 1) + engine, cleanup := newStreamTestEngine(t, created) + defer func() { + cleanup() + engine.Close() + }() + + value, err := engine.RunString(` + __gopeed_blob_create_object_url(new Blob([], { type: "application/empty" })) + `) + if err != nil { + t.Fatal(err) + } + if value != "blob:test-1" { + t.Fatalf("unexpected object URL: %#v", value) + } + + object := <-created + if object.opts.Size != 0 || object.opts.Range { + t.Fatalf("unexpected empty Blob options: %#v", object.opts) + } + reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1}) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if len(data) != 0 { + t.Fatalf("expected empty Blob, got %q", data) + } +} + +func TestBlobObjectURLReaderCloseCancelsPendingJSReader(t *testing.T) { + created := make(chan createdObjectURL, 1) + engine, cleanup := newStreamTestEngine(t, created) + defer func() { + cleanup() + engine.Close() + }() + + value, err := engine.RunString(` + globalThis.__blobPipeCancelState = { + readCalls: 0, + cancelCalls: 0, + releaseCalls: 0, + cancelReason: "", + resolveRead: null, + }; + const pendingSource = { + getReader() { + return { + read() { + __blobPipeCancelState.readCalls++; + return new Promise((resolve) => { + __blobPipeCancelState.resolveRead = resolve; + }); + }, + cancel(reason) { + __blobPipeCancelState.cancelCalls++; + __blobPipeCancelState.cancelReason = String(reason); + if (__blobPipeCancelState.resolveRead) { + __blobPipeCancelState.resolveRead({ done: true, value: undefined }); + } + return Promise.resolve(); + }, + releaseLock() { + __blobPipeCancelState.releaseCalls++; + }, + }; + }, + }; + __gopeed_blob_create_object_url(async () => pendingSource); + `) + if err != nil { + t.Fatal(err) + } + if value != "blob:test-1" { + t.Fatalf("unexpected object URL: %#v", value) + } + + object := <-created + reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1}) + if err != nil { + t.Fatal(err) + } + readDone := make(chan error, 1) + go func() { + _, readErr := reader.Read(make([]byte, 1)) + readDone <- readErr + }() + + waitForJSValue(t, engine, `String(__blobPipeCancelState.readCalls)`, "1") + if err := reader.Close(); err != nil { + t.Fatal(err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + + select { + case readErr := <-readDone: + if !errors.Is(readErr, context.Canceled) { + t.Fatalf("unexpected pending read result: %v", readErr) + } + case <-time.After(2 * time.Second): + t.Fatal("pending Go read was not released by Close") + } + waitForJSValue(t, engine, `JSON.stringify({ + cancelCalls: __blobPipeCancelState.cancelCalls, + releaseCalls: __blobPipeCancelState.releaseCalls, + cancelReason: __blobPipeCancelState.cancelReason, + })`, `{"cancelCalls":1,"releaseCalls":1,"cancelReason":"blob request closed"}`) +} + +func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + var got any + for time.Now().Before(deadline) { + var err error + got, err = engine.RunString(expression) + if err != nil { + t.Fatal(err) + } + if got == want { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for JavaScript value %q, got %#v", want, got) +} + +func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) { + t.Helper() + var next atomic.Int64 + var cleanup func() + e := engine.NewEngine(&engine.Config{ + StreamConfig: &stream.Config{ + CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) { + created <- createdObjectURL{opts: opts, open: open} + return fmt.Sprintf("blob:test-%d", next.Add(1)), nil + }, + RevokeObjectURL: func(string) error { return nil }, + RegisterCleanup: func(fn func()) { + cleanup = fn + }, + }, + }) + if cleanup == nil { + t.Fatal("stream cleanup was not registered") + } + return e, cleanup +} diff --git a/pkg/download/engine/inject/stream/module_test.go b/pkg/download/engine/inject/stream/module_test.go new file mode 100644 index 000000000..793ccbd38 --- /dev/null +++ b/pkg/download/engine/inject/stream/module_test.go @@ -0,0 +1,191 @@ +package stream + +import ( + "context" + "io" + "strings" + "sync/atomic" + "testing" + + "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" + "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" + "github.com/dop251/goja" +) + +func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) { + runtime := goja.New() + buffer := runtime.NewArrayBuffer([]byte("before")) + request := runtime.NewObject() + setFetchRequestDefaults(t, request) + if err := request.Set("body", buffer); err != nil { + t.Fatal(err) + } + + meta, err := exportFetchRequest(runtime, request) + if err != nil { + t.Fatal(err) + } + buffer.Bytes()[0] = 'X' + + body, ok := meta.Body.([]byte) + if !ok { + t.Fatalf("expected byte snapshot, got %T", meta.Body) + } + if got := string(body); got != "before" { + t.Fatalf("ArrayBuffer snapshot changed after export: %q", got) + } +} + +func TestExportFetchRequestSnapshotsFormData(t *testing.T) { + runtime := goja.New() + runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true)) + if err := formdata.Enable(runtime); err != nil { + t.Fatal(err) + } + value, err := runtime.RunString(` + globalThis.form = new FormData(); + form.append("field", "before"); + form; + `) + if err != nil { + t.Fatal(err) + } + request := runtime.NewObject() + setFetchRequestDefaults(t, request) + if err := request.Set("body", value); err != nil { + t.Fatal(err) + } + + meta, err := exportFetchRequest(runtime, request) + if err != nil { + t.Fatal(err) + } + if _, err := runtime.RunString(`form.set("field", "after")`); err != nil { + t.Fatal(err) + } + + body, ok := meta.Body.(*fetchFormDataSnapshot) + if !ok { + t.Fatalf("expected FormData snapshot, got %T", meta.Body) + } + if len(body.entries) != 1 { + t.Fatalf("expected one FormData entry, got %d", len(body.entries)) + } + if got := body.entries[0]; got.name != "field" || got.value != "before" { + t.Fatalf("unexpected FormData snapshot: %#v", got) + } +} + +func setFetchRequestDefaults(t *testing.T, request *goja.Object) { + t.Helper() + for key, value := range map[string]any{ + "url": "https://example.test", + "method": "POST", + "redirect": "follow", + "credentials": "same-origin", + } { + if err := request.Set(key, value); err != nil { + t.Fatal(err) + } + } +} + +func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) { + source := &file.File{ + Reader: strings.NewReader("payload"), + Name: "before.txt", + Size: 7, + } + + value, err := snapshotFetchBody(source) + if err != nil { + t.Fatal(err) + } + snapshot, ok := value.(*file.File) + if !ok { + t.Fatalf("expected file snapshot, got %T", value) + } + source.Name = "after.txt" + source.Size = 99 + + if snapshot.Name != "before.txt" || snapshot.Size != 7 { + t.Fatalf("file metadata was not snapshotted: %#v", snapshot) + } +} + +func TestFetchRegistryCloseAll(t *testing.T) { + registry := newFetchRegistry() + streamCtx, cancel := context.WithCancel(context.Background()) + body := &trackingReadCloser{} + registry.streams["active"] = &fetchStream{ + body: body, + cancel: cancel, + ctx: streamCtx, + ch: make(chan fetchChunk), + } + + registry.CloseAll() + registry.CloseAll() + + select { + case <-streamCtx.Done(): + default: + t.Fatal("active fetch context was not canceled") + } + if got := body.closeCalls.Load(); got != 1 { + t.Fatalf("expected body to close once, got %d", got) + } + registry.mu.Lock() + closed := registry.closed + remaining := len(registry.streams) + registry.mu.Unlock() + if !closed || remaining != 0 { + t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining) + } + if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil { + t.Fatal("expected a closed registry to reject new fetches") + } +} + +func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) { + registry := newFetchRegistry() + ctx, cancel := context.WithCancel(context.Background()) + stream := &fetchStream{ + body: io.NopCloser(strings.NewReader("")), + cancel: cancel, + ctx: ctx, + ch: make(chan fetchChunk, 2), + } + stream.ch <- fetchChunk{data: []byte("abc")} + stream.ch <- fetchChunk{data: []byte("def")} + registry.streams["ordered"] = stream + defer registry.CloseAll() + + var got strings.Builder + for range 4 { + chunk, done, err := registry.Read("ordered", 2) + if err != nil { + t.Fatal(err) + } + if done { + t.Fatal("stream ended before buffered chunks were read") + } + got.Write(chunk) + } + if got.String() != "abcdef" { + t.Fatalf("fetch chunks were reordered: %q", got.String()) + } +} + +type trackingReadCloser struct { + closeCalls atomic.Int32 +} + +func (r *trackingReadCloser) Read([]byte) (int, error) { + return 0, io.EOF +} + +func (r *trackingReadCloser) Close() error { + r.closeCalls.Add(1) + return nil +} diff --git a/pkg/download/engine/inject/stream/stream.js b/pkg/download/engine/inject/stream/stream.js index 22d5c0b5f..c8ac6bea1 100644 --- a/pkg/download/engine/inject/stream/stream.js +++ b/pkg/download/engine/inject/stream/stream.js @@ -443,38 +443,6 @@ return merged; } - function createStreamBackedBlob(stream, contentType, size) { - const blob = new Blob([], { type: contentType || "" }); - if (Number.isFinite(size) && size >= 0) { - try { - Object.defineProperty(blob, "size", { - configurable: true, - enumerable: true, - value: size, - }); - } catch (_) { - } - } - blob.__gopeedStreamBackedBlob = true; - blob.__gopeedBodyStream = stream; - blob.stream = function () { - const bodyStream = this.__gopeedBodyStream; - this.__gopeedBodyStream = null; - if (bodyStream instanceof ReadableStream) { - return bodyStream; - } - return new Blob([], { type: this.type || "" }).stream(); - }; - blob.arrayBuffer = async function () { - const bytes = await readAllFromStream(this.stream(), false); - return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - }; - blob.text = async function () { - return readAllFromStream(this.stream(), true); - }; - return blob; - } - function attachResponseStreaming(response, stream) { response.__gopeedBodyStream = stream; response.__gopeedBodyConsumed = false; @@ -511,8 +479,11 @@ ensureUnused(); markBodyUsed(); const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : ""; - const contentLength = this.headers && this.headers.get ? parseInt(this.headers.get("content-length") || "", 10) : NaN; - return createStreamBackedBlob(stream, contentType, contentLength); + // Response.blob() is a materializing body consumer. Keeping the live + // response stream behind an empty one-shot Blob makes retry, range and a + // second object-URL reader silently produce an empty file. + const bytes = await readAllFromStream(stream, false); + return new Blob([bytes], { type: contentType }); }; response.json = async function () { const text = await this.text(); @@ -771,17 +742,95 @@ releaseReader(reader, "blob request closed"); }; + const blobPipeReaderStates = new Map(); + + function releaseBlobPipeReader(state) { + if (!state || state.released) { + return; + } + state.released = true; + if (state.reader && typeof state.reader.releaseLock === "function") { + try { + state.reader.releaseLock(); + } catch (_) { + } + } + if (blobPipeReaderStates.get(state.pipeId) === state) { + blobPipeReaderStates.delete(state.pipeId); + } + } + + function finishBlobPipeReader(state, cancel, reason) { + if (!state) { + return Promise.resolve(); + } + if (cancel) { + state.cancelRequested = true; + if (reason !== undefined) { + state.cancelReason = reason; + } + } + if (state.cleanupPromise) { + return state.cleanupPromise; + } + if (!state.reader) { + // A cancellation may arrive while the asynchronous opener is pending. + // Keep the state until the reader exists so it can still be canceled. + if (!state.cancelRequested || state.openSettled) { + releaseBlobPipeReader(state); + } + return Promise.resolve(); + } + + state.cleanupPromise = (async function () { + try { + if (state.cancelRequested && typeof state.reader.cancel === "function") { + await state.reader.cancel(state.cancelReason); + } + } catch (_) { + } finally { + releaseBlobPipeReader(state); + } + })(); + return state.cleanupPromise; + } + + globalThis.__gopeed_blob_cancel_pipe_source = function (pipeId, reason) { + const state = blobPipeReaderStates.get(pipeId); + if (!state) { + return; + } + // finishBlobPipeReader invokes reader.cancel() synchronously up to its + // first await. Do not return the cleanup Promise: Go only needs to ensure + // cancellation has started before releasing the source session. + finishBlobPipeReader(state, true, reason).catch(function () {}); + }; + globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) { if (typeof openReadable !== "function") { throw new TypeError("blob opener must be callable"); } + const state = { + pipeId, + reader: null, + cancelRequested: false, + cancelReason: "blob pipe closed", + cleanupPromise: null, + openSettled: false, + released: false, + }; + blobPipeReaderStates.set(pipeId, state); const startPipe = async function () { - let reader; try { const source = await openReadable(normalizeOpenRequest(request)); - reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); + state.openSettled = true; + state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function"); + if (state.cancelRequested) { + await finishBlobPipeReader(state, true, state.cancelReason); + return; + } while (true) { - const { done, value } = await reader.read(); + const { done, value } = await state.reader.read(); if (done) { globalThis.__gopeed_blob_pipe_close(pipeId); return; @@ -789,13 +838,19 @@ const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) { - releaseReader(reader, "blob pipe closed"); + state.cancelRequested = true; + state.cancelReason = "blob pipe closed"; return; } await yieldBlobPipeTask(); } } catch (error) { - globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error)); + if (!state.cancelRequested) { + globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error)); + } + } finally { + state.openSettled = true; + await finishBlobPipeReader(state, state.cancelRequested, state.cancelReason); } }; if (typeof setTimeout === "function") { @@ -809,7 +864,7 @@ const described = describeObjectURLValue(value); if (described.kind === "blob") { const blob = described.value; - const normalized = normalizeBlobObjectURLOptions(blob, options, true); + const normalized = normalizeBlobObjectURLOptions(blob, options, Number(blob.size) > 0); validateBlobObjectURLOptions(normalized); const opener = async (request) => openBlobReadable(blob, request); return createBlobObjectURL(opener, normalized); diff --git a/pkg/download/extension.go b/pkg/download/extension.go index 02ef4f387..25ab000bf 100644 --- a/pkg/download/extension.go +++ b/pkg/download/extension.go @@ -278,12 +278,14 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er return } -func (d *Downloader) triggerOnStart(task *Task) { +func (d *Downloader) triggerOnStart(task *Task) *ExtensionTask { + extTask := NewExtensionTask(d, task) + extTask.deferActions = true doTrigger(d, EventOnStart, task.Meta.Req, &OnStartContext{ - Task: NewExtensionTask(d, task), + Task: extTask, }, func(ext *Extension, gopeed *Instance, ctx *OnStartContext) { // Validate request structure @@ -295,19 +297,22 @@ func (d *Downloader) triggerOnStart(task *Task) { } }, ) - return + return extTask } -func (d *Downloader) triggerOnError(task *Task, err error) { +func (d *Downloader) triggerOnError(task *Task, err error) *ExtensionTask { + extTask := NewExtensionTask(d, task) + extTask.deferActions = true doTrigger(d, EventOnError, task.Meta.Req, &OnErrorContext{ - Task: NewExtensionTask(d, task), + Task: extTask, Error: err, }, nil, ) + return extTask } func (d *Downloader) triggerOnDone(task *Task) { @@ -724,6 +729,10 @@ type OnDoneContext struct { type ExtensionTask struct { download *Downloader + deferActions bool + continueRequested bool + pauseRequested bool + *Task } @@ -738,15 +747,23 @@ func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask { } func (t *ExtensionTask) Continue() error { + if t.deferActions { + t.continueRequested = true + t.pauseRequested = false + return nil + } return t.download.Continue(&TaskFilter{ IDs: []string{t.ID}, }) } func (t *ExtensionTask) Pause() error { - return t.download.Pause(&TaskFilter{ - IDs: []string{t.ID}, - }) + if t.deferActions { + t.pauseRequested = true + t.continueRequested = false + return nil + } + return t.download.Pause(&TaskFilter{IDs: []string{t.ID}}) } func parseSettings(settings []*Setting) map[string]any { diff --git a/pkg/download/extension_engine_session.go b/pkg/download/extension_engine_session.go index 2f1046ca7..44b7fa7d0 100644 --- a/pkg/download/extension_engine_session.go +++ b/pkg/download/extension_engine_session.go @@ -118,7 +118,8 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) { RevokeObjectURL: func(url string) error { return d.blob.Revoke(url) }, - ProxyHandler: d.cfg.Proxy.ToHandler(), + ProxyHandler: d.cfg.Proxy.ToHandler(), + RegisterCleanup: session.OnClose, } e := engine.NewEngine(&engine.Config{ ProxyConfig: d.cfg.Proxy, diff --git a/pkg/download/extension_test.go b/pkg/download/extension_test.go index d229c8b5e..cc9e82ea7 100644 --- a/pkg/download/extension_test.go +++ b/pkg/download/extension_test.go @@ -1,11 +1,12 @@ package download import ( - "encoding/json" "errors" "fmt" + "net" "net/http" "net/http/httptest" + neturl "net/url" "os" "path/filepath" "strconv" @@ -704,11 +705,30 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) { if partialSize <= 0 || partialSize >= int64(len(payload)) { t.Fatalf("expected partial downloaded bytes before restart, got %d", partialSize) } + oldURL := file.Req.URL if err := downloader.Close(); err != nil { t.Fatal(err) } + // Keep the old loopback address reachable after the Registry is gone. This + // makes the stale capability URL exercise the ordinary HTTP failure path + // deterministically (404), rather than the HTTP fetcher's retry-forever + // policy for transient dial errors. + parsedOldURL, err := neturl.Parse(oldURL) + if err != nil { + t.Fatal(err) + } + staleListener, err := net.Listen("tcp", parsedOldURL.Host) + if err != nil { + t.Fatal(err) + } + staleServer := &http.Server{Handler: http.NotFoundHandler()} + go func() { + _ = staleServer.Serve(staleListener) + }() + defer staleServer.Close() + downloader = newDownloader() defer func() { _ = downloader.Clear() @@ -723,8 +743,8 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) { if task == nil { t.Fatal("restored task not found") } - if task.Status != base.DownloadStatusError { - t.Fatalf("expected restored blob task to be marked error before recovery, got %s", task.Status) + if task.Status != base.DownloadStatusPause { + t.Fatalf("expected restored blob task to use normal paused HTTP state, got %s", task.Status) } if task.Protocol != "http" { t.Fatalf("expected restored task protocol http, got %s", task.Protocol) @@ -738,13 +758,18 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) { if task.Meta.Req.Labels == nil || task.Meta.Req.Labels["mode"] != "restart" { t.Fatalf("unexpected restored task labels: %#v", task.Meta.Req.Labels) } - oldURL := task.Meta.Req.URL + if task.Meta.Req.URL != oldURL { + t.Fatalf("restored URL changed before ordinary HTTP handling: old=%q current=%q", oldURL, task.Meta.Req.URL) + } if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { t.Fatal(err) } - deadline = time.Now().Add(2 * time.Second) + // A missing Registry entry is deliberately handled as an ordinary HTTP URL. + // Wait for the HTTP fetcher's normal retry policy to exhaust before onError + // rebuilds the capability URL. + deadline = time.Now().Add(15 * time.Second) for time.Now().Before(deadline) { task = downloader.GetTask(id) if task == nil { @@ -765,12 +790,17 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) { t.Fatalf("expected onError recovery to rebuild blob URL, old=%q current=%q labels=%#v", oldURL, task.Meta.Req.URL, task.Meta.Req.Labels) } - if task.Meta.Req.URL == oldURL { - t.Fatalf("expected rebuilt blob URL, old=%q current=%q", oldURL, task.Meta.Req.URL) + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 15*time.Second) + data, err := os.ReadFile(filepath.Join(downloadDir, file.Name)) + if err != nil { + t.Fatal(err) + } + if string(data) != payload { + t.Fatalf("unexpected restarted blob content length=%d want=%d", len(data), len(payload)) } } -func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing.T) { +func TestDownloader_Extension_BlobPauseContinueKeepsSource(t *testing.T) { payload := strings.Repeat("x", 262144) cases := []struct { name string @@ -797,24 +827,13 @@ func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing. for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { - storageDir := t.TempDir() - downloadDir := t.TempDir() - downloader := NewDownloader(&DownloaderConfig{ - Storage: NewBoltStorage(storageDir), - StorageDir: storageDir, - DownloaderStoreConfig: &base.DownloaderStoreConfig{ - DownloadDir: downloadDir, - }, - }) + downloader := NewDownloader(nil) downloader.cfg.RefreshInterval = 50 if err := downloader.Setup(); err != nil { t.Fatal(err) } - defer func() { - _ = downloader.Clear() - }() + defer downloader.Clear() - downloader.cfg.RefreshInterval = 50 if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_pause_rebuild", false); err != nil { t.Fatal(err) } @@ -830,6 +849,7 @@ func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing. } file := rr.Res.Files[0] + downloadDir := t.TempDir() id, err := downloader.CreateDirect(file.Req, &base.Options{ Path: downloadDir, Name: file.Name, @@ -853,75 +873,29 @@ func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing. if err := downloader.Pause(&TaskFilter{IDs: []string{id}}); err != nil { t.Fatal(err) } - waitForTaskStatus(t, downloader, id, base.DownloadStatusPause, 2*time.Second) - - deadline := time.Now().Add(2 * time.Second) - for { - resp, err := http.Get(oldURL) - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode == http.StatusNotFound { - break - } - if time.Now().After(deadline) { - t.Fatalf("expected paused blob URL to be released, got %d", resp.StatusCode) - } - time.Sleep(20 * time.Millisecond) + task = downloader.GetTask(id) + if task == nil || task.Status != base.DownloadStatusPause { + t.Fatalf("expected paused task, got %#v", task) + } + if !downloader.blob.IsURL(oldURL) { + t.Fatal("paused blob source should remain registered") } + // Continue immediately after Pause returns. Pause is synchronous, so the + // previous run cannot race with this new Start. if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { t.Fatal(err) } - - deadline = time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - task = downloader.GetTask(id) - if task == nil || task.Meta == nil || task.Meta.Req == nil { - t.Fatal("task request missing after continue") - } - if task.Meta.Req.Labels["rebuilt"] == "true" && task.Meta.Req.URL != oldURL { - break - } - time.Sleep(20 * time.Millisecond) - } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 10*time.Second) task = downloader.GetTask(id) if task == nil || task.Meta == nil || task.Meta.Req == nil { - t.Fatal("task request missing after onError") - } - if task.Meta.Req.Labels["rebuilt"] != "true" { - t.Fatalf("expected onError to rebuild blob URL, labels=%#v status=%s", task.Meta.Req.Labels, task.Status) - } - if task.Meta.Req.Labels["source"] != strings.TrimPrefix(tc.url, "https://example.com/pause-rebuild-") { - t.Fatalf("unexpected source label after rebuild: %#v", task.Meta.Req.Labels) - } - if task.Meta.Req.URL == oldURL { - t.Fatalf("expected rebuilt blob URL to differ, url=%q labels=%#v", task.Meta.Req.URL, task.Meta.Req.Labels) - } - - deadline = time.Now().Add(35 * time.Second) - for time.Now().Before(deadline) { - task = downloader.GetTask(id) - if task != nil && task.Status == base.DownloadStatusDone { - break - } - time.Sleep(20 * time.Millisecond) + t.Fatal("task request missing after continue") } - task = downloader.GetTask(id) - if task == nil { - t.Fatal("task not found after rebuild") + if task.Meta.Req.URL != oldURL { + t.Fatalf("pause/continue unexpectedly rebuilt URL: old=%q new=%q", oldURL, task.Meta.Req.URL) } - if task.Status != base.DownloadStatusDone { - var stats any - statsJSON := "" - if task.fetcher != nil { - stats = task.fetcher.Stats() - if data, err := json.Marshal(stats); err == nil { - statsJSON = string(data) - } - } - t.Fatalf("timeout waiting for rebuilt blob task done: status=%s downloaded=%d total=%d labels=%#v stats=%#v statsJSON=%s", task.Status, task.Progress.Downloaded, task.Meta.Res.Size, task.Meta.Req.Labels, stats, statsJSON) + if task.Meta.Req.Labels["rebuilt"] != "" { + t.Fatalf("pause/continue should not invoke recovery: %#v", task.Meta.Req.Labels) } data, err := os.ReadFile(filePath) @@ -929,13 +903,13 @@ func TestDownloader_Extension_BlobPauseContinueRebuildsURLViaOnError(t *testing. t.Fatal(err) } if string(data) != payload { - t.Fatalf("unexpected rebuilt download content length=%d want=%d", len(data), len(payload)) + t.Fatalf("unexpected resumed content length=%d want=%d", len(data), len(payload)) } }) } } -func TestDownloader_Extension_BlobOpenerPauseReleasesSource(t *testing.T) { +func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing.T) { setupDownloader(func(downloader *Downloader) { if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob", false); err != nil { t.Fatal(err) @@ -988,89 +962,140 @@ func TestDownloader_Extension_BlobOpenerPauseReleasesSource(t *testing.T) { t.Fatalf("expected paused file size to remain %d, got %d", pausedSize, stat.Size()) } - deadline := time.Now().Add(2 * time.Second) - for { - resp, err := http.Get(task.Meta.Req.URL) - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode == http.StatusNotFound { - break - } - if time.Now().After(deadline) { - t.Fatalf("expected old blob URL to be released after pause, got %d", resp.StatusCode) - } - time.Sleep(20 * time.Millisecond) + oldURL := task.Meta.Req.URL + if !downloader.blob.IsURL(oldURL) { + t.Fatal("paused unknown-size source should remain registered") + } + if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + task = downloader.GetTask(id) + if task.Meta.Req.URL != oldURL { + t.Fatalf("pause/continue unexpectedly replaced URL: old=%q new=%q", oldURL, task.Meta.Req.URL) + } + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + if string(data) != "line 1\nline 2\n" { + t.Fatalf("unexpected resumed unknown-size content %q", string(data)) } }) } -func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { +func TestDownloader_Extension_BlobOnStartURLChangePauseKeepsNewLease(t *testing.T) { setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_recover", false); err != nil { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_onstart_pause", false); err != nil { t.Fatal(err) } - - rr, err := downloader.Resolve(&base.Request{ - URL: "https://example.com/recover", - }, nil) + rr, err := downloader.Resolve(&base.Request{URL: "https://example.com/onstart-pause"}, nil) if err != nil { t.Fatal(err) } - + file := rr.Res.Files[0] + oldURL := file.Req.URL dir := t.TempDir() - id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ - Path: dir, - Name: rr.Res.Files[0].Name, - }) + id, err := downloader.CreateDirect(file.Req, &base.Options{Path: dir, Name: file.Name}) if err != nil { t.Fatal(err) } - - filePath := filepath.Join(dir, "recover.txt") - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - task := downloader.GetTask(id) - if task != nil && task.Status == base.DownloadStatusDone { - break - } - time.Sleep(20 * time.Millisecond) - } - + waitForTaskStatus(t, downloader, id, base.DownloadStatusPause, 5*time.Second) task := downloader.GetTask(id) - if task == nil { - t.Fatal("task not found after recovery") + if task == nil || task.Meta == nil || task.Meta.Req == nil { + t.Fatal("paused onStart task request missing") } - if task.Status != base.DownloadStatusDone { - var fileSize int64 = -1 - if info, statErr := os.Stat(filePath); statErr == nil { - fileSize = info.Size() - } - t.Fatalf( - "timeout waiting for recovered blob download: status=%s downloaded=%d url=%q rawUrl=%q labels=%#v fileSize=%d", - task.Status, - task.Progress.Downloaded, - task.Meta.Req.URL, - task.Meta.Req.RawURL, - task.Meta.Req.Labels, - fileSize, - ) + newURL := task.Meta.Req.URL + if newURL == oldURL { + t.Fatal("onStart did not replace the Blob URL") + } + if downloader.blob.IsURL(oldURL) { + t.Fatal("onStart Pause retained the old Blob lease") + } + if !downloader.blob.IsURL(newURL) { + t.Fatal("onStart Pause did not acquire the replacement Blob lease") } - data, err := os.ReadFile(filePath) + if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) + data, err := os.ReadFile(filepath.Join(dir, file.Name)) if err != nil { t.Fatal(err) } - if !strings.HasPrefix(string(data), "stale\n") { - t.Fatalf("unexpected partial blob file content: %q", string(data)) + if string(data) != "new-source" { + t.Fatalf("unexpected onStart replacement content: %q", data) } + }) +} - if task.Status != base.DownloadStatusDone { - t.Fatalf("expected partial blob task done, got %s", task.Status) +func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { + setupDownloader(func(downloader *Downloader) { + if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_recover", false); err != nil { + t.Fatal(err) } - if task.Meta.Req.RawURL != "https://example.com/recover" { - t.Fatalf("unexpected raw url: %q", task.Meta.Req.RawURL) + + for _, path := range []string{"recover", "recover-range"} { + path := path + t.Run(path, func(t *testing.T) { + rawURL := "https://example.com/" + path + rr, err := downloader.Resolve(&base.Request{URL: rawURL}, nil) + if err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{ + Path: dir, + Name: rr.Res.Files[0].Name, + }) + if err != nil { + t.Fatal(err) + } + + filePath := filepath.Join(dir, "recover.txt") + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + task := downloader.GetTask(id) + if task != nil && task.Status == base.DownloadStatusDone { + break + } + time.Sleep(20 * time.Millisecond) + } + + task := downloader.GetTask(id) + if task == nil { + t.Fatal("task not found after recovery") + } + if task.Status != base.DownloadStatusDone { + var fileSize int64 = -1 + if info, statErr := os.Stat(filePath); statErr == nil { + fileSize = info.Size() + } + t.Fatalf( + "timeout waiting for recovered blob download: status=%s downloaded=%d url=%q rawUrl=%q labels=%#v fileSize=%d", + task.Status, + task.Progress.Downloaded, + task.Meta.Req.URL, + task.Meta.Req.RawURL, + task.Meta.Req.Labels, + fileSize, + ) + } + + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + if string(data) != "ok\n" { + t.Fatalf("unexpected recovered blob file content: %q", string(data)) + } + + if task.Meta.Req.RawURL != rawURL { + t.Fatalf("unexpected raw url: %q", task.Meta.Req.RawURL) + } + }) } }) } diff --git a/pkg/download/model.go b/pkg/download/model.go index 7e2ae0529..8472d55a7 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -36,6 +36,9 @@ type Task struct { timer *util.Timer statusLock *sync.Mutex lock *sync.Mutex + blobRefLock *sync.Mutex + blobURL string + runGeneration uint64 speedArr []int64 uploadSpeedArr []int64 } diff --git a/pkg/download/testdata/extensions/blob_onstart_pause/index.js b/pkg/download/testdata/extensions/blob_onstart_pause/index.js new file mode 100644 index 000000000..66a4fe165 --- /dev/null +++ b/pkg/download/testdata/extensions/blob_onstart_pause/index.js @@ -0,0 +1,27 @@ +gopeed.events.onResolve(async function (ctx) { + if (!ctx.req.url.endsWith('/onstart-pause')) { + return; + } + const url = gopeed.runtime.blob.createObjectURL(new Blob(['old-source'], { type: 'text/plain' })); + ctx.res = { + name: 'blob-onstart-pause', + files: [ + { + name: 'onstart-pause.txt', + size: 10, + req: { url }, + }, + ], + }; +}); + +gopeed.events.onStart(async function (ctx) { + const req = ctx.task.meta.req; + req.labels = req.labels || {}; + if (req.labels.replaced === 'true') { + return; + } + req.url = gopeed.runtime.blob.createObjectURL(new Blob(['new-source'], { type: 'text/plain' })); + req.labels.replaced = 'true'; + ctx.task.pause(); +}); diff --git a/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json b/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json new file mode 100644 index 000000000..c360ed73b --- /dev/null +++ b/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json @@ -0,0 +1,25 @@ +{ + "name": "blob-onstart-pause", + "title": "gopeed extension blob onStart pause test", + "version": "0.0.1", + "scripts": [ + { + "event": "onResolve", + "match": { + "urls": [ + "*://example.com/onstart-pause" + ] + }, + "entry": "index.js" + }, + { + "event": "onStart", + "match": { + "urls": [ + "*://example.com/onstart-pause" + ] + }, + "entry": "index.js" + } + ] +} diff --git a/pkg/download/testdata/extensions/blob_recover/index.js b/pkg/download/testdata/extensions/blob_recover/index.js index e78a66ecd..9d331fd0c 100644 --- a/pkg/download/testdata/extensions/blob_recover/index.js +++ b/pkg/download/testdata/extensions/blob_recover/index.js @@ -1,5 +1,6 @@ gopeed.events.onResolve(async function (ctx) { - if (!ctx.req.url.endsWith("/recover")) { + const isRange = ctx.req.url.endsWith("/recover-range"); + if (!isRange && !ctx.req.url.endsWith("/recover")) { return; } @@ -9,18 +10,18 @@ gopeed.events.onResolve(async function (ctx) { controller.close(); return; } - controller.enqueue(new TextEncoder().encode("stale\n")); + controller.enqueue(new TextEncoder().encode("stale-prefix-that-must-be-removed\n")); await new Promise((resolve) => setTimeout(resolve, 50)); controller.error(new Error("expired")); }, - }), { size: 10 }); + }), { size: 64, range: isRange }); ctx.res = { name: "blob-recover", files: [ { name: "recover.txt", - size: 10, + size: 64, req: { url, } @@ -31,7 +32,7 @@ gopeed.events.onResolve(async function (ctx) { gopeed.events.onError(async function (ctx) { const req = ctx.task.meta.req; - if (!req.rawUrl || !req.rawUrl.endsWith("/recover")) { + if (!req.rawUrl || (!req.rawUrl.endsWith("/recover") && !req.rawUrl.endsWith("/recover-range"))) { return; } req.labels = req.labels || {}; @@ -40,6 +41,6 @@ gopeed.events.onError(async function (ctx) { } req.labels.recovered = "true"; - req.url = gopeed.runtime.blob.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" })); + req.url = gopeed.runtime.blob.createObjectURL(new Blob(["ok\n"], { type: "text/plain" })); ctx.task.continue(); }); From 695ce4c54862540c44224eff65568def72370410 Mon Sep 17 00:00:00 2001 From: Levi Date: Fri, 10 Jul 2026 15:24:35 +0800 Subject: [PATCH 6/7] update --- pkg/download/downloader.go | 46 +------- pkg/download/downloader_test.go | 101 +++++++----------- pkg/download/extension.go | 46 +++----- pkg/download/extension_test.go | 50 +-------- .../extensions/blob_onstart_pause/index.js | 27 ----- .../blob_onstart_pause/manifest.json | 25 ----- 6 files changed, 54 insertions(+), 241 deletions(-) delete mode 100644 pkg/download/testdata/extensions/blob_onstart_pause/index.js delete mode 100644 pkg/download/testdata/extensions/blob_onstart_pause/manifest.json diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 00a1e1183..dfba2d6da 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -1207,7 +1207,7 @@ func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) { if task.Meta != nil && task.Meta.Req != nil { oldURL = task.Meta.Req.URL } - extTask := d.triggerOnError(task, err) + d.triggerOnError(task, err) newURL := "" if task.Meta != nil && task.Meta.Req != nil { newURL = task.Meta.Req.URL @@ -1219,27 +1219,16 @@ func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) { if task.fetcher != nil { if patchErr := task.fetcher.Patch(&base.Request{URL: newURL}, nil); patchErr != nil { d.Logger.Warn().Err(patchErr).Msgf("patch recovered task url failed, task id: %s", task.ID) - extTask.continueRequested = false } } d.releaseBlobTaskURL(task, oldURL) } - if resetFetcher && extTask.continueRequested { + if resetFetcher && d.taskStatus(task) == base.DownloadStatusRunning { if resetErr := d.resetTaskFetcher(task); resetErr != nil { d.Logger.Warn().Err(resetErr).Msgf("reset recovered task fetcher failed, task id: %s", task.ID) - extTask.continueRequested = false } } task.lock.Unlock() - if extTask.pauseRequested { - if pauseErr := d.Pause(&TaskFilter{IDs: []string{task.ID}}); pauseErr != nil { - d.Logger.Warn().Err(pauseErr).Msgf("extension pause task failed, task id: %s", task.ID) - } - } else if extTask.continueRequested { - if continueErr := d.Continue(&TaskFilter{IDs: []string{task.ID}}); continueErr != nil { - d.Logger.Warn().Err(continueErr).Msgf("extension continue task failed, task id: %s", task.ID) - } - } if d.taskStatus(task) == base.DownloadStatusError { d.releaseBlobTask(task) d.emit(EventKeyError, task, err) @@ -1434,7 +1423,6 @@ func (d *Downloader) doStart(task *Task) (err error) { } started := false - pauseBeforeStart := false handler := func() error { task.lock.Lock() defer task.lock.Unlock() @@ -1447,9 +1435,7 @@ func (d *Downloader) doStart(task *Task) (err error) { return err } } - extTask := d.triggerOnStart(task) - // onStart may synchronously request a pause through ExtensionTask. - // Do not resolve or start a task after that request changed its state. + d.triggerOnStart(task) if !d.taskIsRunningGeneration(task, generation) { return nil } @@ -1461,10 +1447,6 @@ func (d *Downloader) doStart(task *Task) (err error) { if err := d.syncBlobTaskLease(task); err != nil { return err } - if extTask.pauseRequested { - pauseBeforeStart = true - return nil - } if task.Meta.Res == nil { err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts) if err != nil { @@ -1522,13 +1504,6 @@ func (d *Downloader) doStart(task *Task) (err error) { } return } - if pauseBeforeStart { - if err := d.doPauseBeforeStart(task); err != nil { - d.Logger.Error().Stack().Err(err).Msgf("onStart pause task failed, task id: %s", task.ID) - } - d.notifyRunning() - return - } if started { if d.taskIsRunningGeneration(task, generation) { d.emit(EventKeyStart, task) @@ -1556,21 +1531,6 @@ func (d *Downloader) doPause(task *Task) (err error) { return nil } -func (d *Downloader) doPauseBeforeStart(task *Task) (err error) { - generation, isReturn, err := d.preparePause(task) - if err != nil || isReturn { - return err - } - handled, err := d.runPauseHandler(task, generation, false) - if err != nil { - return err - } - if handled { - d.emit(EventKeyPause, task) - } - return nil -} - // doPauseForScheduling physically pauses a running task without emitting while // the caller holds d.lock, then atomically moves it to the wait queue state. func (d *Downloader) doPauseForScheduling(task *Task) (bool, error) { diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index a5dec36ba..259a93d42 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -41,6 +42,38 @@ type generationTestManager struct { pauseOnce sync.Once } +func TestExtensionTaskControlMethodsOnlyExistOnError(t *testing.T) { + taskType := reflect.TypeOf((*Task)(nil)) + if _, ok := taskType.MethodByName("Continue"); ok { + t.Fatal("regular task unexpectedly exposes Continue") + } + if _, ok := taskType.MethodByName("Pause"); ok { + t.Fatal("regular task unexpectedly exposes Pause") + } + + errorTaskType := reflect.TypeOf((*ExtensionTask)(nil)) + if _, ok := errorTaskType.MethodByName("Continue"); !ok { + t.Fatal("onError task does not expose Continue") + } + if _, ok := errorTaskType.MethodByName("Pause"); ok { + t.Fatal("onError task unexpectedly exposes Pause") + } + + for name, contextType := range map[string]reflect.Type{ + "onStart": reflect.TypeOf(OnStartContext{}), + "onDone": reflect.TypeOf(OnDoneContext{}), + } { + field, _ := contextType.FieldByName("Task") + if field.Type != taskType { + t.Fatalf("%s unexpectedly injects a control wrapper: %v", name, field.Type) + } + } + errorTaskField, _ := reflect.TypeOf(OnErrorContext{}).FieldByName("Task") + if errorTaskField.Type != errorTaskType { + t.Fatalf("onError does not inject the Continue wrapper: %v", errorTaskField.Type) + } +} + func (m *generationTestManager) Name() string { return "generation" } func (m *generationTestManager) Filters() []*fetcher.SchemeFilter { return []*fetcher.SchemeFilter{{Type: fetcher.FilterTypeUrl, Pattern: "generation"}} @@ -493,14 +526,14 @@ func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) { } pauseDone := make(chan error, 1) go func() { - pauseDone <- downloader.doPauseBeforeStart(task) + pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}}) }() deadline := time.Now().Add(2 * time.Second) for downloader.taskStatus(task) != base.DownloadStatusPause && time.Now().Before(deadline) { time.Sleep(5 * time.Millisecond) } if downloader.taskStatus(task) != base.DownloadStatusPause { - t.Fatal("deferred pre-start Pause did not update task status") + t.Fatal("Pause did not update task status") } if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { t.Fatal(err) @@ -512,7 +545,7 @@ func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) { t.Fatal(err) } case <-time.After(2 * time.Second): - t.Fatal("deferred pre-start Pause did not finish") + t.Fatal("Pause did not finish") } waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 3*time.Second) @@ -521,68 +554,6 @@ func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) { } } -func TestExtensionTaskDefersOnStartPauseUntilTaskUnlock(t *testing.T) { - downloader := NewDownloader(nil) - if err := downloader.Setup(); err != nil { - t.Fatal(err) - } - defer downloader.Clear() - downloader.cfg.MaxRunning = 0 - - blobURL, err := downloader.blob.CreateBlob([]byte("pause"), "text/plain") - if err != nil { - t.Fatal(err) - } - id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{ - Path: t.TempDir(), - Name: "pause.bin", - }) - if err != nil { - t.Fatal(err) - } - liveTask := downloader.GetTask(id) - if liveTask == nil { - t.Fatal("task not found") - } - patched := make(chan error, 1) - downloader.Listener(func(event *Event) { - if event.Key == EventKeyPause && event.Task.ID == id { - patched <- downloader.Patch(id, &base.Request{Labels: map[string]string{"paused": "true"}}, nil) - } - }) - - // doStart invokes onStart while holding this lock. Record the action on the - // extension wrapper and apply it only after the handler unlocks. - extTask := NewExtensionTask(downloader, liveTask) - extTask.deferActions = true - liveTask.lock.Lock() - err = extTask.Pause() - liveTask.lock.Unlock() - if err != nil { - t.Fatal(err) - } - if !extTask.pauseRequested { - t.Fatal("onStart Pause was not deferred") - } - if err := downloader.doPauseBeforeStart(liveTask); err != nil { - t.Fatal(err) - } - liveTask.statusLock.Lock() - status := liveTask.Status - liveTask.statusLock.Unlock() - if status != base.DownloadStatusPause { - t.Fatalf("extension pause did not update live task: %s", status) - } - select { - case err := <-patched: - if err != nil { - t.Fatalf("Pause listener could not re-enter Patch: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("Pause listener deadlocked while re-entering Patch") - } -} - func TestDownloader_PauseWinsAgainstStaleResolveError(t *testing.T) { releaseResolve := make(chan struct{}) manager := &generationTestManager{ diff --git a/pkg/download/extension.go b/pkg/download/extension.go index 25ab000bf..238a11f12 100644 --- a/pkg/download/extension.go +++ b/pkg/download/extension.go @@ -278,14 +278,12 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er return } -func (d *Downloader) triggerOnStart(task *Task) *ExtensionTask { - extTask := NewExtensionTask(d, task) - extTask.deferActions = true +func (d *Downloader) triggerOnStart(task *Task) { doTrigger(d, EventOnStart, task.Meta.Req, &OnStartContext{ - Task: extTask, + Task: newExtensionTaskSnapshot(task), }, func(ext *Extension, gopeed *Instance, ctx *OnStartContext) { // Validate request structure @@ -297,30 +295,26 @@ func (d *Downloader) triggerOnStart(task *Task) *ExtensionTask { } }, ) - return extTask } -func (d *Downloader) triggerOnError(task *Task, err error) *ExtensionTask { - extTask := NewExtensionTask(d, task) - extTask.deferActions = true +func (d *Downloader) triggerOnError(task *Task, err error) { doTrigger(d, EventOnError, task.Meta.Req, &OnErrorContext{ - Task: extTask, + Task: NewExtensionTask(d, task), Error: err, }, nil, ) - return extTask } func (d *Downloader) triggerOnDone(task *Task) { doTrigger(d, EventOnDone, task.Meta.Req, - &OnErrorContext{ - Task: NewExtensionTask(d, task), + &OnDoneContext{ + Task: newExtensionTaskSnapshot(task), }, nil, ) @@ -711,7 +705,7 @@ type OnResolveContext struct { } type OnStartContext struct { - Task *ExtensionTask `json:"task"` + Task *Task `json:"task"` } type OnErrorContext struct { @@ -729,43 +723,29 @@ type OnDoneContext struct { type ExtensionTask struct { download *Downloader - deferActions bool - continueRequested bool - pauseRequested bool - *Task } -func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask { +func newExtensionTaskSnapshot(task *Task) *Task { // restricts extension scripts to only modify request info newTask := task.clone() newTask.Meta.Req = task.Meta.Req + return newTask +} + +func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask { return &ExtensionTask{ download: download, - Task: newTask, + Task: newExtensionTaskSnapshot(task), } } func (t *ExtensionTask) Continue() error { - if t.deferActions { - t.continueRequested = true - t.pauseRequested = false - return nil - } return t.download.Continue(&TaskFilter{ IDs: []string{t.ID}, }) } -func (t *ExtensionTask) Pause() error { - if t.deferActions { - t.pauseRequested = true - t.continueRequested = false - return nil - } - return t.download.Pause(&TaskFilter{IDs: []string{t.ID}}) -} - func parseSettings(settings []*Setting) map[string]any { m := make(map[string]any) for _, s := range settings { diff --git a/pkg/download/extension_test.go b/pkg/download/extension_test.go index cc9e82ea7..0cdb9bd6c 100644 --- a/pkg/download/extension_test.go +++ b/pkg/download/extension_test.go @@ -984,52 +984,6 @@ func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing }) } -func TestDownloader_Extension_BlobOnStartURLChangePauseKeepsNewLease(t *testing.T) { - setupDownloader(func(downloader *Downloader) { - if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_onstart_pause", false); err != nil { - t.Fatal(err) - } - rr, err := downloader.Resolve(&base.Request{URL: "https://example.com/onstart-pause"}, nil) - if err != nil { - t.Fatal(err) - } - file := rr.Res.Files[0] - oldURL := file.Req.URL - dir := t.TempDir() - id, err := downloader.CreateDirect(file.Req, &base.Options{Path: dir, Name: file.Name}) - if err != nil { - t.Fatal(err) - } - waitForTaskStatus(t, downloader, id, base.DownloadStatusPause, 5*time.Second) - task := downloader.GetTask(id) - if task == nil || task.Meta == nil || task.Meta.Req == nil { - t.Fatal("paused onStart task request missing") - } - newURL := task.Meta.Req.URL - if newURL == oldURL { - t.Fatal("onStart did not replace the Blob URL") - } - if downloader.blob.IsURL(oldURL) { - t.Fatal("onStart Pause retained the old Blob lease") - } - if !downloader.blob.IsURL(newURL) { - t.Fatal("onStart Pause did not acquire the replacement Blob lease") - } - - if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil { - t.Fatal(err) - } - waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second) - data, err := os.ReadFile(filepath.Join(dir, file.Name)) - if err != nil { - t.Fatal(err) - } - if string(data) != "new-source" { - t.Fatalf("unexpected onStart replacement content: %q", data) - } - }) -} - func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { setupDownloader(func(downloader *Downloader) { if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/blob_recover", false); err != nil { @@ -1058,7 +1012,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { deadline := time.Now().Add(15 * time.Second) for time.Now().Before(deadline) { task := downloader.GetTask(id) - if task != nil && task.Status == base.DownloadStatusDone { + if task != nil && downloader.taskStatus(task) == base.DownloadStatusDone { break } time.Sleep(20 * time.Millisecond) @@ -1068,7 +1022,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) { if task == nil { t.Fatal("task not found after recovery") } - if task.Status != base.DownloadStatusDone { + if downloader.taskStatus(task) != base.DownloadStatusDone { var fileSize int64 = -1 if info, statErr := os.Stat(filePath); statErr == nil { fileSize = info.Size() diff --git a/pkg/download/testdata/extensions/blob_onstart_pause/index.js b/pkg/download/testdata/extensions/blob_onstart_pause/index.js deleted file mode 100644 index 66a4fe165..000000000 --- a/pkg/download/testdata/extensions/blob_onstart_pause/index.js +++ /dev/null @@ -1,27 +0,0 @@ -gopeed.events.onResolve(async function (ctx) { - if (!ctx.req.url.endsWith('/onstart-pause')) { - return; - } - const url = gopeed.runtime.blob.createObjectURL(new Blob(['old-source'], { type: 'text/plain' })); - ctx.res = { - name: 'blob-onstart-pause', - files: [ - { - name: 'onstart-pause.txt', - size: 10, - req: { url }, - }, - ], - }; -}); - -gopeed.events.onStart(async function (ctx) { - const req = ctx.task.meta.req; - req.labels = req.labels || {}; - if (req.labels.replaced === 'true') { - return; - } - req.url = gopeed.runtime.blob.createObjectURL(new Blob(['new-source'], { type: 'text/plain' })); - req.labels.replaced = 'true'; - ctx.task.pause(); -}); diff --git a/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json b/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json deleted file mode 100644 index c360ed73b..000000000 --- a/pkg/download/testdata/extensions/blob_onstart_pause/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "blob-onstart-pause", - "title": "gopeed extension blob onStart pause test", - "version": "0.0.1", - "scripts": [ - { - "event": "onResolve", - "match": { - "urls": [ - "*://example.com/onstart-pause" - ] - }, - "entry": "index.js" - }, - { - "event": "onStart", - "match": { - "urls": [ - "*://example.com/onstart-pause" - ] - }, - "entry": "index.js" - } - ] -} From 066d3bbe5b05da22ab083651dd39c781e9995b41 Mon Sep 17 00:00:00 2001 From: Levi Date: Fri, 10 Jul 2026 19:57:46 +0800 Subject: [PATCH 7/7] update --- pkg/download/downloader_test.go | 61 ++++++++++++++++++--------------- pkg/download/extension.go | 21 ++++++++---- pkg/rest/server_test.go | 12 +++++++ 3 files changed, 60 insertions(+), 34 deletions(-) diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index 259a93d42..e29b18403 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -150,9 +150,14 @@ func (f *generationTestFetcher) Meta() *fetcher.FetcherMeta { return f.meta } func (f *generationTestFetcher) Progress() fetcher.Progress { return fetcher.Progress{1} } func (f *generationTestFetcher) Wait() error { return <-f.done } -func newTestDownloadOpt() *base.Options { +func newTestDownloadOpt(t *testing.T) *base.Options { + t.Helper() + return newTestDownloadOptAt(t.TempDir()) +} + +func newTestDownloadOptAt(path string) *base.Options { return &base.Options{ - Path: test.Dir, + Path: path, Name: test.DownloadName, Extra: http.OptsExtra{ Connections: 4, @@ -698,7 +703,8 @@ func TestDownloader_Create(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req, newTestDownloadOpt()) + opts := newTestDownloadOpt(t) + rr, err := downloader.Resolve(req, opts) if err != nil { t.Fatal(err) } @@ -716,7 +722,7 @@ func TestDownloader_Create(t *testing.T) { } wg.Wait() want := test.FileMd5(test.BuildFile) - got := test.FileMd5(test.DownloadFile) + got := test.FileMd5(filepath.Join(opts.Path, opts.Name)) if want != got { t.Errorf("Downloader_Create() got = %v, want %v", got, want) } @@ -738,7 +744,7 @@ func TestDownloader_CreateNotInWhite(t *testing.T) { } // With new fetcher design, white list check happens during Resolve (not Create) // because Resolve now requires Options which includes the download path - _, err := downloader.Resolve(req, newTestDownloadOpt()) + _, err := downloader.Resolve(req, newTestDownloadOpt(t)) if err == nil { t.Error("TestDownloader_CreateNotInWhite() expected error but got nil") } @@ -785,9 +791,10 @@ func TestDownloader_CreateDirectBatch(t *testing.T) { } }) + opts := newTestDownloadOpt(t) _, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ Reqs: reqs, - Opts: newTestDownloadOpt(), + Opts: opts, }) if err != nil { t.Fatal(err) @@ -813,7 +820,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) { // Check that all task files exist for name := range taskNames { - if _, err := os.Stat(test.Dir + "/" + name); os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(opts.Path, name)); os.IsNotExist(err) { t.Errorf("CreateDirectBatch() file not exist: %v", name) } } @@ -966,6 +973,7 @@ func TestDownloader_CreateRename(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } + downloadDir := t.TempDir() var wg sync.WaitGroup wg.Add(2) downloader.Listener(func(event *Event) { @@ -975,7 +983,7 @@ func TestDownloader_CreateRename(t *testing.T) { }) for i := 0; i < 2; i++ { _, err := downloader.CreateDirect(req, &base.Options{ - Path: test.Dir, + Path: downloadDir, Name: test.DownloadName, Extra: http.OptsExtra{ Connections: 4, @@ -988,11 +996,11 @@ func TestDownloader_CreateRename(t *testing.T) { wg.Wait() want := test.FileMd5(test.BuildFile) - got := test.FileMd5(test.DownloadFile) + got := test.FileMd5(filepath.Join(downloadDir, test.DownloadName)) if want != got { t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want) } - got = test.FileMd5(test.DownloadRenameFile) + got = test.FileMd5(filepath.Join(downloadDir, test.DownloadRename)) if want != got { t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want) } @@ -1002,8 +1010,9 @@ func TestDownloader_StoreAndRestore(t *testing.T) { listener := test.StartTestSlowFileServer(time.Millisecond * 2000) defer listener.Close() + storageDir := t.TempDir() downloader := NewDownloader(&DownloaderConfig{ - Storage: NewBoltStorage("./"), + Storage: NewBoltStorage(storageDir), }) if err := downloader.Setup(); err != nil { t.Fatal(err) @@ -1012,7 +1021,8 @@ func TestDownloader_StoreAndRestore(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req, newTestDownloadOpt()) + opts := newTestDownloadOpt(t) + rr, err := downloader.Resolve(req, opts) if err != nil { t.Fatal(err) } @@ -1029,7 +1039,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) { downloader.Close() downloader = NewDownloader(&DownloaderConfig{ - Storage: NewBoltStorage("./"), + Storage: NewBoltStorage(storageDir), }) if err := downloader.Setup(); err != nil { t.Fatal(err) @@ -1052,7 +1062,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) { t.Fatal(err) } want := test.FileMd5(test.BuildFile) - got := test.FileMd5(test.DownloadFile) + got := test.FileMd5(filepath.Join(opts.Path, opts.Name)) if want != got { t.Errorf("StoreAndResume() got = %v, want %v", got, want) } @@ -1146,15 +1156,10 @@ func TestDownloader_GetTasksByFilter(t *testing.T) { } }) + batchOpts := newTestDownloadOpt(t) taskIds, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ Reqs: reqs, - Opts: &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }, + Opts: batchOpts, }) if err != nil { t.Fatal(err) @@ -1281,7 +1286,7 @@ func TestDownloader_Stats(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req, newTestDownloadOpt()) + rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) if err != nil { t.Fatal(err) } @@ -1324,6 +1329,7 @@ func TestDownloader_Delete(t *testing.T) { // Create multiple tasks var wg sync.WaitGroup taskCount := 3 + downloadDir := t.TempDir() wg.Add(taskCount) downloader.Listener(func(event *Event) { if event.Key == EventKeyDone { @@ -1336,7 +1342,7 @@ func TestDownloader_Delete(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - taskId, err := downloader.CreateDirect(req, newTestDownloadOpt()) + taskId, err := downloader.CreateDirect(req, newTestDownloadOptAt(downloadDir)) if err != nil { t.Fatal(err) } @@ -1400,7 +1406,7 @@ func TestDownloader_PauseAndContinue(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req, newTestDownloadOpt()) + rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) if err != nil { t.Fatal(err) } @@ -1471,7 +1477,7 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req, newTestDownloadOpt()) + rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) if err != nil { t.Fatal(err) } @@ -2131,6 +2137,7 @@ func TestDownloader_DeleteAll(t *testing.T) { // Create multiple tasks var wg sync.WaitGroup taskCount := 3 + downloadDir := t.TempDir() wg.Add(taskCount) downloader.Listener(func(event *Event) { if event.Key == EventKeyDone { @@ -2143,7 +2150,7 @@ func TestDownloader_DeleteAll(t *testing.T) { URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } _, err := downloader.CreateDirect(req, &base.Options{ - Path: test.Dir, + Path: downloadDir, Name: test.DownloadName, Extra: http.OptsExtra{ Connections: 4, @@ -3297,7 +3304,7 @@ func TestDownloader_PatchTask_HTTP(t *testing.T) { }, } opts := &base.Options{ - Path: test.Dir, + Path: t.TempDir(), Name: test.DownloadName, } diff --git a/pkg/download/extension.go b/pkg/download/extension.go index 238a11f12..a38038c49 100644 --- a/pkg/download/extension.go +++ b/pkg/download/extension.go @@ -283,7 +283,7 @@ func (d *Downloader) triggerOnStart(task *Task) { EventOnStart, task.Meta.Req, &OnStartContext{ - Task: newExtensionTaskSnapshot(task), + Task: newOnStartExtensionTask(task), }, func(ext *Extension, gopeed *Instance, ctx *OnStartContext) { // Validate request structure @@ -302,7 +302,7 @@ func (d *Downloader) triggerOnError(task *Task, err error) { EventOnError, task.Meta.Req, &OnErrorContext{ - Task: NewExtensionTask(d, task), + Task: newOnErrorExtensionTask(d, task), Error: err, }, nil, @@ -314,7 +314,7 @@ func (d *Downloader) triggerOnDone(task *Task) { EventOnDone, task.Meta.Req, &OnDoneContext{ - Task: newExtensionTaskSnapshot(task), + Task: newOnDoneExtensionTask(task), }, nil, ) @@ -726,20 +726,27 @@ type ExtensionTask struct { *Task } -func newExtensionTaskSnapshot(task *Task) *Task { - // restricts extension scripts to only modify request info +func cloneExtensionTask(task *Task) *Task { newTask := task.clone() newTask.Meta.Req = task.Meta.Req return newTask } -func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask { +func newOnStartExtensionTask(task *Task) *Task { + return cloneExtensionTask(task) +} + +func newOnErrorExtensionTask(download *Downloader, task *Task) *ExtensionTask { return &ExtensionTask{ download: download, - Task: newExtensionTaskSnapshot(task), + Task: cloneExtensionTask(task), } } +func newOnDoneExtensionTask(task *Task) *Task { + return cloneExtensionTask(task) +} + func (t *ExtensionTask) Continue() error { return t.download.Continue(&TaskFilter{ IDs: []string{t.ID}, diff --git a/pkg/rest/server_test.go b/pkg/rest/server_test.go index 7768c1a55..65b9da03c 100644 --- a/pkg/rest/server_test.go +++ b/pkg/rest/server_test.go @@ -935,10 +935,22 @@ func doTest(handler func()) { func doTest0(onStart func(cfg *model.StartConfig), handler func()) { testFunc := func(storage model.Storage) { + downloadDir, err := os.MkdirTemp("", "gopeed-rest-test-download-") + if err != nil { + panic(err) + } + defer os.RemoveAll(downloadDir) + oldCreatePath := createOpts.Path + createOpts.Path = downloadDir + defer func() { + createOpts.Path = oldCreatePath + }() + var cfg = &model.StartConfig{} cfg.Init() cfg.Storage = storage cfg.StorageDir = ".test_storage" + cfg.DownloadConfig = (&base.DownloaderStoreConfig{DownloadDir: downloadDir}).Init() cfg.WebEnable = true if onStart != nil { onStart(cfg)