refactor vcache and move loader to bsup/loader

runtime/vcache/any.go → bsup/loader/any.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/array.go → bsup/loader/array.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/bool.go → bsup/loader/bool.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/bytes.go → bsup/loader/bytes.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/const.go → bsup/loader/const.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super"

runtime/vcache/dict.go → bsup/loader/dict.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/dynamic.go → bsup/loader/dynamic.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"io"

runtime/vcache/empty.go → bsup/loader/empty.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super"

runtime/vcache/enum.go → bsup/loader/enum.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super/bsup"

runtime/vcache/error.go → bsup/loader/error.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/float.go → bsup/loader/float.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

bsup/loader/frame.go

@@ -0,0 +1,55 @@
+package loader
+
+import (
+	"github.com/superdb/super"
+	"github.com/superdb/super/bsup"
+	"github.com/superdb/super/pkg/field"
+	"github.com/superdb/super/vector"
+)
+
+// FrameLoader is the interface to load a BSUP column frame from storage into
+// memory and perform projections (or whole value reads) of the in-memory data.
+// This is also suitable for one-pass use where the data is read on demand,
+// used for processing, then discarded.  Frames maybe be accessed concurrently
+// the super.Context in use is passed in for each vector constructed from
+// its in-memory, sctx-independent shadow. This way one shadow can be used
+// across multiple queries with different sctx.
+type FrameLoader struct {
+	frame *bsup.ColFrame
+	root  shadow
+}
+
+func NewFrameLoader(frame *bsup.ColFrame) *FrameLoader {
+	return &FrameLoader{frame: frame}
+}
+
+// Load returns the indicated projection of data in this BSUP object.
+// If any required data is not memory resident, it will be fetched from
+// storage and cached in memory so that subsequent calls run from memory.
+// The vectors returned will have types from the provided sctx.  Multiple
+// Load calls to the same object may run concurrently.
+func (f *FrameLoader) Load(sctx *super.Context, projection field.Projection) (vector.Any, error) {
+	cctx := f.frame.Context()
+	f.root = newShadow(cctx, f.frame.Root())
+	f.root.unmarshal(cctx, projection)
+	loader := &loader{cctx, sctx, f.frame.DataReader()}
+	return loader.load(projection, f.root)
+}
+
+// LoadUnordered is like Load, but if o's root vector is dynamic,
+// LoadUnordered returns the underlying values vectors instead of a
+// vector.Dynamic.
+func (f *FrameLoader) LoadUnordered(vecs []vector.Any, sctx *super.Context, projection field.Projection) ([]vector.Any, error) {
+	cctx := f.frame.Context()
+	f.root = newShadow(cctx, f.frame.Root())
+	f.root.unmarshal(cctx, projection)
+	loader := &loader{cctx: cctx, sctx: sctx, r: f.frame.DataReader()}
+	if d, ok := f.root.(*dynamic); ok {
+		return d.projectUnordered(vecs, loader, projection), nil
+	}
+	vec, err := loader.load(projection, f.root)
+	if err != nil {
+		return nil, err
+	}
+	return append(vecs, vec), nil
+}

runtime/vcache/fusion.go → bsup/loader/fusion.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/int.go → bsup/loader/int.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/ip.go → bsup/loader/ip.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"net/netip"

runtime/vcache/loader.go → bsup/loader/loader.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"io"

runtime/vcache/map.go → bsup/loader/map.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/named.go → bsup/loader/named.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super/bsup"

runtime/vcache/net.go → bsup/loader/net.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"net/netip"

runtime/vcache/none.go → bsup/loader/none.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super/bsup"

runtime/vcache/null.go → bsup/loader/null.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"github.com/superdb/super/bsup"

runtime/vcache/option.go → bsup/loader/option.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/record.go → bsup/loader/record.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"fmt"

runtime/vcache/set.go → bsup/loader/set.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/shadow.go → bsup/loader/shadow.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"fmt"

runtime/vcache/type.go → bsup/loader/type.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"io"

runtime/vcache/uint.go → bsup/loader/uint.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

runtime/vcache/union.go → bsup/loader/union.go

@@ -1,4 +1,4 @@
-package vcache
+package loader
 
 import (
 	"sync"

cmd/super/dev/vector/copy/command.go

@@ -6,12 +6,12 @@ import (
 
 	"github.com/superdb/super"
 	"github.com/superdb/super/bsup"
+	"github.com/superdb/super/bsup/loader"
 	"github.com/superdb/super/cli/outputflags"
 	"github.com/superdb/super/cmd/super/dev/vector"
 	"github.com/superdb/super/pkg/charm"
 	"github.com/superdb/super/pkg/storage"
 	"github.com/superdb/super/runtime"
-	"github.com/superdb/super/runtime/vcache"
 	"github.com/superdb/super/sbuf"
 	"github.com/superdb/super/vector/vio"
 )
@@ -72,14 +72,13 @@ func (c *Command) Run(args []string) error {
 	if !ok {
 		return errors.New("input not in BSUP column form")
 	}
-	object := vcache.NewReader(colFrame)
-	defer object.Close()
+	loader := loader.NewFrameLoader(colFrame)
 	writer, err := c.outputFlags.Open(ctx, local)
 	if err != nil {
 		return err
 	}
 	sctx := super.NewContext()
-	puller := runtime.NewProjection(sctx, object, nil)
+	puller := runtime.NewProjection(sctx, loader, nil)
 	if err := vio.Copy(writer, sbuf.NewDematerializer(sctx, puller)); err != nil {
 		writer.Close()
 		return err

cmd/super/dev/vector/project/command.go

@@ -11,10 +11,7 @@ import (
 	"github.com/superdb/super/pkg/charm"
 	"github.com/superdb/super/pkg/field"
 	"github.com/superdb/super/pkg/storage"
-	"github.com/superdb/super/runtime"
 	"github.com/superdb/super/runtime/vcache"
-	"github.com/superdb/super/sbuf"
-	"github.com/superdb/super/vector/vio"
 )
 
 var spec = &charm.Spec{
@@ -70,16 +67,19 @@ func (c *Command) Run(args []string) error {
 	if err != nil {
 		return err
 	}
-	defer object.Close()
-	sctx := super.NewContext()
-	projection := runtime.NewProjection(sctx, object, paths)
 	writer, err := c.outputFlags.Open(ctx, local)
-	if err != nil {
-		return err
-	}
-	if err := vio.Copy(writer, sbuf.NewDematerializer(sctx, projection)); err != nil {
-		writer.Close()
-		return err
+	projection := field.NewProjection(paths)
+	sctx := super.NewContext()
+	for _, loader := range object.Loaders() {
+		vec, err := loader.Load(sctx, projection)
+		if err != nil {
+			writer.Close()
+			return err
+		}
+		if err := writer.Push(vec); err != nil {
+			writer.Close()
+			return err
+		}
 	}
 	return writer.Close()
 }

runtime/projection.go

@@ -2,39 +2,39 @@ package runtime
 
 import (
 	"github.com/superdb/super"
+	"github.com/superdb/super/bsup/loader"
 	"github.com/superdb/super/pkg/field"
-	"github.com/superdb/super/runtime/vcache"
 	"github.com/superdb/super/sbuf"
 	"github.com/superdb/super/vector"
 	"github.com/superdb/super/vector/vio"
 )
 
 type Projection struct {
 	sctx       *super.Context
-	object     *vcache.Object
+	loader     *loader.FrameLoader
 	projection field.Projection
 }
 
-func NewProjection(sctx *super.Context, o *vcache.Object, paths []field.Path) sbuf.Puller {
+func NewProjection(sctx *super.Context, frame *loader.FrameLoader, paths []field.Path) sbuf.Puller {
 	return sbuf.NewMaterializer(&Projection{
 		sctx:       sctx,
-		object:     o,
+		loader:     frame,
 		projection: field.NewProjection(paths),
 	})
 }
 
-func NewVectorProjection(sctx *super.Context, o *vcache.Object, paths []field.Path) vio.Puller {
+func NewVectorProjection(sctx *super.Context, frame *loader.FrameLoader, paths []field.Path) vio.Puller {
 	return &Projection{
 		sctx:       sctx,
-		object:     o,
+		loader:     frame,
 		projection: field.NewProjection(paths),
 	}
 }
 
 func (p *Projection) Pull(bool) (vector.Any, error) {
-	if o := p.object; o != nil {
-		p.object = nil
-		return o.Fetch(p.sctx, p.projection)
+	if loader := p.loader; loader != nil {
+		p.loader = nil
+		return loader.Load(p.sctx, p.projection)
 	}
 	return nil, nil
 }

runtime/vcache/cache.go

@@ -2,7 +2,6 @@ package vcache
 
 import (
 	"context"
-	"errors"
 	"sync"
 
 	"github.com/segmentio/ksuid"
@@ -11,6 +10,11 @@ import (
 	"github.com/superdb/super/pkg/storage"
 )
 
+// Cache manages all accesses to storage objects used by databases
+// providing a means to retrieve metadata, fused types, and column
+// frames from objects effeciently from storage and cached in memory.
+// It works for both BSUP columns and rows frames but is generally used
+// for BSUP columns data stored in databases.
 type Cache struct {
 	mu     sync.Mutex
 	engine storage.Engine
@@ -72,16 +76,10 @@ func (c *Cache) Fetch(ctx context.Context, uri *storage.URI, id ksuid.KSUID) (*O
 	}
 	// XXX we need to refactor this interface since it's no longer aligned one
 	// cached entity per file (cache should operate on frames like parquet row groups)
-	fit := bsup.NewSeekable(super.NewContext(), r)
-	frame, err := fit.Next()
+	object, err = NewObject(bsup.NewSeekable(super.NewContext(), r))
 	if err != nil {
 		return nil, err
 	}
-	colFrame, ok := frame.(*bsup.ColFrame)
-	if !ok {
-		return nil, errors.New("input not in BSUP column form")
-	}
-	object = NewReader(colFrame)
 	c.mu.Lock()
 	c.objects[id] = object
 	c.mu.Unlock()