diff --git a/book/src/command/db.md b/book/src/command/db.md index 25187360b..b37fee9fd 100644 --- a/book/src/command/db.md +++ b/book/src/command/db.md @@ -155,12 +155,12 @@ super db -c 'from logs | ts >= 2018-03-24T17:36:30.090766Z and ts <= 2018-03-24T When querying data to the [BSUP](../formats/bsup.md) output format, output from a pool can be easily piped to other commands like `super`, e.g., ``` -super db -f bsup -c 'from logs' | super -f table -c 'count() by field' - +super db -f bsup -c 'from logs' | super -s -c 'count() by field' - ``` Of course, it's even more efficient to run the query inside of the pool traversal like this: ``` -super db -f table -c 'from logs | count() by field' +super db -s -c 'from logs | count() by field' ``` By default, the `query` command scans pool data in sort-key order though the query optimizer may, in general, reorder the scan to optimize searches, diff --git a/book/src/tutorials/jq.md b/book/src/tutorials/jq.md index 8158bed9d..d30af6077 100644 --- a/book/src/tutorials/jq.md +++ b/book/src/tutorials/jq.md @@ -451,13 +451,12 @@ But more powerfully, types can be used anywhere a value can be used and in particular, they can be grouping keys, e.g., ```mdtest-command echo '{x:1,y:2}{s:"foo"}{x:3,y:4}' | - super -f table -c "count() by this['shape']:=typeof(this) | sort count" - + super -s -c "count() by this['shape']:=typeof(this) | sort count" - ``` produces ```mdtest-output -shape count -<{s:string}> 1 -<{x:int64,y:int64}> 2 +{shape:<{s:string}>,count:1} +{shape:<{x:int64,y:int64}>,count:2} ``` When run over large data sets, this gives you an insightful count of each "shape" of data in the input. This is a powerful building block for @@ -733,7 +732,7 @@ We can take the output from `blend | shapes` and list the fields with and their "kind". Note that when we do an `unnest this` with records as input, we get a new record value for each field structured as a key/value pair: ```mdtest-command-skip dir=book/src/tutorials -super -f table -c ' +super -s -c ' unnest this | blend | shapes @@ -967,7 +966,7 @@ to put your clean data into all the right places. Let's start with something simple. How about we output a "PR Report" listing the title of each PR along with its PR number and creation date: ```mdtest-command-skip dir=book/src/tutorials -super -f table -c '{DATE:created_at,NUMBER:f"PR #{number}",TITLE:title}' prs.bsup +super -s -c '{DATE:created_at,NUMBER:f"PR #{number}",TITLE:title}' prs.bsup ``` and you'll see this output... ```mdtest-output-skip head @@ -986,7 +985,7 @@ Instead of old PRs, we can get the latest list of PRs using the [`tail` operator](../super-sql/operators/tail.md) since we know the data is sorted chronologically. This command retrieves the last five PRs in the dataset: ```mdtest-command-skip dir=book/src/tutorials -super -f table -c ' +super -s -c ' tail 5 | {DATE:created_at,"NUMBER":f"PR #{number}",TITLE:title} ' prs.bsup diff --git a/cli/outputflags/flags.go b/cli/outputflags/flags.go index 1dbc46b57..fbf85eb65 100644 --- a/cli/outputflags/flags.go +++ b/cli/outputflags/flags.go @@ -69,7 +69,7 @@ func (f *Flags) SetFormatFlags(fs *flag.FlagSet) { f.Format = "bsup" } fUsage := fmt.Sprintf( - "format for output data [arrows,bsup,csv,db,json,line,parquet,sup,table,tsv,zeek] (default %s)", + "format for output data [arrows,bsup,csv,db,json,line,parquet,sup,tsv,zeek] (default %s)", f.Format) fs.Func("f", fUsage, func(s string) error { f.Format = s diff --git a/runtime/ztests/op/aggregate/count-assign.yaml b/runtime/ztests/op/aggregate/count-assign.yaml index d9d1d8521..0e97e2eff 100644 --- a/runtime/ztests/op/aggregate/count-assign.yaml +++ b/runtime/ztests/op/aggregate/count-assign.yaml @@ -1,6 +1,6 @@ spq: c:=count() -flags: -f table +flags: -f csv input: | {_path:"conn",foo:"1"} diff --git a/service/ztests/ls-pools.yaml b/service/ztests/ls-pools.yaml index f76b144b4..b210893aa 100644 --- a/service/ztests/ls-pools.yaml +++ b/service/ztests/ls-pools.yaml @@ -11,7 +11,7 @@ script: | # Verify that output flags are honored. super db ls -f sup | super -i sup -f sup -c "cut name | sort name" - echo === - super db ls -f bsup | super -i bsup -f table -c "cut name | sort name" - + super db ls -f bsup | super -i bsup -f csv -c "cut name | sort name" - echo === inputs: diff --git a/sio/anyio/writer.go b/sio/anyio/writer.go index 4889ed95d..d9fd10e05 100644 --- a/sio/anyio/writer.go +++ b/sio/anyio/writer.go @@ -14,7 +14,6 @@ import ( "github.com/superdb/super/sio/lineio" "github.com/superdb/super/sio/parquetio" "github.com/superdb/super/sio/supio" - "github.com/superdb/super/sio/tableio" "github.com/superdb/super/sio/zeekio" "github.com/superdb/super/vector" "github.com/superdb/super/vector/vio" @@ -54,8 +53,6 @@ func NewWriter(w io.WriteCloser, opts WriterOpts) (vio.PushCloser, error) { w = newDefuser(w) } return w, nil - case "table": - return newDefuser(tableio.NewWriter(w)), nil case "tsv": opts.CSV.Delim = '\t' return newDefuser(csvio.NewWriter(w, opts.CSV)), nil diff --git a/sio/sio.go b/sio/sio.go index 677dd0bba..cafc9c4ff 100644 --- a/sio/sio.go +++ b/sio/sio.go @@ -22,8 +22,6 @@ func Extension(format string) string { return ".parquet" case "sup": return ".sup" - case "table": - return ".tbl" case "zeek": return ".log" default: diff --git a/sio/tableio/writer.go b/sio/tableio/writer.go deleted file mode 100644 index 93c2c7944..000000000 --- a/sio/tableio/writer.go +++ /dev/null @@ -1,102 +0,0 @@ -package tableio - -import ( - "fmt" - "io" - "strings" - "text/tabwriter" - "time" - - "github.com/superdb/super" - "github.com/superdb/super/runtime/sam/expr" - "github.com/superdb/super/sbuf" - "github.com/superdb/super/sio/zeekio" - "github.com/superdb/super/sup" - "github.com/superdb/super/vector" -) - -type Writer struct { - writer io.WriteCloser - flattener *expr.Flattener - table *tabwriter.Writer - typ *super.TypeRecord - limit int - nline int -} - -func NewWriter(w io.WriteCloser) *Writer { - table := tabwriter.NewWriter(w, 0, 8, 1, ' ', 0) - return &Writer{ - writer: w, - flattener: expr.NewFlattener(super.NewContext()), - table: table, - limit: 1000, - } -} - -func (w *Writer) Push(vec vector.Any) error { - return sbuf.WriteVec(w, vec) -} - -func (w *Writer) Write(r super.Value) error { - if r.Type().Kind() != super.RecordKind { - return fmt.Errorf("table output encountered non-record value: %s", sup.FormatValue(r)) - } - r, err := w.flattener.Flatten(r) - if err != nil { - return err - } - if r.Type() != w.typ { - if w.typ != nil { - w.flush() - w.nline = 0 - } - // First time, or new descriptor, print header - typ := super.TypeRecordOf(r.Type()) - w.writeHeader(typ) - w.typ = typ - } - if w.nline >= w.limit { - w.flush() - w.writeHeader(w.typ) - w.nline = 0 - } - var out []string - for k, f := range r.Fields() { - var v string - value := r.DerefByColumn(k).MissingAsNull() - if f.Type == super.TypeTime { - if !value.IsNull() { - v = super.DecodeTime(value.Bytes()).Time().Format(time.RFC3339Nano) - } - } else { - v = zeekio.FormatValue(value) - } - out = append(out, v) - } - w.nline++ - _, err = fmt.Fprintf(w.table, "%s\n", strings.Join(out, "\t")) - return err -} - -func (w *Writer) flush() error { - return w.table.Flush() -} - -func (w *Writer) writeHeader(typ *super.TypeRecord) { - for i, f := range typ.Fields { - if i > 0 { - w.table.Write([]byte{'\t'}) - } - w.table.Write([]byte(f.Name)) - } - w.table.Write([]byte{'\n'}) -} - -func (w *Writer) Close() error { - err := w.flush() - if closeErr := w.writer.Close(); err == nil { - err = closeErr - } - return err -} diff --git a/sio/tableio/ztests/flatten.yaml b/sio/tableio/ztests/flatten.yaml deleted file mode 100644 index f4622d2e7..000000000 --- a/sio/tableio/ztests/flatten.yaml +++ /dev/null @@ -1,12 +0,0 @@ -skip: needs optional fields - -spq: pass - -flags: -f table - -input: | - {_path:"conn",ts:2015-03-05T14:25:14.419939Z,uid:"CogZFI3py5JsFZGik",id:null::(null|{orig_h:ip,orig_p:port=uint16,resp_h:ip,resp_p:port})} - -output: | - _path ts uid id.orig_h id.orig_p id.resp_h id.resp_p - conn 2015-03-05T14:25:14.419939Z CogZFI3py5JsFZGik - - - - diff --git a/sio/tableio/ztests/nested-record-named.yaml b/sio/tableio/ztests/nested-record-named.yaml deleted file mode 100644 index 96990b772..000000000 --- a/sio/tableio/ztests/nested-record-named.yaml +++ /dev/null @@ -1,11 +0,0 @@ -spq: pass - -flags: -f table - -input: | - type record_named={b:int64} - {a:{b:1}::record_named} - -output: | - a.b - 1 diff --git a/sio/tableio/ztests/non-record-error.yaml b/sio/tableio/ztests/non-record-error.yaml deleted file mode 100644 index 85d0ea03c..000000000 --- a/sio/tableio/ztests/non-record-error.yaml +++ /dev/null @@ -1,14 +0,0 @@ -spq: pass - -flags: -f table - -input: | - {a:1} - [1,2] - -output: | - a - 1 - -error: | - table output encountered non-record value: [1,2] diff --git a/sio/tableio/ztests/type-value.yaml b/sio/tableio/ztests/type-value.yaml deleted file mode 100644 index c42f49b50..000000000 --- a/sio/tableio/ztests/type-value.yaml +++ /dev/null @@ -1,10 +0,0 @@ -spq: 'cut typeof(this)' - -flags: -f table - -input: | - {a:1} - -output: | - typeof - <{a:int64}>