handle missing fields in SQL as null

book/src/super-sql/sql/join.md

@@ -99,7 +99,7 @@ JOIN U ON x=z
 ---
 
 _Left outer join_
-```mdtest-spq-skip
+```mdtest-spq
 # spq
 WITH T(x,y) AS (
     VALUES (1,2), (3,4), (5,6)
@@ -114,15 +114,15 @@ ORDER BY x
 # input
 
 # expected output
-{x:1,y:2,z:error("missing")}
+{x:1,y:2,z:null}
 {x:3,y:4,z:3}
-{x:5,y:6,z:error("missing")}
+{x:5,y:6,z:null}
 ```
 
 ---
 
 _Right outer join_
-```mdtest-spq-skip
+```mdtest-spq
 # spq
 WITH T(x,y) AS (
     VALUES (1,2), (3,4), (5,6)
@@ -138,7 +138,7 @@ ORDER BY x
 
 # expected output
 {x:3,y:4,z:3}
-{x:error("missing"),y:error("missing"),z:2}
+{x:null,y:null,z:2}
 ```
 
 ---

book/src/super-sql/sql/values.md

@@ -59,15 +59,15 @@ FROM (VALUES ('hello, world'),('to be or not to be')) T(message)
 
 ---
 
-_Column variation filled in with missing values_
-```mdtest-spq-skip
+_Column variation filled in with null values_
+```mdtest-spq
 # spq
 SELECT * FROM (VALUES (1,2),(3)) T(x,y)
 # input
 
 # expected output
 {x:1,y:2}
-{x:3,y:error("missing")}
+{x:3,y:null}
 ```
 
----
+---

compiler/dag/expr.go

@@ -66,6 +66,7 @@ type (
 		LHS     Expr   `json:"lhs"`
 		RHS     string `json:"rhs"`
 		Noneish bool   `json:"noneish"`
+		Nullish bool   `json:"nullish"`
 	}
 	IndexExpr struct {
 		Kind  string `json:"kind" unpack:""`

compiler/optimizer/optimizer.go

@@ -599,11 +599,11 @@ func liftFilterOps(seq dag.Seq) dag.Seq {
 						return newErrorMissing()
 					}
 					// Copy spread so f and y don't share dag.Exprs.
-					e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain.Path())
+					e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain)
 					return e
 				}
 				// Copy e1 so f and y don't share dag.Exprs.
-				e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain.Path()[1:])
+				e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain[1:])
 				return e
 			})
 			if liftOK {
@@ -644,10 +644,10 @@ func mergeValuesOps(seq dag.Seq) dag.Seq {
 					if v1TopLevelSpread == nil {
 						return newErrorMissing()
 					}
-					e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain.Path())
+					e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain)
 					return e
 				}
-				e, mergeOK = addPathToExpr(v1Expr, this.Chain.Path()[1:])
+				e, mergeOK = addPathToExpr(v1Expr, this.Chain[1:])
 				return e
 			}
 			var mergedOp dag.Op
@@ -703,8 +703,8 @@ func hasThisWithEmptyPath(v any) bool {
 //   - It returns a dag.This when possible.
 //   - It descends to a dag.RecordExpr.Elem when possible.
 //   - It returns false when it cannot descend to a dag.RecordExpr.Elem.
-func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
-	if len(path) == 0 {
+func addPathToExpr(e dag.Expr, chain field.Chain) (dag.Expr, bool) {
+	if len(chain) == 0 {
 		return e, true
 	}
 	switch e := e.(type) {
@@ -713,14 +713,14 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
 		for _, elem := range slices.Backward(e.Elems) {
 			switch elem := elem.(type) {
 			case *dag.Field:
-				if elem.Name != path[0] {
+				if elem.Name != chain[0].ID {
 					continue
 				}
 				if spread != nil {
 					// Don't know which will win.
 					return e, false
 				}
-				return addPathToExpr(elem.Value, path[1:])
+				return addPathToExpr(elem.Value, chain[1:])
 			case *dag.Spread:
 				if spread != nil {
 					// Don't know which will win.
@@ -732,12 +732,12 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
 		if spread == nil {
 			return e, false
 		}
-		return addPathToExpr(spread.Expr, path)
+		return addPathToExpr(spread.Expr, chain)
 	case *dag.ThisExpr:
-		return dag.NewThis(slices.Concat(e.Chain, field.NewChain(path...))), true
+		return dag.NewThis(slices.Concat(e.Chain, chain)), true
 	}
-	for _, elem := range path {
-		e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem}
+	for _, elem := range chain {
+		e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem.ID, Noneish: elem.Noneish, Nullish: elem.Nullish}
 	}
 	return e, true
 }

compiler/rungen/vexpr.go

@@ -155,7 +155,7 @@ func (b *Builder) compileVamDotExpr(dot *dag.DotExpr) (vamexpr.Evaluator, error)
 	if err != nil {
 		return nil, err
 	}
-	return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish), nil
+	return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish, dot.Nullish), nil
 }
 
 func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, error) {

compiler/semantic/dagen.go

@@ -402,6 +402,7 @@ func (d *dagen) expr(e sem.Expr) dag.Expr {
 			LHS:     d.expr(e.LHS),
 			RHS:     e.RHS,
 			Noneish: e.Noneish,
+			Nullish: e.Nullish,
 		}
 	case *sem.IndexExpr:
 		return &dag.IndexExpr{

compiler/semantic/schema.go

@@ -395,7 +395,7 @@ func (j *joinUsingScope) star(n ast.Node, table string, path field.Path) ([]*sem
 			return nil, err
 		}
 		p := append(append(path, "left"), left...)
-		this := sem.NewThis(n, field.NewChain(p...))
+		this := sem.NewThis(n, field.NewChainNullish(p...))
 		out = append(out, this)
 	}
 	var err error
@@ -430,7 +430,7 @@ func (s *staticTable) star(n ast.Node, table string, path field.Path) ([]*sem.Th
 	var out []*sem.ThisExpr
 	if table == "" || s.table == table {
 		for _, col := range s.typ.Fields {
-			path := field.NewChain(path...).Append(col.Name, false)
+			path := field.NewChainNullish(path...).AppendNullish(col.Name)
 			out = append(out, sem.NewThis(n, path))
 		}
 	}

compiler/semantic/scope.go

@@ -198,7 +198,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super
 				return badExpr, t.checker.unknown
 			}
 		}
-		this := sem.NewThis(n, field.NewChain(append(out, path[1:]...)...))
+		this := sem.NewThis(n, field.NewChainNullish(append(out, path[1:]...)...))
 		return this, t.checker.this(n, this, inType)
 	}
 	if scope, ok := scope.(*selectScope); ok && scope.lateral && inputFirst {
@@ -227,7 +227,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super
 			return badExpr, t.checker.unknown
 		}
 		if out != nil {
-			this := sem.NewThis(n, field.NewChain(append(out, path[2:]...)...))
+			this := sem.NewThis(n, field.NewChainNullish(append(out, path[2:]...)...))
 			return this, t.checker.this(n, this, inType)
 		}
 		if p, _, _ := scope.resolveTable(n, path[0], nil); p != nil {
@@ -244,7 +244,7 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr {
 		return e
 	}
 	if this, ok := e.(*sem.ThisExpr); ok {
-		return sem.NewThis(n, append(this.Chain, field.NewChain(rest...)...))
+		return sem.NewThis(n, append(this.Chain, field.NewChainNullish(rest...)...))
 	}
 	out := &sem.DotExpr{
 		Node: n,
@@ -253,9 +253,10 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr {
 	}
 	for _, f := range rest[1:] {
 		out = &sem.DotExpr{
-			Node: n,
-			LHS:  out,
-			RHS:  f,
+			Node:    n,
+			LHS:     out,
+			RHS:     f,
+			Nullish: true,
 		}
 	}
 	return out

compiler/semantic/sem/expr.go

@@ -58,6 +58,7 @@ type (
 		LHS     Expr
 		RHS     string
 		Noneish bool
+		Nullish bool
 	}
 	IndexExpr struct {
 		ast.Node
@@ -336,6 +337,7 @@ func CopyExpr(e Expr) Expr {
 			LHS:     CopyExpr(e.LHS),
 			RHS:     e.RHS,
 			Noneish: e.Noneish,
+			Nullish: e.Nullish,
 		}
 	case *IndexExpr:
 		return &IndexExpr{

compiler/semantic/sql.go

@@ -413,7 +413,7 @@ func mapColumns(sctx *super.Context, in *super.TypeRecord, alias *ast.TableAlias
 			elems = append(elems, &sem.FieldElem{
 				Node:  alias.Columns[k],
 				Name:  out[k],
-				Value: sem.NewThis(alias.Columns[k], field.NewChain(in.Fields[k].Name)),
+				Value: sem.NewThis(alias.Columns[k], field.NewChainNullish(in.Fields[k].Name)),
 			})
 			fields = append(fields, super.NewField(out[k], in.Fields[k].Type))
 		}
@@ -634,8 +634,8 @@ func (t *translator) sqlJoinCond(cond ast.JoinCond, typ super.Type) sem.Expr {
 				t.error(id, fmt.Errorf("column %q in USING clause does not exist in right table", id.Name))
 				continue
 			}
-			lhs := sem.NewThis(id, field.NewChain(append([]string{"left"}, left...)...))
-			rhs := sem.NewThis(id, field.NewChain(append([]string{"right"}, right...)...))
+			lhs := sem.NewThis(id, field.NewChainNullish(append([]string{"left"}, left...)...))
+			rhs := sem.NewThis(id, field.NewChainNullish(append([]string{"right"}, right...)...))
 			exprs = append(exprs, sem.NewBinaryExpr(id, "==", lhs, rhs))
 		}
 		if len(exprs) == 0 {
@@ -711,7 +711,7 @@ func (t *translator) resolveOrdinalOuter(ts tableScope, n ast.Node, prefix strin
 		} else {
 			path = []string{ts.typ.Fields[col-1].Name}
 		}
-		return sem.NewThis(n, field.NewChain(path...))
+		return sem.NewThis(n, field.NewChainNullish(path...))
 	default:
 		panic(ts)
 	}

compiler/sfmt/shared.go

@@ -31,19 +31,26 @@ func (s *shared) fieldchain(chain field.Chain) {
 		return
 	}
 	for k, elem := range chain {
-		if elem.Noneish {
-			s.write("?")
-		}
-		if sup.IsIdentifier(elem.ID) {
-			if k != 0 {
+		if elem.Noneish || elem.Nullish {
+			if k == 0 {
+				s.write("this")
+			}
+			if elem.Noneish {
+				s.write("?")
+			} else {
+				s.write("!")
+			}
+			if k == 0 {
 				s.write(".")
 			}
+		}
+		if k != 0 {
+			s.write(".")
+		}
+		if sup.IsIdentifier(elem.ID) {
 			s.write(elem.ID)
 		} else {
-			if k == 0 {
-				s.write("this")
-			}
-			s.write("[%q]", elem.ID)
+			s.write("`%s`", elem.ID)
 		}
 	}
 }

compiler/sfmt/ztests/parallel.yaml

@@ -9,7 +9,7 @@ outputs:
       | fork
         (
           aggregate
-              count:=count() by x:=this["@foo"]
+              count:=count() by x:=`@foo`
         )
         (
           aggregate

compiler/ztests/merge-values.yaml

@@ -49,7 +49,7 @@ outputs:
       ===
       file f unordered fields a,b
       | aggregate
-          t0:=max(b) by k0:=a
+          t0:=max(this!.b) by k0:=this!.a
       | aggregate
           min:=min(t0) by a:=k0
       | output main

compiler/ztests/sql/agg-dups.yaml

@@ -6,9 +6,9 @@ outputs:
     data: |
       null
       | values {c0:1}, {c0:2}
-      | values {a:c0}
+      | values {a:this!.c0}
       | aggregate
-          t0:=max(a)
+          t0:=max(this!.a)
       | values {g:this,out:{max:t0,max_1:t0}}
       | sort g.t0 asc nulls last
       | values out

compiler/ztests/sql/as-implied.yaml

@@ -31,9 +31,9 @@ outputs:
       {x:4,y:4,z:2}
       ===
       file t.json format json fields a
-      | values {"a+1":a+1,"a+2":a+2,"a+1_1":a+1,"a+3":a+3,"a+1_2":a+1}
+      | values {"a+1":this!.a+1,"a+2":this!.a+2,"a+1_1":this!.a+1,"a+3":this!.a+3,"a+1_2":this!.a+1}
       | output main
       ===
       file t.json format json fields a
-      | values {"a+1":a+1,"a+1_1":a+1}
+      | values {"a+1":this!.a+1,"a+1_1":this!.a+1}
       | output main

compiler/ztests/sql/join-filter-pullup.yaml

@@ -23,14 +23,14 @@ outputs:
       | fork
         (
           values {id:1,team:"badgers"}
-          | where team=="badgers"
+          | where this!.team=="badgers"
         )
         (
           values {id:1,team_id:1,player:"blake",position:"P"}
-          | where position=="P"
+          | where this!.position=="P"
         )
-      | inner hashjoin as {left,right} on id==team_id
-      | values {id:left.id,team:left.team,id_1:right.id,team_id:right.team_id,player:right.player,position:right.position}
+      | inner hashjoin as {left,right} on this!.id==this!.team_id
+      | values {id:this!.left!.id,team:this!.left!.team,id_1:this!.right!.id,team_id:this!.right!.team_id,player:this!.right!.player,position:this!.right!.position}
       | output main
       // ===
       null
@@ -39,19 +39,19 @@ outputs:
           fork
             (
               values {a1:1}
-              | where a1 in [1,5,9] and a1 in set[1,5,9]
+              | where this!.a1 in [1,5,9] and this!.a1 in set[1,5,9]
             )
             (
               values {a2:1}
-              | where a2==1 or a2==2
+              | where this!.a2==1 or this!.a2==2
             )
           | cross join as {left,right}
         )
         (
           values {a3:1}
-          | where a3 in {c0:1,c1:5,c2:[9,11]}
+          | where this!.a3 in {c0:1,c1:5,c2:[9,11]}
         )
       | cross join as {left,right}
       | where 1==1
-      | values {a1:left.left.a1,a2:left.right.a2,a3:right.a3}
+      | values {a1:this!.left!.left!.a1,a2:this!.left!.right!.a2,a3:this!.right!.a3}
       | output main