cue: support optional field lookup in LookupPath

This is done by allowing Selectors to be converted
to an optional form (if they aren't already).

It also supports the use of the Selectors AnyIndex and
AnyString for getting `T` for  `[string]: T` and `[...T]`.

This also allows deprecating Elem and Template, which
will be done in a separate CL.

Change-Id: I6074382f12259c3dd87557471d80f82720a84779
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/9347
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/cue/path.go b/cue/path.go
index 29edd0b..4ffb2db 100644
--- a/cue/path.go
+++ b/cue/path.go
@@ -70,11 +70,18 @@
 	anyLabel  = Selector{sel: anySelector(adt.AnyRegular)}
 )
 
+// Optional converts sel into an optional equivalent.
+//     foo -> foo?
+func (sel Selector) Optional() Selector {
+	return wrapOptional(sel)
+}
+
 type selector interface {
 	String() string
 
 	feature(ctx adt.Runtime) adt.Feature
 	kind() adt.FeatureType
+	optional() bool
 }
 
 // A Path is series of selectors to query a CUE value.
@@ -144,6 +151,17 @@
 	return b.String()
 }
 
+// Optional returns the optional form of a Path. For instance,
+//    foo.bar  --> foo?.bar?
+//
+func (p Path) Optional() Path {
+	q := make([]Selector, 0, len(p.path))
+	for _, s := range p.path {
+		q = appendSelector(q, wrapOptional(s))
+	}
+	return Path{path: q}
+}
+
 func toSelectors(expr ast.Expr) []Selector {
 	switch x := expr.(type) {
 	case *ast.Ident:
@@ -293,6 +311,7 @@
 func (s scopedSelector) String() string {
 	return s.name
 }
+func (scopedSelector) optional() bool { return false }
 
 func (s scopedSelector) kind() adt.FeatureType {
 	switch {
@@ -331,6 +350,8 @@
 	return string(d)
 }
 
+func (d definitionSelector) optional() bool { return false }
+
 func (d definitionSelector) kind() adt.FeatureType {
 	return adt.DefinitionLabel
 }
@@ -354,6 +375,7 @@
 	return str
 }
 
+func (s stringSelector) optional() bool        { return false }
 func (s stringSelector) kind() adt.FeatureType { return adt.StringLabel }
 
 func (s stringSelector) feature(r adt.Runtime) adt.Feature {
@@ -376,6 +398,7 @@
 }
 
 func (s indexSelector) kind() adt.FeatureType { return adt.IntLabel }
+func (s indexSelector) optional() bool        { return false }
 
 func (s indexSelector) feature(r adt.Runtime) adt.Feature {
 	return adt.Feature(s)
@@ -385,6 +408,7 @@
 type anySelector adt.Feature
 
 func (s anySelector) String() string        { return "_" }
+func (s anySelector) optional() bool        { return true }
 func (s anySelector) kind() adt.FeatureType { return adt.Feature(s).Typ() }
 
 func (s anySelector) feature(r adt.Runtime) adt.Feature {
@@ -398,16 +422,27 @@
 // func ImportPath(s string) Selector {
 // 	return importSelector(s)
 // }
+type optionalSelector struct {
+	selector
+}
 
-// type importSelector string
+func wrapOptional(sel Selector) Selector {
+	if !sel.sel.optional() {
+		sel = Selector{optionalSelector{sel.sel}}
+	}
+	return sel
+}
 
-// func (s importSelector) String() string {
-// 	return literal.String.Quote(string(s))
+// func isOptional(sel selector) bool {
+// 	_, ok := sel.(optionalSelector)
+// 	return ok
 // }
 
-// func (s importSelector) feature(r adt.Runtime) adt.Feature {
-// 	return adt.InvalidLabel
-// }
+func (s optionalSelector) optional() bool { return true }
+
+func (s optionalSelector) String() string {
+	return s.selector.String() + "?"
+}
 
 // TODO: allow looking up in parent scopes?
 
@@ -429,6 +464,7 @@
 }
 
 func (p pathError) String() string        { return p.Error.Error() }
+func (p pathError) optional() bool        { return false }
 func (p pathError) kind() adt.FeatureType { return 0 }
 func (p pathError) feature(r adt.Runtime) adt.Feature {
 	return adt.InvalidLabel
diff --git a/cue/query.go b/cue/query.go
index 3cef7c2..542a9af 100644
--- a/cue/query.go
+++ b/cue/query.go
@@ -55,7 +55,12 @@
 
 // LookupPath reports the value for path p relative to v.
 func (v Value) LookupPath(p Path) Value {
+	if v.v == nil {
+		return Value{}
+	}
 	n := v.v
+	ctx := v.ctx().opCtx
+
 outer:
 	for _, sel := range p.path {
 		f := sel.sel.feature(v.idx.Runtime)
@@ -65,7 +70,18 @@
 				continue outer
 			}
 		}
-		// TODO: if optional, look up template for name.
+		if sel.sel.optional() {
+			x := &adt.Vertex{
+				Parent: v.v,
+				Label:  sel.sel.feature(ctx),
+			}
+			n.MatchAndInsert(ctx, x)
+			if len(x.Conjuncts) > 0 {
+				x.Finalize(ctx)
+				n = x
+				continue
+			}
+		}
 
 		var x *adt.Bottom
 		if err, ok := sel.sel.(pathError); ok {
diff --git a/cue/query_test.go b/cue/query_test.go
new file mode 100644
index 0000000..5301be2
--- /dev/null
+++ b/cue/query_test.go
@@ -0,0 +1,125 @@
+// Copyright 2021 CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cue_test
+
+import (
+	"bytes"
+	"testing"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/internal/diff"
+)
+
+func TestLookupPath(t *testing.T) {
+	r := &cue.Runtime{}
+
+	testCases := []struct {
+		in       string
+		path     cue.Path
+		out      string `test:"update"` // :nerdSnipe:
+		notExist bool   `test:"update"` // :nerdSnipe:
+	}{{
+		in: `
+		[Name=string]: { a: Name }
+		`,
+		path:     cue.MakePath(cue.Str("a")),
+		notExist: true,
+	}, {
+		in: `
+		#V: {
+			x: int
+		}
+		#X: {
+			[string]: int64
+		} & #V
+		v: #X
+		`,
+		path: cue.ParsePath("v.x"),
+		out:  `int64`,
+	}, {
+		in: `
+		a: [...int]
+		`,
+		path: cue.MakePath(cue.Str("a"), cue.AnyIndex),
+		out:  `int`,
+	}, {
+		in: `
+		[Name=string]: { a: Name }
+		`,
+		path: cue.MakePath(cue.AnyString, cue.Str("a")),
+		out:  `string`,
+	}, {
+		in: `
+		[Name=string]: { a: Name }
+		`,
+		path: cue.MakePath(cue.Str("b").Optional(), cue.Str("a")),
+		out:  `"b"`,
+	}, {
+		in: `
+		[Name=string]: { a: Name }
+		`,
+		path: cue.MakePath(cue.AnyString),
+		out:  `{a: string}`,
+	}, {
+		in: `
+		a: [Foo=string]: [Bar=string]: { b: Foo+Bar }
+		`,
+		path: cue.MakePath(cue.Str("a"), cue.Str("b"), cue.Str("c")).Optional(),
+		out:  `{b: "bc"}`,
+	}, {
+		in: `
+		a: [Foo=string]: b: [Bar=string]: { c: Foo }
+		a: foo: b: [Bar=string]: { d: Bar }
+		`,
+		path: cue.MakePath(cue.Str("a"), cue.Str("foo"), cue.Str("b"), cue.AnyString),
+		out:  `{c: "foo", d: string}`,
+	}, {
+		in: `
+		[Name=string]: { a: Name }
+		`,
+		path:     cue.MakePath(cue.Str("a")),
+		notExist: true,
+	}}
+	for _, tc := range testCases {
+		t.Run(tc.path.String(), func(t *testing.T) {
+			v := compileT(t, r, tc.in)
+
+			v = v.LookupPath(tc.path)
+
+			if exists := v.Exists(); exists != !tc.notExist {
+				t.Fatalf("exists: got %v; want: %v", exists, !tc.notExist)
+			} else if !exists {
+				return
+			}
+
+			w := compileT(t, r, tc.out)
+
+			if k, d := diff.Diff(v, w); k != diff.Identity {
+				b := &bytes.Buffer{}
+				diff.Print(b, d)
+				t.Error(b)
+			}
+		})
+	}
+}
+
+func compileT(t *testing.T, r *cue.Runtime, s string) cue.Value {
+	t.Helper()
+	inst, err := r.Compile("", s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return inst.Value()
+}
diff --git a/internal/core/adt/feature.go b/internal/core/adt/feature.go
index d2f4111..d6f44b2 100644
--- a/internal/core/adt/feature.go
+++ b/internal/core/adt/feature.go
@@ -127,6 +127,7 @@
 	if !f.IsRegular() {
 		panic("not a regular label")
 	}
+	// TODO: Handle special regular values: invalid and AnyRegular.
 	if f.IsInt() {
 		return ctx.NewInt64(int64(f.Index()))
 	}
diff --git a/internal/core/adt/optional.go b/internal/core/adt/optional.go
index bcb8e65..a58947a8 100644
--- a/internal/core/adt/optional.go
+++ b/internal/core/adt/optional.go
@@ -36,18 +36,22 @@
 		}
 	}
 
-	if !arc.Label.IsRegular() {
+	f := arc.Label
+	if !f.IsRegular() {
 		return
 	}
+	if int64(f.Index()) == MaxIndex {
+		f = 0
+	}
 
 	var label Value
-	if o.types&HasComplexPattern != 0 && arc.Label.IsString() {
-		label = arc.Label.ToValue(c)
+	if o.types&HasComplexPattern != 0 && f.IsString() {
+		label = f.ToValue(c)
 	}
 
 	if len(o.Bulk) > 0 {
 		bulkEnv := *env
-		bulkEnv.DynamicLabel = arc.Label
+		bulkEnv.DynamicLabel = f
 		bulkEnv.Deref = nil
 		bulkEnv.Cycles = nil
 
@@ -56,7 +60,7 @@
 			// if matched && f.additional {
 			// 	continue
 			// }
-			if matchBulk(c, env, b, arc.Label, label) {
+			if matchBulk(c, env, b, f, label) {
 				matched = true
 				info := closeInfo.SpawnSpan(b.Value, ConstraintSpan)
 				arc.AddConjunct(MakeConjunct(&bulkEnv, b, info))