cue: some API adjustments for new-style definitions

The API is now rather unsuitable for handling definitions.
Add some temporary workarounds until we have a new API.

Also updates some old-style defintions to new style.

Change-Id: Ib5c3210016aca971c45c7eddbc9dcb65f7fdc4e1
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/6000
Reviewed-by: Marcel van Lohuizen <mpvl@google.com>
diff --git a/cmd/cue/cmd/help.go b/cmd/cue/cmd/help.go
index 90218c6..a2240c9 100644
--- a/cmd/cue/cmd/help.go
+++ b/cmd/cue/cmd/help.go
@@ -416,7 +416,7 @@
 
 	command: [Name]: Command
 
-	Command :: {
+	Command: {
 		// Tasks specifies the things to run to complete a command. Tasks are
 		// typically underspecified and completed by the particular internal
 		// handler that is running them. Tasks can be a single task, or a full
@@ -447,7 +447,7 @@
 	}
 
 	// Name defines a valid task or command name.
-	Name :: =~#"^\PL([-](\PL|\PN))*$"#
+	Name: =~#"^\PL([-](\PL|\PN))*$"#
 
 	// A Task defines a step in the execution of a command.
 	Task: {
diff --git a/cue/builtins.go b/cue/builtins.go
index 969bc46..1dd46a7 100644
--- a/cue/builtins.go
+++ b/cue/builtins.go
@@ -3639,7 +3639,7 @@
 		$id:     =~"\\."
 		$after?: Task | [...Task]
 	}
-	Name :: =~"^\\PL([-](\\PL|\\PN))*$"
+	Name: =~"^\\PL([-](\\PL|\\PN))*$"
 }`,
 	},
 	"tool/cli": {
@@ -3739,8 +3739,8 @@
 	"tool/os": {
 		native: []*builtin{{}},
 		cue: `{
-	Name ::  !="" & !~"^[$]"
-	Value :: bool | number | *string | null
+	Name:  !="" & !~"^[$]"
+	Value: bool | number | *string | null
 	Setenv: {
 		$id: "tool/os.Setenv"
 		{[Name]: Value}
diff --git a/cue/examples_test.go b/cue/examples_test.go
index ddb3f78..889fdf1 100644
--- a/cue/examples_test.go
+++ b/cue/examples_test.go
@@ -73,7 +73,7 @@
 	inst, err := r.Compile("apis", `
 	// Release notes:
 	// - You can now specify your age and your hobby!
-	V1 :: {
+	#V1: {
 		age:   >=0 & <=100
 		hobby: string
 	}
@@ -81,7 +81,7 @@
 	// Release notes:
 	// - People get to be older than 100, so we relaxed it.
 	// - It seems not many people have a hobby, so we made it optional.
-	V2 :: {
+	#V2: {
 		age:    >=0 & <=150 // people get older now
 		hobby?: string      // some people don't have a hobby
 	}
@@ -89,7 +89,7 @@
 	// Release notes:
 	// - Actually no one seems to have a hobby nowadays anymore,
 	//   so we dropped the field.
-	V3 :: {
+	#V3: {
 		age: >=0 & <=150
 	}`)
 
@@ -97,9 +97,9 @@
 		fmt.Println(err)
 		// handle error
 	}
-	v1, err1 := inst.LookupField("V1")
-	v2, err2 := inst.LookupField("V2")
-	v3, err3 := inst.LookupField("V3")
+	v1, err1 := inst.Value().FieldByName("#V1", true)
+	v2, err2 := inst.Value().FieldByName("#V2", true)
+	v3, err3 := inst.Value().FieldByName("#V3", true)
 	if err1 != nil || err2 != nil || err3 != nil {
 		log.Println(err1, err2, err3)
 	}
@@ -113,5 +113,5 @@
 
 	// Output:
 	// V2 is backwards compatible with V1: <nil>
-	// V3 is backwards compatible with V2: V2: conflicting values <=150 and string (mismatched types number and string)
+	// V3 is backwards compatible with V2: #V2: conflicting values <=150 and string (mismatched types number and string)
 }
diff --git a/cue/export_test.go b/cue/export_test.go
index b531b23..79f6018 100644
--- a/cue/export_test.go
+++ b/cue/export_test.go
@@ -93,10 +93,10 @@
 	}, {
 		// Here the failed lookups are permanent failures as the structs are
 		// closed.
-		in: `{ a :: { b: 2.0, s: "abc" }, b: a.b, c: a.c, d: a["d"], e: a.t[2:3] }`,
+		in: `{ #a: { b: 2.0, s: "abc" }, b: #a.b, c: #a.c, d: #a["d"], e: #a.t[2:3] }`,
 		out: unindent(`
 			{
-				a :: {
+				#a: {
 					b: 2.0
 					s: "abc"
 				}
diff --git a/cue/format/testdata/expressions.golden b/cue/format/testdata/expressions.golden
index b5fd507..cfc4124 100644
--- a/cue/format/testdata/expressions.golden
+++ b/cue/format/testdata/expressions.golden
@@ -38,7 +38,7 @@
 		aaa: 10
 	}
 
-	someDefinition :: {
+	#someDefinition: {
 		embedding
 
 		field: 2
diff --git a/cue/format/testdata/expressions.input b/cue/format/testdata/expressions.input
index b2974d4..9ea36d0 100644
--- a/cue/format/testdata/expressions.input
+++ b/cue/format/testdata/expressions.input
@@ -38,7 +38,7 @@
         aaa: 10
     }
 
-    someDefinition :: {
+    #someDefinition: {
         embedding
 
         field: 2
diff --git a/cue/instance.go b/cue/instance.go
index 5cf2e9f..fe4c300 100644
--- a/cue/instance.go
+++ b/cue/instance.go
@@ -15,8 +15,6 @@
 package cue
 
 import (
-	goast "go/ast"
-
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/errors"
@@ -299,21 +297,24 @@
 // path is not. The empty path returns v itself.
 //
 // It cannot look up hidden or unexported fields.
+//
+// Deprecated: this API does not work with new-style definitions. Use
+// FieldByName defined on inst.Value().
 func (inst *Instance) LookupField(path ...string) (f FieldInfo, err error) {
 	idx := inst.index
 	ctx := idx.newContext()
 	v := newValueRoot(ctx, inst.rootValue)
-	for i, k := range path {
+	for _, k := range path {
 		s, err := v.Struct()
 		if err != nil {
 			return f, err
 		}
 
-		f, err = s.FieldByName(k)
+		f, err = s.FieldByName(k, true)
 		if err != nil {
 			return f, err
 		}
-		if f.IsHidden || (i == 0 || f.IsDefinition) && !goast.IsExported(f.Name) {
+		if f.IsHidden {
 			return f, errNotFound
 		}
 		v = f.Value
diff --git a/cue/types.go b/cue/types.go
index 7071dbe..e9a3dd7 100644
--- a/cue/types.go
+++ b/cue/types.go
@@ -18,7 +18,6 @@
 	"bytes"
 	"encoding/json"
 	"fmt"
-	goast "go/ast"
 	"io"
 	"math"
 	"math/big"
@@ -565,8 +564,10 @@
 		a = append(a, strconv.FormatInt(int64(v.index), 10))
 	case structKind:
 		f := idx.labelStr(v.arc.feature)
-		if !isIdent(f) && !isNumber(f) {
-			f = quote(f, '"')
+		if v.arc.feature&(hidden|definition) == 0 {
+			if !isIdent(f) && !isNumber(f) {
+				f = quote(f, '"')
+			}
 		}
 		a = append(a, f)
 	}
@@ -648,7 +649,7 @@
 	return Value{obj.ctx.index, &valueData{obj.path, uint32(i), a}}
 }
 
-// Dereference reports to the value v refers to if v is a reference or v itself
+// Dereference reports the value v refers to if v is a reference or v itself
 // otherwise.
 func Dereference(v Value) Value {
 	if v.path == nil {
@@ -680,12 +681,15 @@
 	}
 
 	cached := p.cache
-	if cached != nil {
+	if cached == nil {
 		cached = p.v.evalPartial(ctx)
 	}
 	s := cached.(*structLit)
 	for _, f := range a {
 		a := s.lookup(ctx, f)
+		if a.v == nil {
+			return Value{}
+		}
 		p = &valueData{parent: p, arc: a} // index
 		s, _ = a.cache.(*structLit)
 	}
@@ -1408,8 +1412,11 @@
 	return FieldInfo{str, i, v, a.definition, a.optional, a.feature&hidden != 0}
 }
 
-func (s *Struct) FieldByName(name string) (FieldInfo, error) {
-	f := s.v.ctx().strLabel(name)
+// FieldByName looks up a field for the given name. If isIdent is true, it will
+// look up a definition or hidden field (starting with `_` or `_#`). Otherwise
+// it interprets name as an arbitrary string for a regular field.
+func (s *Struct) FieldByName(name string, isIdent bool) (FieldInfo, error) {
+	f := s.v.ctx().label(name, isIdent)
 	for i, a := range s.s.arcs {
 		if a.feature == f {
 			return s.Field(i), nil
@@ -1491,18 +1498,31 @@
 
 var errNotFound = errors.Newf(token.NoPos, "field not found")
 
+// FieldByName looks up a field for the given name. If isIdent is true, it will
+// look up a definition or hidden field (starting with `_` or `_#`). Otherwise
+// it interprets name as an arbitrary string for a regular field.
+func (v Value) FieldByName(name string, isIdent bool) (f FieldInfo, err error) {
+	s, err := v.Struct()
+	if err != nil {
+		return f, err
+	}
+	return s.FieldByName(name, isIdent)
+}
+
 // LookupField reports information about a field of v.
+//
+// Deprecated: this API does not work with new-style definitions. Use FieldByName.
 func (v Value) LookupField(name string) (FieldInfo, error) {
 	s, err := v.Struct()
 	if err != nil {
 		// TODO: return a Value at the same location and a new error?
 		return FieldInfo{}, err
 	}
-	f, err := s.FieldByName(name)
+	f, err := s.FieldByName(name, true)
 	if err != nil {
 		return f, err
 	}
-	if f.IsHidden || f.IsDefinition && !goast.IsExported(name) {
+	if f.IsHidden {
 		return f, errNotFound
 	}
 	return f, err
diff --git a/cue/types_test.go b/cue/types_test.go
index ceac420..05a8df2 100644
--- a/cue/types_test.go
+++ b/cue/types_test.go
@@ -704,13 +704,13 @@
 func TestLookup(t *testing.T) {
 	var runtime = new(Runtime)
 	inst, err := runtime.Compile("x.cue", `
-V :: {
+#V: {
 	x: int
 }
-X :: {
+#X: {
 	[string]: int64
-} & V
-v: X
+} & #V
+v: #X
 `)
 	if err != nil {
 		t.Fatalf("compile: %v", err)
@@ -747,7 +747,7 @@
 			if err != nil {
 				t.Fatal(err)
 			}
-			fi, err := s.FieldByName(ref)
+			fi, err := s.FieldByName(ref, false)
 			if err != nil {
 				t.Fatal(err)
 			}
diff --git a/pkg/tool/doc.go b/pkg/tool/doc.go
index 51e62b2..6475f97 100644
--- a/pkg/tool/doc.go
+++ b/pkg/tool/doc.go
@@ -52,18 +52,19 @@
 //     	// long is a longer description that spans multiple lines and
 //     	// likely contain examples of usage of the command.
 //     	$long?: string
-//
-//     	// TODO: child commands.
 //     }
 //
+//     // TODO:
+//     // - child commands?
+//
 //     // Tasks defines a hierarchy of tasks. A command completes if all tasks have
 //     // run to completion.
 //     Tasks: Task | {
 //     	[name=Name]: Tasks
 //     }
 //
-//     // Name defines a valid task or command sname.
-//     Name :: =~#"^\PL(\PL|\PN|-(\PL|\PN))*$"#
+//     // #Name defines a valid task or command name.
+//     Name: =~#"^\PL([-](\PL|\PN))*$"#
 //
 //     // A Task defines a step in the execution of a command.
 //     Task: {
@@ -72,6 +73,15 @@
 //     	// kind indicates the operation to run. It must be of the form
 //     	// packagePath.Operation.
 //     	$id: =~#"\."#
+//
+//     	// $after can be used to specify a task is run after another one, when
+//     	// it does not otherwise refer to an output of that task.
+//     	$after?: Task | [...Task]
 //     }
 //
+//     // TODO: consider these options:
+//     //   $success: bool
+//     //   $runif: a.b.$success or $guard: a.b.$success
+//     // With this `$after: a.b` would just be a shorthand for `$guard: a.b.$success`.
+//
 package tool
diff --git a/pkg/tool/os/doc.go b/pkg/tool/os/doc.go
index b32321f..b4a66c1 100644
--- a/pkg/tool/os/doc.go
+++ b/pkg/tool/os/doc.go
@@ -6,10 +6,10 @@
 //
 //     // A Value are all possible values allowed in flags.
 //     // A null value unsets an environment variable.
-//     Value :: bool | number | *string | null
+//     Value: bool | number | *string | null
 //
 //     // Name indicates a valid flag name.
-//     Name :: !="" & !~"^[$]"
+//     Name: !="" & !~"^[$]"
 //
 //     // Setenv defines a set of command line flags, the values of which will be set
 //     // at run time. The doc comment of the flag is presented to the user in help.
@@ -19,14 +19,14 @@
 //     Setenv: {
 //         $id: "tool/os.Setenv"
 //
-//         [Name]: Value
+//         {[Name]: Value}
 //     }
 //
 //     // Getenv gets and parses the specific command line variables.
 //     Getenv: {
 //         $id: "tool/os.Getenv"
 //
-//         [Name]: Value
+//         {[Name]: Value}
 //     }
 //
 //     // Environ populates a struct with all environment variables.
@@ -37,7 +37,7 @@
 //         // Individual entries may be specified ahead of time to enable
 //         // validation and parsing. Values that are marked as required
 //         // will fail the task if they are not found.
-//         [Name]: Value
+//         {[Name]: Value}
 //     }
 //
 //     // Clearenv clears all environment variables.
diff --git a/pkg/tool/os/os.cue b/pkg/tool/os/os.cue
index 6881bd9..56e1724 100644
--- a/pkg/tool/os/os.cue
+++ b/pkg/tool/os/os.cue
@@ -1,11 +1,11 @@
 // Copyright 2019 The CUE Authors
-// 
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
+//
 //     http://www.apache.org/licenses/LICENSE-2.0
-// 
+//
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,10 +16,10 @@
 
 // A Value are all possible values allowed in flags.
 // A null value unsets an environment variable.
-Value :: bool | number | *string | null
+Value: bool | number | *string | null
 
 // Name indicates a valid flag name.
-Name :: !="" & !~"^[$]"
+Name: !="" & !~"^[$]"
 
 // Setenv defines a set of command line flags, the values of which will be set
 // at run time. The doc comment of the flag is presented to the user in help.
diff --git a/pkg/tool/tool.cue b/pkg/tool/tool.cue
index ed0bf2f..bac14b8 100644
--- a/pkg/tool/tool.cue
+++ b/pkg/tool/tool.cue
@@ -58,8 +58,8 @@
 	[name=Name]: Tasks
 }
 
-// Name defines a valid task or command name.
-Name :: =~#"^\PL([-](\PL|\PN))*$"#
+// #Name defines a valid task or command name.
+Name: =~#"^\PL([-](\PL|\PN))*$"#
 
 // A Task defines a step in the execution of a command.
 Task: {