cmd/cue/cmd: allow commands to be documented by comments

Change-Id: I957030fe94ce64a1f4c9fcea003a0c50fc3254cd
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/4380
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/cmd/cue/cmd/custom.go b/cmd/cue/cmd/custom.go
index f508ae8..7ac1b95 100644
--- a/cmd/cue/cmd/custom.go
+++ b/cmd/cue/cmd/custom.go
@@ -26,6 +26,7 @@
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"sync"
 
 	"cuelang.org/go/cue"
@@ -44,12 +45,21 @@
 	taskSection    = "task"
 )
 
-func lookupString(obj cue.Value, key string) string {
+func lookupString(obj cue.Value, key, def string) string {
 	str, err := obj.Lookup(key).String()
-	if err != nil {
-		return ""
+	if err == nil {
+		def = str
 	}
-	return str
+	return strings.TrimSpace(def)
+}
+
+// splitLine splits the first line and the rest of the string.
+func splitLine(s string) (line, tail string) {
+	line = s
+	if p := strings.IndexByte(s, '\n'); p >= 0 {
+		line, tail = strings.TrimSpace(s[:p]), strings.TrimSpace(s[p+1:])
+	}
+	return
 }
 
 // Variables used for testing.
@@ -68,15 +78,26 @@
 	if !o.Exists() {
 		return nil, o.Err()
 	}
-
-	usage := lookupString(o, "usage")
-	if usage == "" {
+	docs := o.Doc()
+	var usage, short, long string
+	if len(docs) > 0 {
+		txt := docs[0].Text()
+		short, txt = splitLine(txt)
+		short = lookupString(o, "short", short)
+		if strings.HasPrefix(txt, "Usage:") {
+			usage, txt = splitLine(txt[len("Usage:"):])
+		}
+		usage = lookupString(o, "usage", usage)
+		usage = lookupString(o, "$usage", usage)
+		long = lookupString(o, "long", txt)
+	}
+	if !strings.HasPrefix(usage, name+" ") {
 		usage = name
 	}
 	sub := &cobra.Command{
 		Use:   usage,
-		Short: lookupString(o, "short"),
-		Long:  lookupString(o, "long"),
+		Short: lookupString(o, "$short", short),
+		Long:  lookupString(o, "$long", long),
 		RunE: mkRunE(c, func(cmd *Command, args []string) error {
 			// TODO:
 			// - parse flags and env vars
diff --git a/cmd/cue/cmd/root.go b/cmd/cue/cmd/root.go
index 896f43d..41198d7 100644
--- a/cmd/cue/cmd/root.go
+++ b/cmd/cue/cmd/root.go
@@ -57,10 +57,12 @@
 
 Commands are defined in CUE as follows:
 
-	command deploy: {
+	import "tool/exec"
+	command: deploy: {
+		exec.Run
 		cmd:   "kubectl"
 		args:  [ "-f", "deploy" ]
-		in:    json.Encode($) // encode the emitted configuration.
+		in:    json.Encode(userValue) // encode the emitted configuration.
 	}
 
 cue can also combine the results of http or grpc request with the input
@@ -270,12 +272,10 @@
 }
 
 func addSubcommands(cmd *Command, sub map[string]*subSpec, args []string) error {
-	if len(args) == 0 {
-		return nil
-	}
-
-	if _, ok := sub[args[0]]; ok {
-		args = args[1:]
+	if len(args) > 0 {
+		if _, ok := sub[args[0]]; ok {
+			args = args[1:]
+		}
 	}
 
 	if len(args) > 0 {
diff --git a/cmd/cue/cmd/testdata/script/help_cmd.txt b/cmd/cue/cmd/testdata/script/help_cmd.txt
new file mode 100644
index 0000000..ac15b70
--- /dev/null
+++ b/cmd/cue/cmd/testdata/script/help_cmd.txt
@@ -0,0 +1,234 @@
+cue help cmd
+cmp stdout expect-stdout
+
+-- cue.mod --
+-- task_tool.cue --
+package home
+
+import "tool/cli"
+
+// say hello to someone
+command: hello: {
+    task: say: {
+        cli.Print
+        text: "Hello world!"
+    }
+}
+
+// echo something back
+command: echo: {
+    task: echo: {
+        cli.Print
+        text: "ECHO Echo echo..."
+    }
+}
+
+-- expect-stdout --
+cmd executes defined the named command for each of the named instances.
+
+Commands define actions on instances. For example, they may specify
+how to upload a configuration to Kubernetes. Commands are defined
+directly in tool files, which are regular CUE files within the same
+package with a filename ending in _tool.cue. These are typically
+defined at the top of the module root so that they apply to all
+instances.
+
+Each command consists of one or more tasks. A task may load or write
+a file, consult a user on the command line, fetch a web page, and
+so on. Each task has inputs and outputs. Outputs are typically are
+filled out by the task implementation as the task completes.
+
+Inputs of tasks my refer to outputs of other tasks. The cue tool does
+a static analysis of the configuration and only starts tasks that are
+fully specified. Upon completion of each task, cue rewrites the instance,
+filling in the completed task, and reevaluates which other tasks can
+now start, and so on until all tasks have completed.
+
+Commands are defined at the top-level of the configuration:
+
+	command: [Name=string]: { // from tool.Command
+		// usage gives a short usage pattern of the command.
+		// Example:
+		//    fmt [-n] [-x] [packages]
+		usage?: Name | string
+
+		// short gives a brief on-line description of the command.
+		// Example:
+		//    reformat package sources
+		short?: string
+
+		// long gives a detailed description of the command, including a
+		// description of flags usage and examples.
+		long?: string
+
+		// A task defines a single action to be run as part of this command.
+		// Each task can have inputs and outputs, depending on the type
+		// task. The outputs are initially unspecified, but are filled out
+		// by the tooling
+		task: [string]: { // from "tool".Task
+			// supported fields depend on type
+		}
+
+		VarValue = string | bool | int | float | [...string|int|float]
+
+		// var declares values that can be set by command line flags or
+		// environment variables.
+		//
+		// Example:
+		//   // environment to run in
+		//   var env: "test" | "prod"
+		// The tool would print documentation of this flag as:
+		//   Flags:
+		//      --env string    environment to run in: test(default) or prod
+		var: [string]: VarValue
+
+		// flag defines a command line flag.
+		//
+		// Example:
+		//   var env: "test" | "prod"
+		//
+		//   // augment the flag information for var
+		//   flag env: {
+		//       shortFlag:   "e"
+		//       description: "environment to run in"
+		//   }
+		//
+		// The tool would print documentation of this flag as:
+		//   Flags:
+		//     -e, --env string    environment to run in: test(default), staging, or prod
+		//
+		flag [Name=_]: { // from "tool".Flag
+			// value defines the possible values for this flag.
+			// The default is string. Users can define default values by
+			// using disjunctions.
+			value: *env[Name].value | VarValue
+
+			// name, if set, allows var to be set with the command-line flag
+			// of the given name. null disables the command line flag.
+			name?: *Name | string
+
+			// short defines an abbreviated version of the flag.
+			// Disabled by default.
+			short?: string
+		}
+
+		// populate flag with the default values for
+		for k, v in var {
+			flag: { "\(k)": { value: v } | null  }
+		}
+
+		// env defines environment variables. It is populated with values
+		// for var.
+		//
+		// To specify a var without an equivalent environment variable,
+		// either specify it as a flag directly or disable the equally
+		// named env entry explicitly:
+		//
+		//     var foo: string
+		//     env foo: null  // don't use environment variables for foo
+		//
+		env: [Name=_]: {
+			// name defines the environment variable that sets this flag.
+			name?: *"CUE_VAR_" + strings.Upper(Name) | string
+
+			// The value retrieved from the environment variable or null
+			// if not set.
+			value?: string | bytes
+		}
+		env: {
+			for k, v in var {
+				"\(k)": { value: v } | null
+			}
+		}
+	}
+
+Available tasks can be found in the package documentation at
+
+	https://godoc.org/cuelang.org/go/pkg/tool
+
+More on tasks can be found in the tasks topic.
+
+Examples:
+
+A simple file using command line execution:
+
+	$ cat <<EOF > hello_tool.cue
+	package foo
+
+	import "tool/exec"
+
+	city: "Amsterdam"
+
+	// Say hello!
+	command: hello: {
+		// whom to say hello to
+		var: who: *"World" | string
+
+		task: print: exec.Run & {
+			cmd: "echo Hello \(var.who)! Welcome to \(city)."
+		}
+	}
+	EOF
+
+	$ cue cmd hello
+	Hello World! Welcome to Amsterdam.
+
+	$ cue cmd hello -who you  # Setting arguments is not supported yet by cue
+	Hello you! Welcome to Amsterdam.
+
+
+An example using pipes:
+
+	package foo
+
+	import "tool/exec"
+
+	city: "Amsterdam"
+
+	// Say hello!
+	command: hello: {
+		var: file: "out.txt" | string // save transcript to this file
+
+		task: ask: cli.Ask & {
+			prompt:   "What is your name?"
+			response: string
+		}
+
+		// starts after ask
+		task: echo: exec.Run & {
+			cmd:    ["echo", "Hello", task.ask.response + "!"]
+			stdout: string // capture stdout
+		}
+
+		// starts after echo
+		task: write: file.Append & {
+			filename: var.file
+			contents: task.echo.stdout
+		}
+
+		// also starts after echo
+		task: print: cli.Print & {
+			contents: task.echo.stdout
+		}
+	}
+
+Usage:
+  cue cmd <name> [-x] [instances] [flags]
+  cue cmd [command]
+
+Available Commands:
+  echo        echo something back
+  hello       say hello to someone
+
+Flags:
+  -h, --help   help for cmd
+
+Global Flags:
+      --debug            give detailed error info
+  -i, --ignore           proceed in the presence of errors
+  -p, --package string   CUE package to evaluate
+  -s, --simplify         simplify output
+      --trace            trace computation
+  -v, --verbose          print information about progress
+
+Use "cue cmd [command] --help" for more information about a command.
diff --git a/cmd/cue/cmd/testdata/script/help_hello.txt b/cmd/cue/cmd/testdata/script/help_hello.txt
new file mode 100644
index 0000000..f8b38ba
--- /dev/null
+++ b/cmd/cue/cmd/testdata/script/help_hello.txt
@@ -0,0 +1,37 @@
+cue help cmd hello
+cmp stdout expect-stdout
+
+-- cue.mod --
+-- task_tool.cue --
+package home
+
+import "tool/cli"
+
+// say hello to someone
+//
+//   Usage: hello
+//
+// Hello can be used to say hello to the world.
+command: hello: {
+    task: say: {
+        cli.Print
+        text: "Hello world!"
+    }
+}
+
+-- expect-stdout --
+Hello can be used to say hello to the world.
+
+Usage:
+  cue cmd hello [flags]
+
+Flags:
+  -h, --help   help for hello
+
+Global Flags:
+      --debug            give detailed error info
+  -i, --ignore           proceed in the presence of errors
+  -p, --package string   CUE package to evaluate
+  -s, --simplify         simplify output
+      --trace            trace computation
+  -v, --verbose          print information about progress
diff --git a/cue/builtins.go b/cue/builtins.go
index 5305379..5c2a585 100644
--- a/cue/builtins.go
+++ b/cue/builtins.go
@@ -3474,12 +3474,14 @@
 		native: []*builtin{{}},
 		cue: `{
 	Command: {
-		usage?: string
-		short?: string
-		long?:  string
 		tasks: {
 			[name=string]: Task
 		}
+		$type:   "tool.Command"
+		$name:   !=""
+		$usage?: =~"^\($name) "
+		$short?: string
+		$long?:  string
 	}
 	Task: {
 		kind: =~"\\."
@@ -3551,10 +3553,10 @@
 		response: {
 			body: *bytes | string
 			header: {
-				[Name=string]: string | [...string]
+				[string]: string | [...string]
 			}
 			trailer: {
-				[Name=string]: string | [...string]
+				[string]: string | [...string]
 			}
 			status:     string
 			statusCode: int
@@ -3563,10 +3565,10 @@
 		request: {
 			body: *bytes | string
 			header: {
-				[Name=string]: string | [...string]
+				[string]: string | [...string]
 			}
 			trailer: {
-				[Name=string]: string | [...string]
+				[string]: string | [...string]
 			}
 		}
 	}
diff --git a/pkg/tool/doc.go b/pkg/tool/doc.go
index 0ddc762..32ea0db 100644
--- a/pkg/tool/doc.go
+++ b/pkg/tool/doc.go
@@ -20,20 +20,34 @@
 // The following definitions are for defining commands in tool files:
 //
 //     // A Command specifies a user-defined command.
+//     //
+//     // Descriptions are derived from the doc comment, if they are not provided
+//     // structurally, using the following format:
+//     //
+//     //    // short description on one line
+//     //    //
+//     //    // Usage: <name> usage (optional)
+//     //    //
+//     //    // long description covering the remainder of the doc comment.
+//     //
 //     Command: {
+//     	$type: "tool.Command"
+//
+//     	$name: !=""
+//
 //     	//
 //     	// Example:
 //     	//     mycmd [-n] names
-//     	usage?: string
+//     	$usage?: =~"^\($name) "
 //
 //     	// short is short description of what the command does.
-//     	short?: string
+//     	$short?: string
 //
 //     	// long is a longer description that spans multiple lines and
 //     	// likely contain examples of usage of the command.
-//     	long?: string
+//     	$long?: string
 //
-//     	// TODO: define flags and environment variables.
+//     	// TODO: child commands.
 //
 //     	// tasks specifies the list of things to do to run command. Tasks are
 //     	// typically underspecified and completed by the particular internal
diff --git a/pkg/tool/http/doc.go b/pkg/tool/http/doc.go
index c587f90..2aa2260 100644
--- a/pkg/tool/http/doc.go
+++ b/pkg/tool/http/doc.go
@@ -17,30 +17,29 @@
 //
 //     	request: {
 //     		body: *bytes | string
-//     		header <Name>:  string | [...string]
-//     		trailer <Name>: string | [...string]
+//     		header: [string]:  string | [...string]
+//     		trailer: [string]: string | [...string]
 //     	}
 //     	response: {
 //     		status:     string
 //     		statusCode: int
 //
 //     		body: *bytes | string
-//     		header <Name>:  string | [...string]
-//     		trailer <Name>: string | [...string]
+//     		header: [string]:  string | [...string]
+//     		trailer: [string]: string | [...string]
 //     	}
 //     }
 //
-//     /* TODO: support serving once we have the cue serve command.
-//     Serve: {
-//     	port: int
-//
-//     	cert: string
-//     	key:  string
-//
-//     	handle <Pattern>: Message & {
-//     		pattern: Pattern
-//     	}
-//     }
-//     */
+//     //  TODO: support serving once we have the cue serve command.
+//     // Serve: {
+//     //  port: int
+//     //
+//     //  cert: string
+//     //  key:  string
+//     //
+//     //  handle <Pattern>: Message & {
+//     //   pattern: Pattern
+//     //  }
+//     // }
 //
 package http
diff --git a/pkg/tool/tool.cue b/pkg/tool/tool.cue
index ce50b09..ba61cbb 100644
--- a/pkg/tool/tool.cue
+++ b/pkg/tool/tool.cue
@@ -15,20 +15,34 @@
 package tool
 
 // A Command specifies a user-defined command.
+//
+// Descriptions are derived from the doc comment, if they are not provided
+// structurally, using the following format:
+//
+//    // short description on one line
+//    //
+//    // Usage: <name> usage (optional)
+//    //
+//    // long description covering the remainder of the doc comment.
+//
 Command: {
+	$type: "tool.Command"
+
+	$name: !=""
+
 	//
 	// Example:
 	//     mycmd [-n] names
-	usage?: string
+	$usage?: =~"^\($name) "
 
 	// short is short description of what the command does.
-	short?: string
+	$short?: string
 
 	// long is a longer description that spans multiple lines and
 	// likely contain examples of usage of the command.
-	long?: string
+	$long?: string
 
-	// TODO: define flags and environment variables.
+	// TODO: child commands.
 
 	// tasks specifies the list of things to do to run command. Tasks are
 	// typically underspecified and completed by the particular internal