cmd/cue: first implementation of cue tool
Change-Id: I9f6882d60c1f0dc6e4184fb8883f86ed0cacfa6e
diff --git a/cmd/cue/cmd/cmd.go b/cmd/cue/cmd/cmd.go
new file mode 100644
index 0000000..b1b7f6a
--- /dev/null
+++ b/cmd/cue/cmd/cmd.go
@@ -0,0 +1,220 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+)
+
+// TODO: generate long description from documentation.
+
+// cmdCmd represents the cmd command
+var cmdCmd = &cobra.Command{
+ Use: "cmd <name> [-x] [instances]",
+ Short: "run a user-defined shell command",
+ Long: `cmd executes defined the named command for each of the named instances.
+
+Commands define actions on instances. For example, they may specify
+how to upload a configuration to Kubernetes. Commands are defined
+directly in tool files, which are regular CUE files within the same
+package with a filename ending in _tool.cue. These are typically
+defined at the top of the module root so that they apply to all
+instances.
+
+Each command consists of one or more tasks. A task may load or write
+a file, consult a user on the command line, fetch a web page, and
+so on. Each task has inputs and outputs. Outputs are typically are
+filled out by the task implementation as the task completes.
+
+Inputs of tasks my refer to outputs of other tasks. The cue tool does
+a static analysis of the configuration and only starts tasks that are
+fully specified. Upon completion of each task, cue rewrites the instance,
+filling in the completed task, and reevaluates which other tasks can
+now start, and so on until all tasks have completed.
+
+Commands are defined at the top-level of the configuration:
+
+ command <Name>: { // from tool.Command
+ // usage gives a short usage pattern of the command.
+ // Example:
+ // fmt [-n] [-x] [packages]
+ usage: Name | string
+
+ // short gives a brief on-line description of the command.
+ // Example:
+ // reformat package sources
+ short: "" | string
+
+ // long gives a detailed description of the command, including a
+ // description of flags usage and examples.
+ long: "" | string
+
+ // A task defines a single action to be run as part of this command.
+ // Each task can have inputs and outputs, depending on the type
+ // task. The outputs are initially unspecified, but are filled out
+ // by the tooling
+ task <Name>: { // from "tool".Task
+ // supported fields depend on type
+ }
+
+ VarValue = string | bool | int | float | [...string|int|float]
+
+ // var declares values that can be set by command line flags or
+ // environment variables.
+ //
+ // Example:
+ // // environment to run in
+ // var env: "test" | "prod"
+ // The tool would print documentation of this flag as:
+ // Flags:
+ // --env string environment to run in: test(default) or prod
+ var <Name>: VarValue
+
+ // flag defines a command line flag.
+ //
+ // Example:
+ // var env: "test" | "prod"
+ //
+ // // augment the flag information for var
+ // flag env: {
+ // shortFlag: "e"
+ // description: "environment to run in"
+ // }
+ //
+ // The tool would print documentation of this flag as:
+ // Flags:
+ // -e, --env string environment to run in: test(default), staging, or prod
+ //
+ flag <Name>: { // from "tool".Flag
+ // value defines the possible values for this flag.
+ // The default is string. Users can define default values by
+ // using disjunctions.
+ value: env[Name].value | VarValue
+
+ // name, if set, allows var to be set with the command-line flag
+ // of the given name. null disables the command line flag.
+ name: Name | null | string
+
+ // short defines an abbreviated version of the flag.
+ // Disabled by default.
+ short: null | string
+ }
+
+ // populate flag with the default values for
+ flag: { "\(k)": { value: v } | null for k, v in var }
+
+ // env defines environment variables. It is populated with values
+ // for var.
+ //
+ // To specify a var without an equivalent environment variable,
+ // either specify it as a flag directly or disable the equally
+ // named env entry explicitly:
+ //
+ // var foo: string
+ // env foo: null // don't use environment variables for foo
+ //
+ env <Name>: {
+ // name defines the environment variable that sets this flag.
+ name: "CUE_VAR_" + strings.Upper(Name) | string | null
+
+ // The value retrieved from the environment variable or null
+ // if not set.
+ value: null | string | bytes
+ }
+ env: { "\(k)": { value: v } | null for k, v in var }
+ }
+
+Available tasks can be found in the package documentation at
+
+ cuelang.org/pkg/tool.
+
+More on tasks can be found in the tasks topic.
+
+Examples:
+
+A simple file using command line execution:
+
+ $ cat <<EOF > hello_tool.cue
+ package foo
+
+ import "tool/exec"
+
+ city: "Amsterdam"
+
+ // Say hello!
+ command hello: {
+ // whom to say hello to
+ var who: "World" | string
+
+ task print: exec.Run({
+ cmd: "echo Hello \(var.who)! Welcome to \(city)."
+ })
+ }
+ EOF
+
+ $ cue cmd echo
+ Hello World! Welcome to Amsterdam.
+
+ $ cue cmd echo -who you
+ Hello you! Welcome to Amsterdam.
+
+
+An example using pipes:
+
+ package foo
+
+ import "tool/exec"
+
+ city: "Amsterdam"
+
+ // Say hello!
+ command hello: {
+ var file: "out.txt" | string // save transcript to this file
+
+ task ask: cli.Ask({
+ prompt: "What is your name?"
+ response: string
+ })
+
+ // starts after ask
+ task echo: exec.Run({
+ cmd: ["echo", "Hello", task.ask.response + "!"]
+ stdout: string // capture stdout
+ })
+
+ // starts after echo
+ task write: file.Append({
+ filename: var.file
+ contents: task.echo.stdout
+ })
+
+ // also starts after echo
+ task print: cli.Print({
+ contents: task.echo.stdout
+ })
+ }
+
+`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ fmt.Println("cmd run but shouldn't")
+ return nil
+ },
+}
+
+func init() {
+ RootCmd.AddCommand(cmdCmd)
+}
diff --git a/cmd/cue/cmd/cmd_test.go b/cmd/cue/cmd/cmd_test.go
new file mode 100644
index 0000000..514a49d
--- /dev/null
+++ b/cmd/cue/cmd/cmd_test.go
@@ -0,0 +1,50 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "testing"
+
+ "cuelang.org/go/cue/errors"
+ "github.com/spf13/cobra"
+)
+
+func TestCmd(t *testing.T) {
+ testCases := []string{
+ "echo",
+ "run",
+ "run_list",
+ "baddisplay",
+ "http",
+ }
+ for _, name := range testCases {
+ run := func(cmd *cobra.Command, args []string) error {
+ testOut = cmd.OutOrStdout()
+ defer func() { testOut = nil }()
+
+ tools := buildTools(RootCmd, args)
+ cmd, err := addCustom(RootCmd, "command", name, tools)
+ if err != nil {
+ return err
+ }
+ err = executeTasks("command", name, tools)
+ if err != nil {
+ errors.Print(testOut, err)
+ }
+ return nil
+ }
+ runCommand(t, run, "cmd_"+name)
+ }
+}
diff --git a/cmd/cue/cmd/common.go b/cmd/cue/cmd/common.go
new file mode 100644
index 0000000..1f93ab9
--- /dev/null
+++ b/cmd/cue/cmd/common.go
@@ -0,0 +1,86 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "cuelang.org/go/cue"
+ "cuelang.org/go/cue/build"
+ "cuelang.org/go/cue/errors"
+ "cuelang.org/go/cue/load"
+ "github.com/spf13/cobra"
+)
+
+func exitIfErr(cmd *cobra.Command, err error) {
+ if err != nil {
+ errors.Print(cmd.OutOrStderr(), err)
+ exit()
+ }
+}
+
+func buildFromArgs(cmd *cobra.Command, args []string) []*cue.Instance {
+ binst := loadFromArgs(cmd, args)
+ if binst == nil {
+ return nil
+ }
+ return buildInstances(cmd, binst)
+}
+
+func loadFromArgs(cmd *cobra.Command, args []string) []*build.Instance {
+ log.SetOutput(cmd.OutOrStderr())
+ binst := load.Instances(args, nil)
+ if len(binst) == 0 {
+ return nil
+ }
+ return binst
+}
+
+func buildInstances(cmd *cobra.Command, binst []*build.Instance) []*cue.Instance {
+ instances := cue.Build(binst)
+ for _, inst := range instances {
+ // TODO: consider merging errors of multiple files, but ensure
+ // duplicates are removed.
+ exitIfErr(cmd, inst.Err)
+ }
+
+ for _, inst := range instances {
+ // TODO: consider merging errors of multiple files, but ensure
+ // duplicates are removed.
+ exitIfErr(cmd, inst.Value().Validate())
+ }
+ return instances
+}
+
+func buildTools(cmd *cobra.Command, args []string) *cue.Instance {
+ binst := loadFromArgs(cmd, args)
+ if len(binst) == 0 {
+ return nil
+ }
+
+ included := map[string]bool{}
+
+ ti := binst[0].Context().NewInstance(binst[0].Root, nil)
+ for _, inst := range binst {
+ for _, f := range inst.ToolCUEFiles {
+ if file := inst.Abs(f); !included[file] {
+ ti.AddFile(file, nil)
+ included[file] = true
+ }
+ }
+ }
+
+ inst := cue.Merge(buildInstances(cmd, binst)...).Build(ti)
+ exitIfErr(cmd, inst.Err)
+ return inst
+}
diff --git a/cmd/cue/cmd/common_test.go b/cmd/cue/cmd/common_test.go
new file mode 100644
index 0000000..fe71588
--- /dev/null
+++ b/cmd/cue/cmd/common_test.go
@@ -0,0 +1,109 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "regexp"
+ "testing"
+
+ "cuelang.org/go/cue/errors"
+ "github.com/spf13/cobra"
+ "golang.org/x/sync/errgroup"
+)
+
+var _ = errors.Print
+
+var update = flag.Bool("update", false, "update the test files")
+
+func runCommand(t *testing.T, f func(cmd *cobra.Command, args []string) error, name string, args ...string) {
+ t.Helper()
+ log.SetFlags(0)
+
+ cwd, err := os.Getwd()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ const dir = "./testdata"
+ filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ t.Run(path, func(t *testing.T) {
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !info.IsDir() || dir == path {
+ return
+ }
+ testfile := filepath.Join(path, name+".out")
+ bWant, err := ioutil.ReadFile(testfile)
+ if err != nil {
+ // Don't write the file if it doesn't exist, even in *update
+ // mode. We don't want to need to support all commands for all
+ // directories. Touch the file and use *update to create it.
+ return
+ }
+
+ cmd := &cobra.Command{RunE: f}
+ cmd.SetArgs(append(args, "./"+path))
+ rOut, wOut := io.Pipe()
+ cmd.SetOutput(wOut)
+ var bOut []byte
+ g := errgroup.Group{}
+ g.Go(func() error {
+ defer wOut.Close()
+ defer func() {
+ if e := recover(); e != nil {
+ if err, ok := e.(error); ok {
+ errors.Print(wOut, err)
+ } else {
+ fmt.Fprintln(wOut, e)
+ }
+ }
+ }()
+ cmd.Execute()
+ return nil
+ })
+ g.Go(func() error {
+ bOut, err = ioutil.ReadAll(rOut)
+ return err
+ })
+ if err := g.Wait(); err != nil {
+ t.Error(err)
+ }
+ bOut = bytes.Replace(bOut, []byte(cwd), []byte("$CWD"), -1)
+ re := regexp.MustCompile("/.*/cue/")
+ bOut = re.ReplaceAll(bOut, []byte(`$$HOME/cue/`))
+ if *update {
+ ioutil.WriteFile(testfile, bOut, 0644)
+ return
+ }
+ got, want := string(bOut), string(bWant)
+ if got != want {
+ t.Errorf("\n got: %v\nwant: %v", got, want)
+ }
+ })
+ return nil
+ })
+}
+
+func TestLoadError(t *testing.T) {
+ runCommand(t, evalCmd.RunE, "loaderr", "non-existing", ".")
+}
diff --git a/cmd/cue/cmd/custom.go b/cmd/cue/cmd/custom.go
new file mode 100644
index 0000000..fd7ead9
--- /dev/null
+++ b/cmd/cue/cmd/custom.go
@@ -0,0 +1,490 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+// This file contains code or initializing and running custom commands.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "strings"
+ "sync"
+
+ "cuelang.org/go/cue"
+ "github.com/spf13/cobra"
+ "golang.org/x/sync/errgroup"
+)
+
+const (
+ commandSection = "command"
+ taskSection = "task"
+)
+
+func lookupString(obj cue.Value, key string) string {
+ str, err := obj.Lookup(key).String()
+ if err != nil {
+ return ""
+ }
+ return str
+}
+
+func addCustom(parent *cobra.Command, typ, name string, tools *cue.Instance) (*cobra.Command, error) {
+ if tools == nil {
+ return nil, errors.New("no commands defined")
+ }
+
+ // TODO: validate allowing incomplete.
+ o := tools.Lookup(typ, name)
+ if !o.Exists() {
+ return nil, o.Err()
+ }
+
+ usage := lookupString(o, "usage")
+ if usage == "" {
+ usage = name
+ }
+ sub := &cobra.Command{
+ Use: usage,
+ Short: lookupString(o, "short"),
+ Long: lookupString(o, "long"),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ // - parse flags and env vars
+ // - constrain current config with config section
+
+ return doTasks(typ, name, tools)
+ },
+ }
+ parent.AddCommand(sub)
+
+ // TODO: implement var/flag handling.
+ return sub, nil
+}
+
+type taskKey struct {
+ typ string
+ name string
+ task string
+}
+
+func (k taskKey) keyForTask(taskName string) taskKey {
+ k.task = taskName
+ return k
+}
+
+func keyForReference(ref []string) (k taskKey) {
+ // command <command> task <task>
+ if len(ref) >= 4 && ref[2] == taskSection {
+ k.typ = ref[0]
+ k.name = ref[1]
+ k.task = ref[3]
+ }
+ return k
+}
+
+func (k taskKey) taskPath(task string) []string {
+ k.task = task
+ return []string{k.typ, k.name, taskSection, task}
+}
+
+func (k *taskKey) lookupTasks(root *cue.Instance) cue.Value {
+ return root.Lookup(k.typ, k.name, taskSection)
+}
+
+func doTasks(typ, command string, root *cue.Instance) error {
+ if err := executeTasks(typ, command, root); err != nil {
+ return fmt.Errorf("failed to run instance %q: %v", root.Dir, err)
+ }
+ return nil
+}
+
+// executeTasks runs user-defined tasks as part of a user-defined command.
+//
+// All tasks are started at once, but will block until tasks that they depend
+// on will continue.
+func executeTasks(typ, command string, root *cue.Instance) error {
+ spec := taskKey{typ, command, ""}
+ tasks := spec.lookupTasks(root)
+
+ index := map[taskKey]*task{}
+
+ // Create task entries from spec.
+ queue := []*task{}
+ iter, err := tasks.Fields()
+ if err != nil {
+ return err
+ }
+ for i := 0; iter.Next(); i++ {
+ t, err := newTask(i, iter.Label(), iter.Value())
+ if err != nil {
+ return err
+ }
+ queue = append(queue, t)
+ index[spec.keyForTask(iter.Label())] = t
+ }
+
+ // Mark dependencies for unresolved nodes.
+ for _, t := range queue {
+ tasks.Lookup(t.name).Walk(func(v cue.Value) bool {
+ // if v.IsIncomplete() {
+ for _, r := range v.References() {
+ if dep, ok := index[keyForReference(r)]; ok {
+ v := root.Lookup(r...)
+ if v.IsIncomplete() && v.Kind() != cue.StructKind {
+ t.dep[dep] = true
+ }
+ }
+ }
+ // }
+ return true
+ }, nil)
+ }
+
+ if isCyclic(queue) {
+ return errors.New("cyclic dependency in tasks") // TODO: better message.
+ }
+
+ ctx := context.Background()
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var m sync.Mutex
+
+ g, ctx := errgroup.WithContext(ctx)
+ for _, t := range queue {
+ t := t
+ g.Go(func() error {
+ for d := range t.dep {
+ <-d.done
+ }
+ defer close(t.done)
+ m.Lock()
+ obj := tasks.Lookup(t.name)
+ m.Unlock()
+ update, err := t.Run(ctx, obj)
+ if err == nil && update != nil {
+ m.Lock()
+ root, err = root.Fill(update, spec.taskPath(t.name)...)
+
+ if err == nil {
+ tasks = spec.lookupTasks(root)
+ }
+ m.Unlock()
+ }
+ if err != nil {
+ cancel()
+ }
+ return err
+ })
+ }
+ return g.Wait()
+}
+
+func isCyclic(tasks []*task) bool {
+ cc := cycleChecker{
+ visited: make([]bool, len(tasks)),
+ stack: make([]bool, len(tasks)),
+ }
+ for _, t := range tasks {
+ if cc.isCyclic(t) {
+ return true
+ }
+ }
+ return false
+}
+
+type cycleChecker struct {
+ visited, stack []bool
+}
+
+func (cc *cycleChecker) isCyclic(t *task) bool {
+ i := t.index
+ if !cc.visited[i] {
+ cc.visited[i] = true
+ cc.stack[i] = true
+
+ for d := range t.dep {
+ if !cc.visited[d.index] && cc.isCyclic(d) {
+ return true
+ } else if cc.stack[d.index] {
+ return true
+ }
+ }
+ }
+ cc.stack[i] = false
+ return false
+}
+
+type task struct {
+ Runner
+
+ index int
+ name string
+ done chan error
+ dep map[*task]bool
+}
+
+func newTask(index int, name string, v cue.Value) (*task, error) {
+ kind, err := v.Lookup("kind").String()
+ if err != nil {
+ return nil, err
+ }
+ rf, ok := runners[kind]
+ if !ok {
+ return nil, fmt.Errorf("runner of kind %q not found", kind)
+ }
+ runner, err := rf(v)
+ if err != nil {
+ return nil, err
+ }
+ return &task{
+ Runner: runner,
+ index: index,
+ name: name,
+ done: make(chan error),
+ dep: make(map[*task]bool),
+ }, nil
+}
+
+// A Runner defines a command type.
+type Runner interface {
+ // Init is called with the original configuration before any task is run.
+ // As a result, the configuration may be incomplete, but allows some
+ // validation before tasks are kicked off.
+ // Init(v cue.Value)
+
+ // Runner runs given the current value and returns a new value which is to
+ // be unified with the original result.
+ Run(ctx context.Context, v cue.Value) (results interface{}, err error)
+}
+
+// A RunnerFunc creates a Runner.
+type RunnerFunc func(v cue.Value) (Runner, error)
+
+var runners = map[string]RunnerFunc{
+ "print": newPrintCmd,
+ "exec": newExecCmd,
+ "http": newHTTPCmd,
+ "testserver": newTestServerCmd,
+}
+
+type printCmd struct{}
+
+func newPrintCmd(v cue.Value) (Runner, error) {
+ return &printCmd{}, nil
+}
+
+// TODO: get rid of this hack
+var testOut io.Writer
+
+func (c *printCmd) Run(ctx context.Context, v cue.Value) (res interface{}, err error) {
+ str, err := v.Lookup("text").String()
+ if err != nil {
+ return nil, err
+ }
+ if testOut != nil {
+ fmt.Fprintln(testOut, str)
+ } else {
+ fmt.Println(str)
+ }
+ return nil, nil
+}
+
+type execCmd struct{}
+
+func newExecCmd(v cue.Value) (Runner, error) {
+ return &execCmd{}, nil
+}
+
+func (c *execCmd) Run(ctx context.Context, v cue.Value) (res interface{}, err error) {
+ // TODO: set environment variables, if defined.
+ var bin string
+ var args []string
+ switch v := v.Lookup("cmd"); v.Kind() {
+ case cue.StringKind:
+ str, _ := v.String()
+ if str == "" {
+ return cue.Value{}, errors.New("empty command")
+ }
+ list := strings.Fields(str)
+ bin = list[0]
+ for _, s := range list[1:] {
+ args = append(args, s)
+ }
+
+ case cue.ListKind:
+ list, _ := v.List()
+ if !list.Next() {
+ return cue.Value{}, errors.New("empty command list")
+ }
+ bin, err = list.Value().String()
+ if err != nil {
+ return cue.Value{}, err
+ }
+ for list.Next() {
+ str, err := list.Value().String()
+ if err != nil {
+ return cue.Value{}, err
+ }
+ args = append(args, str)
+ }
+ }
+
+ cmd := exec.CommandContext(ctx, bin, args...)
+
+ if v := v.Lookup("stdin"); v.IsValid() {
+ if cmd.Stdin, err = v.Reader(); err != nil {
+ return nil, fmt.Errorf("cue: %v", err)
+ }
+ }
+ captureOut := !v.Lookup("stdout").IsNull()
+ if !captureOut {
+ cmd.Stdout = os.Stdout
+ }
+ captureErr := !v.Lookup("stderr").IsNull()
+ if captureErr {
+ cmd.Stderr = os.Stderr
+ }
+
+ update := map[string]interface{}{}
+ var stdout, stderr []byte
+ if captureOut {
+ stdout, err = cmd.Output()
+ update["stdout"] = string(stdout)
+ } else {
+ err = cmd.Run()
+ }
+ update["success"] = err == nil
+ if err != nil {
+ if exit, ok := err.(*exec.ExitError); ok && captureErr {
+ stderr = exit.Stderr
+ } else {
+ return nil, fmt.Errorf("cue: %v", err)
+ }
+ }
+ if captureErr {
+ update["stderr"] = string(stderr)
+ }
+ return update, nil
+}
+
+type httpCmd struct{}
+
+func newHTTPCmd(v cue.Value) (Runner, error) {
+ return &httpCmd{}, nil
+}
+
+func (c *httpCmd) Run(ctx context.Context, v cue.Value) (res interface{}, err error) {
+ // v.Validate()
+ var header, trailer http.Header
+ method := lookupString(v, "method")
+ u := lookupString(v, "url")
+ var r io.Reader
+ if obj := v.Lookup("request"); v.Exists() {
+ if v := obj.Lookup("body"); v.Exists() {
+ r, err = v.Reader()
+ if err != nil {
+ return nil, err
+ }
+ }
+ if header, err = parseHeaders(obj, "header"); err != nil {
+ return nil, err
+ }
+ if trailer, err = parseHeaders(obj, "trailer"); err != nil {
+ return nil, err
+ }
+ }
+ req, err := http.NewRequest(method, u, r)
+ if err != nil {
+ return nil, err
+ }
+ req.Header = header
+ req.Trailer = trailer
+
+ // TODO:
+ // - retry logic
+ // - TLS certs
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ b, err := ioutil.ReadAll(resp.Body)
+ // parse response body and headers
+ return map[string]interface{}{
+ "response": map[string]interface{}{
+ "body": string(b),
+ "header": resp.Header,
+ "trailer": resp.Trailer,
+ },
+ }, err
+}
+
+func parseHeaders(obj cue.Value, label string) (http.Header, error) {
+ m := obj.Lookup(label)
+ if !m.Exists() {
+ return nil, nil
+ }
+ iter, err := m.Fields()
+ if err != nil {
+ return nil, err
+ }
+ var h http.Header
+ for iter.Next() {
+ str, err := iter.Value().String()
+ if err != nil {
+ return nil, err
+ }
+ h.Add(iter.Label(), str)
+ }
+ return h, nil
+}
+
+func isValid(v cue.Value) bool {
+ return v.Kind() == cue.BottomKind
+}
+
+var testOnce sync.Once
+
+func newTestServerCmd(v cue.Value) (Runner, error) {
+ server := ""
+ testOnce.Do(func() {
+ s := httptest.NewServer(http.HandlerFunc(
+ func(w http.ResponseWriter, req *http.Request) {
+ data, _ := ioutil.ReadAll(req.Body)
+ d := map[string]interface{}{
+ "data": string(data),
+ "when": "now",
+ }
+ enc := json.NewEncoder(w)
+ enc.Encode(d)
+ }))
+ server = s.URL
+ })
+ return testServerCmd(server), nil
+}
+
+type testServerCmd string
+
+func (s testServerCmd) Run(ctx context.Context, v cue.Value) (x interface{}, err error) {
+ return map[string]interface{}{"url": string(s)}, nil
+}
diff --git a/cmd/cue/cmd/custom_test.go b/cmd/cue/cmd/custom_test.go
new file mode 100644
index 0000000..f6e78aa
--- /dev/null
+++ b/cmd/cue/cmd/custom_test.go
@@ -0,0 +1,74 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "strconv"
+ "strings"
+ "testing"
+)
+
+func TestIsCyclic(t *testing.T) {
+ testCases := []struct {
+ // semi-colon-separated list of nodes with comma-separated list
+ // of dependencies.
+ tasks string
+ cycle bool
+ }{{
+ tasks: "",
+ }, {
+ tasks: "0",
+ cycle: true,
+ }, {
+ tasks: "1; 0",
+ cycle: true,
+ }, {
+ tasks: "1; 2; 3; 4;",
+ }, {
+ tasks: "1; 2; ; 4; 5; ",
+ }, {
+ tasks: "1; 2; 3; 4; 0",
+ cycle: true,
+ }}
+ for _, tc := range testCases {
+ t.Run(tc.tasks, func(t *testing.T) {
+ deps := strings.Split(tc.tasks, ";")
+ tasks := make([]*task, len(deps))
+ for i := range tasks {
+ tasks[i] = &task{index: i, dep: map[*task]bool{}}
+ }
+ for i, d := range deps {
+ if d == "" {
+ continue
+ }
+ for _, num := range strings.Split(d, ",") {
+ num = strings.TrimSpace(num)
+ if num == "" {
+ continue
+ }
+ x, err := strconv.Atoi(num)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("%d -> %d", i, x)
+ tasks[i].dep[tasks[x]] = true
+ }
+ }
+ if got := isCyclic(tasks); got != tc.cycle {
+ t.Errorf("got %v; want %v", got, tc.cycle)
+ }
+ })
+ }
+}
diff --git a/cmd/cue/cmd/eval.go b/cmd/cue/cmd/eval.go
new file mode 100644
index 0000000..af90e21
--- /dev/null
+++ b/cmd/cue/cmd/eval.go
@@ -0,0 +1,191 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "strings"
+ "text/tabwriter"
+
+ "cuelang.org/go/cue"
+ "cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/format"
+ "cuelang.org/go/cue/parser"
+ "cuelang.org/go/cue/token"
+ "github.com/spf13/cobra"
+)
+
+// evalCmd represents the eval command
+var evalCmd = &cobra.Command{
+ Use: "eval",
+ Short: "evaluate and print a configuration",
+ Long: `eval evaluates, validates, and prints a configuration.
+
+Printing is skipped if validation fails.
+
+The --expression flag is used to evaluate an expression within the
+configuration file, instead of the entire configuration file itself.
+
+Examples:
+
+ $ cat <<EOF > foo.cue
+ a: [ "a", "b", "c" ]
+ EOF
+
+ $ cue eval foo.cue -e a[0] -e a[2]
+ "a"
+ "c"
+`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ instances := buildFromArgs(cmd, args)
+
+ var exprs []ast.Expr
+ for _, e := range *expressions {
+ expr, err := parser.ParseExpr(token.NewFileSet(), "<expression flag>", e)
+ if err != nil {
+ return err
+ }
+ exprs = append(exprs, expr)
+ }
+
+ tw := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', 0)
+ defer tw.Flush()
+ for _, inst := range instances {
+ // TODO: retrieve the fileset from the instance. Probably using an
+ // internal structure.
+ p := evalPrinter{w: tw, fset: nil}
+ if exprs == nil {
+ p.print(inst.Value())
+ fmt.Fprintln(tw)
+ }
+ for _, e := range exprs {
+ p.print(inst.Eval(e))
+ fmt.Fprintln(tw)
+ }
+ }
+ return nil
+ },
+}
+
+func init() {
+ RootCmd.AddCommand(evalCmd)
+
+ expressions = evalCmd.Flags().StringArrayP("expression", "e", nil, "evaluate this expression only")
+
+}
+
+var (
+ expressions *[]string
+)
+
+type evalPrinter struct {
+ w io.Writer
+ fset *token.FileSet
+ indent int
+ newline bool
+ formfeed bool
+}
+
+type ws byte
+
+const (
+ unindent = -1
+ indent = 1
+ newline ws = '\n'
+ vtab ws = '\v'
+ space ws = ' '
+
+ // maxDiffLen is the maximum different in length for object keys for which
+ // to still align keys and values.
+ maxDiffLen = 5
+)
+
+func (p *evalPrinter) print(args ...interface{}) {
+ for _, a := range args {
+ if d, ok := a.(int); ok {
+ p.indent += d
+ continue
+ }
+ if p.newline {
+ nl := '\n'
+ if p.formfeed {
+ nl = '\f'
+ }
+ p.w.Write([]byte{byte(nl)})
+ fmt.Fprint(p.w, strings.Repeat(" ", int(p.indent)))
+ p.newline = false
+ }
+ switch v := a.(type) {
+ case ws:
+ switch v {
+ case newline:
+ p.newline = true
+ default:
+ p.w.Write([]byte{byte(v)})
+ }
+ case string:
+ fmt.Fprint(p.w, v)
+ case cue.Value:
+ switch v.Kind() {
+ case cue.StructKind:
+ iter, err := v.AllFields()
+ must(err)
+ lastLen := 0
+ p.print("{", indent, newline)
+ for iter.Next() {
+ value := iter.Value()
+ key := iter.Label()
+ newLen := len([]rune(key)) // TODO: measure cluster length.
+ if lastLen > 0 && abs(lastLen, newLen) > maxDiffLen {
+ p.formfeed = true
+ } else {
+ k := value.Kind()
+ p.formfeed = k == cue.StructKind || k == cue.ListKind
+ }
+ p.print(key, ":", vtab, value, newline)
+ p.formfeed = false
+ lastLen = newLen
+ }
+ p.print(unindent, "}")
+ case cue.ListKind:
+ list, err := v.List()
+ must(err)
+ p.print("[", indent, newline)
+ for list.Next() {
+ p.print(list.Value(), newline)
+ }
+ p.print(unindent, "]")
+ default:
+ format.Node(p.w, v.Syntax())
+ }
+ }
+ }
+}
+
+func abs(a, b int) int {
+ a -= b
+ if a < 0 {
+ return -a
+ }
+ return a
+}
+
+func max(a, b int) int {
+ if a > b {
+ return a
+ }
+ return b
+}
diff --git a/cmd/cue/cmd/eval_test.go b/cmd/cue/cmd/eval_test.go
new file mode 100644
index 0000000..67e7c41
--- /dev/null
+++ b/cmd/cue/cmd/eval_test.go
@@ -0,0 +1,21 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import "testing"
+
+func TestEval(t *testing.T) {
+ runCommand(t, evalCmd.RunE, "eval")
+}
diff --git a/cmd/cue/cmd/export.go b/cmd/cue/cmd/export.go
new file mode 100644
index 0000000..d4f8fa0
--- /dev/null
+++ b/cmd/cue/cmd/export.go
@@ -0,0 +1,95 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "encoding/json"
+
+ "github.com/spf13/cobra"
+)
+
+// exportCmd represents the emit command
+var exportCmd = &cobra.Command{
+ Use: "export",
+ Short: "output data in a standard format",
+ Long: `export evaluates the configuration found in the current
+directory and prints the emit value to stdout.
+
+Examples:
+Evaluated and emit
+
+ # a single file
+ cue export config.cue
+
+ # multiple files: these are combined at the top-level. Order doesn't matter.
+ cue export file1.cue foo/file2.cue
+
+ # all files within the "mypkg" package: this includes all files in the
+ # current directory and its ancestor directories that are marked with the
+ # same package.
+ cue export -p cloud
+
+ # the -p flag can be omitted if the directory only contains files for
+ # the "mypkg" package.
+ cue export
+
+Emit value:
+For CUE files, the generated configuration is derived from the top-level
+single expression, the emit value. For example, the file
+
+ // config.cue
+ arg1: 1
+ arg2: "my string"
+
+ {
+ a: arg1
+ b: arg2
+ }
+
+yields the following JSON:
+
+ {
+ "a": 1,
+ "b", "my string"
+ }
+
+In absence of arguments, the current directory is loaded as a package instance.
+A package instance for a directory contains all files in the directory and its
+ancestor directories, up to the module root, belonging to the same package.
+If the package is not explicitly defined by the '-p' flag, it must be uniquely
+defined by the files in the current directory.
+`,
+
+ Run: func(cmd *cobra.Command, args []string) {
+ instances := buildFromArgs(cmd, args)
+ e := json.NewEncoder(cmd.OutOrStdout())
+ e.SetIndent("", " ")
+ e.SetEscapeHTML(*escape)
+
+ root := instances[0].Value()
+ must(e.Encode(root))
+ },
+}
+
+var (
+ escape *bool
+)
+
+func init() {
+ RootCmd.AddCommand(exportCmd)
+
+ exportCmd.Flags().StringP("output", "o", "json", "output format (json only for now)")
+ escape = exportCmd.Flags().BoolP("escape", "e", false, "use HTML escaping")
+}
diff --git a/cmd/cue/cmd/fmt.go b/cmd/cue/cmd/fmt.go
new file mode 100644
index 0000000..7ee1fe7
--- /dev/null
+++ b/cmd/cue/cmd/fmt.go
@@ -0,0 +1,73 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "io/ioutil"
+ "os"
+
+ "cuelang.org/go/cue/format"
+ "cuelang.org/go/cue/load"
+ "github.com/spf13/cobra"
+)
+
+// fmtCmd represents the fmt command
+var fmtCmd = &cobra.Command{
+ Use: "fmt [-s] [packages]",
+ Short: "formats CUE configuration files.",
+ Long: `Fmt formats the given files or the files for the given packages in place
+`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ for _, inst := range load.Instances(args, nil) {
+ all := []string{}
+ all = append(all, inst.CUEFiles...)
+ all = append(all, inst.ToolCUEFiles...)
+ all = append(all, inst.TestCUEFiles...)
+ for _, path := range all {
+ fullpath := inst.Abs(path)
+
+ stat, err := os.Stat(fullpath)
+ if err != nil {
+ return err
+ }
+
+ b, err := ioutil.ReadFile(fullpath)
+ if err != nil {
+ return err
+ }
+
+ opts := []format.Option{}
+ if *fSimplify {
+ opts = append(opts, format.Simplify())
+ }
+
+ b, err = format.Source(b, opts...)
+ if err != nil {
+ return err
+ }
+
+ err = ioutil.WriteFile(fullpath, b, stat.Mode())
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ },
+}
+
+func init() {
+ RootCmd.AddCommand(fmtCmd)
+}
diff --git a/cmd/cue/cmd/import.go b/cmd/cue/cmd/import.go
new file mode 100644
index 0000000..5ead4fd
--- /dev/null
+++ b/cmd/cue/cmd/import.go
@@ -0,0 +1,761 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "unicode"
+
+ "cuelang.org/go/cue"
+ "cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/encoding"
+ "cuelang.org/go/cue/format"
+ "cuelang.org/go/cue/load"
+ "cuelang.org/go/cue/parser"
+ "cuelang.org/go/cue/token"
+ "cuelang.org/go/internal"
+ "cuelang.org/go/internal/third_party/yaml"
+ "github.com/spf13/cobra"
+ "golang.org/x/sync/errgroup"
+)
+
+// importCmd represents the import command
+var importCmd = &cobra.Command{
+ Use: "import",
+ Short: "convert other data formats to CUE files",
+ Long: `import converts other data formats, like JSON and YAML to CUE files
+
+The following file formats are currently supported:
+
+ Format Extensions
+ JSON .json .jsonl .ndjson
+ YAML .yaml .yml
+
+Files can either be specified explicitly, or inferred from the specified
+packages. In either case, the file extension is replaced with .cue. It will
+fail if the file already exists by default. The -f flag overrides this.
+
+Examples:
+
+ # Convert individual files:
+ $ cue import foo.json bar.json # create foo.yaml and bar.yaml
+
+ # Convert all json files in the indicated directories:
+ $ cue import ./... -type=json
+
+
+The -path flag
+
+By default the parsed files are included as emit values. This default can be
+overridden by specifying a sequence of labels as you would in a CUE file.
+An identifier or string label are interpreted as usual. A label expression is
+evaluated within the context of the imported file. label expressions may also
+refer to builtin packages, which will be implicitly imported.
+
+
+Handling multiple documents or streams
+
+To handle Multi-document files, such as concatenated JSON objects or
+YAML files with document separators (---) the user must specify either the
+-path, -list, or -files flag. The -path flag assign each element to a path
+(identical paths are treated as usual); -list concatenates the entries, and
+-files causes each entry to be written to a different file. The -files flag
+may only be used if files are explicitly imported. The -list flag may be
+used in combination with the -path flag, concatenating each entry to the
+mapped location.
+
+
+Examples:
+
+ $ cat <<EOF > foo.yaml
+ kind: Service
+ name: booster
+ EOF
+
+ # include the parsed file as an emit value:
+ $ cue import foo.yaml
+ $ cat foo.cue
+ {
+ kind: Service
+ name: booster
+ }
+
+ # include the parsed file at the root of the CUE file:
+ $ cue import -f -l "" foo.yaml
+ $ cat foo.cue
+ kind: Service
+ name: booster
+
+ # include the import config at the mystuff path
+ $ cue import -f -l mystuff foo.yaml
+ $ cat foo.cue
+ myStuff: {
+ kind: Service
+ name: booster
+ }
+
+ # append another object to the input file
+ $ cat <<EOF >> foo.yaml
+ ---
+ kind: Deployment
+ name: booster
+ replicas: 1
+
+ # base the path values on th input
+ $ cue import -f -l '"\(strings.ToLower(kind))" "\(x.name)"' foo.yaml
+ $ cat foo.cue
+ service booster: {
+ kind: "Service"
+ name: "booster"
+ }
+
+ deployment booster: {
+ kind: "Deployment"
+ name: "booster
+ replicas: 1
+ }
+
+ # base the path values on th input
+ $ cue import -f -list -foo.yaml
+ $ cat foo.cue
+ [{
+ kind: "Service"
+ name: "booster"
+ }, {
+ kind: "Deployment"
+ name: "booster
+ replicas: 1
+ }]
+
+ # base the path values on th input
+ $ cue import -f -list -l '"\(strings.ToLower(kind))"' foo.yaml
+ $ cat foo.cue
+ service: [{
+ kind: "Service"
+ name: "booster"
+ }
+ deployment: [{
+ kind: "Deployment"
+ name: "booster
+ replicas: 1
+ }]
+
+
+Embedded data files
+
+The --recursive or -R flag enables the parsing of fields that are string
+representations of data formats themselves. A field that can be parsed is
+replaced with a call encoding the data from a structured form that is placed
+in a sibling field.
+
+It is also possible to recursively hoist data formats:
+
+Example:
+ $ cat <<EOF > example.json
+ "a": {
+ "data": '{ "foo": 1, "bar": 2 }',
+ }
+ EOF
+
+ $ cue import -R example.json
+ $ cat example.cue
+ import "encode/json"
+
+ a: {
+ data: json.Encode(_data),
+ _data: {
+ foo: 1
+ bar: 2
+ }
+ }
+`,
+ RunE: runImport,
+}
+
+func init() {
+ RootCmd.AddCommand(importCmd)
+
+ out = importCmd.Flags().StringP("out", "o", "", "alternative output or - for stdout")
+ name = importCmd.Flags().StringP("name", "n", "", "glob filter for file names")
+ typ = importCmd.Flags().String("type", "", "only apply to files of this type")
+ force = importCmd.Flags().BoolP("force", "f", false, "force overwriting existing files")
+ dryrun = importCmd.Flags().Bool("dryrun", false, "force overwriting existing files")
+
+ node = importCmd.Flags().StringP("path", "l", "", "path to include root")
+ list = importCmd.Flags().Bool("list", false, "concatenate multiple objects into a list")
+ files = importCmd.Flags().Bool("files", false, "split multiple entries into different files")
+ parseStrings = importCmd.Flags().BoolP("recursive", "R", false, "recursively parse string values")
+
+ importCmd.Flags().String("fix", "", "apply given fix")
+}
+
+var (
+ force *bool
+ name *string
+ typ *string
+ node *string
+ out *string
+ dryrun *bool
+ list *bool
+ files *bool
+ parseStrings *bool
+)
+
+type importFunc func(path string, r io.Reader) ([]ast.Expr, error)
+
+type encodingInfo struct {
+ fn importFunc
+ typ string
+}
+
+var (
+ jsonEnc = &encodingInfo{handleJSON, "json"}
+ yamlEnc = &encodingInfo{handleYAML, "yaml"}
+)
+
+func getExtInfo(ext string) *encodingInfo {
+ enc := encoding.MapExtension(ext)
+ if enc == nil {
+ return nil
+ }
+ switch enc.Name() {
+ case "json":
+ return jsonEnc
+ case "yaml":
+ return yamlEnc
+ }
+ return nil
+}
+
+func runImport(cmd *cobra.Command, args []string) error {
+ log.SetOutput(cmd.OutOrStderr())
+
+ var group errgroup.Group
+
+ group.Go(func() error {
+ if len(args) > 0 && len(filepath.Ext(args[0])) > len(".") {
+ for _, a := range args {
+ group.Go(func() error { return handleFile(cmd, *fPackage, a) })
+ }
+ return nil
+ }
+
+ done := map[string]bool{}
+
+ inst := load.Instances(args, &load.Config{DataFiles: true})
+ for _, pkg := range inst {
+ pkgName := *fPackage
+ if pkgName == "" {
+ pkgName = pkg.PkgName
+ }
+ if pkgName == "" && len(inst) > 1 {
+ return fmt.Errorf("must specify package name with the -p flag")
+ }
+ dir := pkg.Dir
+ if err := pkg.Err; err != nil {
+ return err
+ }
+ if done[dir] {
+ continue
+ }
+ done[dir] = true
+
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ for _, file := range files {
+ ext := filepath.Ext(file.Name())
+ if enc := getExtInfo(ext); enc == nil || (*typ != "" && *typ != enc.typ) {
+ continue
+ }
+ path := filepath.Join(dir, file.Name())
+ group.Go(func() error { return handleFile(cmd, pkgName, path) })
+ }
+ }
+ return nil
+ })
+
+ err := group.Wait()
+ if err != nil {
+ return fmt.Errorf("Import failed: %v", err)
+ }
+ return nil
+}
+
+func handleFile(cmd *cobra.Command, pkg, filename string) error {
+ re, err := regexp.Compile(*name)
+ if err != nil {
+ return err
+ }
+ if !re.MatchString(filepath.Base(filename)) {
+ return nil
+ }
+ f, err := os.Open(filename)
+ if err != nil {
+ return fmt.Errorf("error opening file: %v", err)
+ }
+ defer f.Close()
+
+ ext := filepath.Ext(filename)
+ handler := getExtInfo(ext)
+
+ if handler == nil {
+ return fmt.Errorf("unsupported extension %q", ext)
+ }
+ objs, err := handler.fn(filename, f)
+ if err != nil {
+ return err
+ }
+
+ if *files {
+ for i, f := range objs {
+ err := combineExpressions(cmd, pkg, newName(filename, i), f)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ } else if len(objs) > 1 {
+ if !*list && *node == "" && !*files {
+ return fmt.Errorf("list, flag, or files flag needed to handle multiple objects in file %q", filename)
+ }
+ }
+ return combineExpressions(cmd, pkg, newName(filename, 0), objs...)
+}
+
+func combineExpressions(cmd *cobra.Command, pkg, cueFile string, objs ...ast.Expr) error {
+ if *out != "" {
+ cueFile = *out
+ }
+ if cueFile != "-" {
+ switch _, err := os.Stat(cueFile); {
+ case os.IsNotExist(err):
+ case err == nil:
+ if !*force {
+ log.Printf("skipping file %q: already exists", cueFile)
+ return nil
+ }
+ default:
+ return fmt.Errorf("error creating file: %v", err)
+ }
+ }
+
+ f := &ast.File{}
+ if pkg != "" {
+ f.Name = ast.NewIdent(pkg)
+ }
+
+ h := hoister{
+ fields: map[string]bool{},
+ altNames: map[string]*ast.Ident{},
+ }
+
+ index := newIndex()
+ for _, expr := range objs {
+ if *parseStrings {
+ h.hoist(expr)
+ }
+
+ // Compute a path different from root.
+ var pathElems []ast.Label
+
+ switch {
+ case *node != "":
+ inst, err := cue.FromExpr(nil, expr)
+ if err != nil {
+ return err
+ }
+
+ labels, err := parsePath(*node)
+ if err != nil {
+ return err
+ }
+ for _, l := range labels {
+ switch x := l.(type) {
+ case *ast.Interpolation:
+ v := inst.Eval(x)
+ if v.Kind() == cue.BottomKind {
+ return v.Err()
+ }
+ pathElems = append(pathElems, v.Syntax().(ast.Label))
+
+ case *ast.Ident, *ast.BasicLit:
+ pathElems = append(pathElems, x)
+
+ case *ast.ExprLabel:
+ v := inst.Eval(x.Label)
+ switch v.Kind() {
+ case cue.StringKind:
+ pathElems = append(pathElems, v.Syntax().(ast.Label))
+ case cue.NullKind,
+ cue.NumberKind,
+ cue.BoolKind:
+ pathElems = append(pathElems, newString(fmt.Sprint(v)))
+ case cue.BottomKind:
+ return v.Err()
+ default:
+ return fmt.Errorf("expression %q in path is not a label", internal.DebugStr(v.Syntax()))
+ }
+
+ case *ast.TemplateLabel:
+ return fmt.Errorf("template labels not supported in path flag")
+ }
+ }
+ }
+
+ if *list {
+ idx := index
+ for _, e := range pathElems {
+ idx = idx.label(e)
+ }
+ if idx.field.Value == nil {
+ idx.field.Value = &ast.ListLit{
+ Lbrack: token.Pos(token.NoSpace),
+ Rbrack: token.Pos(token.NoSpace),
+ }
+ }
+ list := idx.field.Value.(*ast.ListLit)
+ list.Elts = append(list.Elts, expr)
+ } else if len(pathElems) == 0 {
+ obj, ok := expr.(*ast.StructLit)
+ if !ok {
+ return fmt.Errorf("cannot map non-struct to object root")
+ }
+ f.Decls = append(f.Decls, obj.Elts...)
+ } else {
+ field := &ast.Field{Label: pathElems[0]}
+ f.Decls = append(f.Decls, field)
+ for _, e := range pathElems[1:] {
+ newField := &ast.Field{Label: e}
+ newVal := &ast.StructLit{Elts: []ast.Decl{newField}}
+ field.Value = newVal
+ field = newField
+ }
+ field.Value = expr
+ }
+ }
+
+ if len(h.altNames) > 0 {
+ imports := &ast.ImportDecl{}
+
+ for _, enc := range encoding.All() {
+ if ident, ok := h.altNames[enc.Name()]; ok {
+ short := enc.Name()
+ name := h.uniqueName(short, "")
+ ident.Name = name
+ if name == short {
+ ident = nil
+ }
+
+ path := fmt.Sprintf(`"encoding/%s"`, short)
+ imports.Specs = append(imports.Specs, &ast.ImportSpec{
+ Name: ident,
+ Path: &ast.BasicLit{Kind: token.STRING, Value: path},
+ })
+ }
+ }
+ f.Decls = append([]ast.Decl{imports}, f.Decls...)
+ }
+
+ if *list {
+ switch x := index.field.Value.(type) {
+ case *ast.StructLit:
+ f.Decls = append(f.Decls, x.Elts...)
+ case *ast.ListLit:
+ f.Decls = append(f.Decls, &ast.EmitDecl{Expr: x})
+ default:
+ panic("unreachable")
+ }
+ }
+
+ var buf bytes.Buffer
+ err := format.Node(&buf, f, format.Simplify())
+ if err != nil {
+ return fmt.Errorf("error formatting file: %v", err)
+ }
+
+ if cueFile == "-" {
+ _, err := io.Copy(cmd.OutOrStdout(), &buf)
+ return err
+ }
+ return ioutil.WriteFile(cueFile, buf.Bytes(), 0644)
+}
+
+type listIndex struct {
+ index map[string]*listIndex
+ file *ast.File // top-level only
+ field *ast.Field
+}
+
+func newIndex() *listIndex {
+ return &listIndex{
+ index: map[string]*listIndex{},
+ field: &ast.Field{},
+ }
+}
+
+func newString(s string) *ast.BasicLit {
+ return &ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(s)}
+}
+
+func (x *listIndex) label(label ast.Label) *listIndex {
+ key := internal.DebugStr(label)
+ idx := x.index[key]
+ if idx == nil {
+ if x.field.Value == nil {
+ x.field.Value = &ast.StructLit{}
+ }
+ obj := x.field.Value.(*ast.StructLit)
+ newField := &ast.Field{Label: label}
+ obj.Elts = append(obj.Elts, newField)
+ idx = &listIndex{
+ index: map[string]*listIndex{},
+ field: newField,
+ }
+ x.index[key] = idx
+ }
+ return idx
+}
+
+func parsePath(exprs string) (p []ast.Label, err error) {
+ fset := token.NewFileSet()
+ f, err := parser.ParseFile(fset, "<path flag>", exprs+": _")
+ if err != nil {
+ return nil, fmt.Errorf("parser error in path %q: %v", exprs, err)
+ }
+
+ if len(f.Decls) != 1 {
+ return nil, errors.New("path flag must be a space-separated sequence of labels")
+ }
+
+ for d := f.Decls[0]; ; {
+ field, ok := d.(*ast.Field)
+ if !ok {
+ // This should never happen
+ return nil, errors.New("%q not a sequence of labels")
+ }
+
+ p = append(p, field.Label)
+
+ v, ok := field.Value.(*ast.StructLit)
+ if !ok {
+ break
+ }
+
+ if len(v.Elts) != 1 {
+ // This should never happen
+ return nil, errors.New("path value may not contain a struct")
+ }
+
+ d = v.Elts[0]
+ }
+ return p, nil
+}
+
+func newName(filename string, i int) string {
+ ext := filepath.Ext(filename)
+ filename = filename[:len(filename)-len(ext)]
+ if i > 0 {
+ filename += fmt.Sprintf("-%d", i)
+ }
+ filename += ".cue"
+ return filename
+}
+
+var fset = token.NewFileSet()
+
+func handleJSON(path string, r io.Reader) (objects []ast.Expr, err error) {
+ d := json.NewDecoder(r)
+
+ for {
+ var raw json.RawMessage
+ err := d.Decode(&raw)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, fmt.Errorf("could not parse JSON: %v", err)
+ }
+ expr, err := parser.ParseExpr(fset, path, []byte(raw))
+ if err != nil {
+ return nil, fmt.Errorf("invalid input: %v %q", err, raw)
+ }
+ objects = append(objects, expr)
+ }
+ return objects, nil
+}
+
+func handleYAML(path string, r io.Reader) (objects []ast.Expr, err error) {
+ d, err := yaml.NewDecoder(fset, path, r)
+ if err != nil {
+ return nil, err
+ }
+ for i := 0; ; i++ {
+ expr, err := d.Decode()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ objects = append(objects, expr)
+ }
+ return objects, nil
+}
+
+type hoister struct {
+ fields map[string]bool
+ altNames map[string]*ast.Ident
+}
+
+func (h *hoister) hoist(expr ast.Expr) {
+ ast.Walk(expr, nil, func(n ast.Node) {
+ name := ""
+ switch x := n.(type) {
+ case *ast.Field:
+ name, _ = ast.LabelName(x.Label)
+ case *ast.Alias:
+ name = x.Ident.Name
+ }
+ if name != "" {
+ h.fields[name] = true
+ }
+ })
+
+ ast.Walk(expr, func(n ast.Node) bool {
+ switch n.(type) {
+ case *ast.ComprehensionDecl:
+ return false
+ }
+ return true
+
+ }, func(n ast.Node) {
+ obj, ok := n.(*ast.StructLit)
+ if !ok {
+ return
+ }
+ for i := 0; i < len(obj.Elts); i++ {
+ f, ok := obj.Elts[i].(*ast.Field)
+ if !ok {
+ continue
+ }
+
+ name, ident := ast.LabelName(f.Label)
+ if name == "" || !ident {
+ continue
+ }
+
+ lit, ok := f.Value.(*ast.BasicLit)
+ if !ok || lit.Kind != token.STRING {
+ continue
+ }
+
+ str, err := cue.Unquote(lit.Value)
+ if err != nil {
+ continue
+ }
+
+ expr, enc := tryParse(str)
+ if expr == nil {
+ continue
+ }
+
+ if h.altNames[enc.typ] == nil {
+ h.altNames[enc.typ] = &ast.Ident{Name: "cue"} // set name later
+ }
+
+ // found a replacable string
+ dataField := h.uniqueName(name, "cue")
+
+ f.Value = &ast.CallExpr{
+ Fun: &ast.SelectorExpr{
+ X: h.altNames[enc.typ],
+ Sel: ast.NewIdent("Marshal"),
+ },
+ Args: []ast.Expr{
+ ast.NewIdent(dataField),
+ },
+ }
+
+ obj.Elts = append(obj.Elts, nil)
+ copy(obj.Elts[i+1:], obj.Elts[i:])
+
+ obj.Elts[i+1] = &ast.Alias{
+ Ident: ast.NewIdent(dataField),
+ Expr: expr,
+ }
+
+ h.hoist(expr)
+ }
+ })
+}
+
+func tryParse(str string) (s ast.Expr, format *encodingInfo) {
+ b := []byte(str)
+ fset := token.NewFileSet()
+ if json.Valid(b) {
+ expr, err := parser.ParseExpr(fset, "", b)
+ if err != nil {
+ // TODO: report error
+ return nil, nil
+ }
+ switch expr.(type) {
+ case *ast.StructLit, *ast.ListLit:
+ default:
+ return nil, nil
+ }
+ return expr, jsonEnc
+ }
+
+ if expr, err := yaml.Unmarshal(fset, "", b); err == nil {
+ switch expr.(type) {
+ case *ast.StructLit, *ast.ListLit:
+ default:
+ return nil, nil
+ }
+ return expr, yamlEnc
+ }
+
+ return nil, nil
+}
+
+func (h *hoister) uniqueName(base, typ string) string {
+ base = strings.Map(func(r rune) rune {
+ if unicode.In(r, unicode.L, unicode.N) {
+ return r
+ }
+ return '_'
+ }, base)
+
+ name := base
+ for {
+ if !h.fields[name] {
+ return name
+ }
+ name = typ + "_" + base
+ typ += "x"
+ }
+}
diff --git a/cmd/cue/cmd/import_test.go b/cmd/cue/cmd/import_test.go
new file mode 100644
index 0000000..caf74b7
--- /dev/null
+++ b/cmd/cue/cmd/import_test.go
@@ -0,0 +1,40 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import "testing"
+
+func TestImport(t *testing.T) {
+ importCmd.ParseFlags([]string{
+ "-o", "-", "-f", "--files",
+ })
+ runCommand(t, importCmd.RunE, "import_files")
+
+ *files = false
+ importCmd.ParseFlags([]string{
+ "-f", "-l", `"\(strings.ToLower(kind))" "\(name)"`,
+ })
+ runCommand(t, importCmd.RunE, "import_path")
+
+ importCmd.ParseFlags([]string{
+ "-f", "-l", `"\(strings.ToLower(kind))"`, "--list",
+ })
+ runCommand(t, importCmd.RunE, "import_list")
+
+ importCmd.ParseFlags([]string{
+ "-f", "-l", `"\(strings.ToLower(kind))" "\(name)"`, "--recursive",
+ })
+ runCommand(t, importCmd.RunE, "import_hoiststr")
+}
diff --git a/cmd/cue/cmd/root.go b/cmd/cue/cmd/root.go
new file mode 100644
index 0000000..2f9dbdc
--- /dev/null
+++ b/cmd/cue/cmd/root.go
@@ -0,0 +1,185 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "fmt"
+ logger "log"
+ "os"
+
+ homedir "github.com/mitchellh/go-homedir"
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+)
+
+// TODO: commands
+// fix: rewrite/refactor configuration files
+// -i interactive: open diff and ask to update
+// serve: like cmd, but for servers
+// extract: extract cue from other languages, like proto and go.
+// gen: generate files for other languages
+// generate like go generate (also convert cue to go doc)
+// test load and fully evaluate test files.
+//
+// TODO: documentation of concepts
+// tasks the key element for cmd, serve, and fix
+
+var log = logger.New(os.Stderr, "", logger.Lshortfile)
+
+var cfgFile string
+
+// RootCmd represents the base command when called without any subcommands
+var RootCmd = &cobra.Command{
+ Use: "cue",
+ Short: "cue emits configuration files to user-defined commands.",
+ Long: `cue evaluates CUE files, an extension of JSON, and sends them
+to user-defined commands for processing.
+
+Commands are defined in CUE as follows:
+
+ command deploy: {
+ cmd: "kubectl"
+ args: [ "-f", "deploy" ]
+ in: json.Encode($) // encode the emitted configuration.
+ }
+
+cue can also combine the results of http or grpc request with the input
+configuration for further processing. For more information on defining commands
+run 'gcfg help commands' or go to cuelang.org/pkg/cmd.
+
+For more information on writing CUE configuration files see cuelang.org.`,
+ // Uncomment the following line if your bare application
+ // has an action associated with it:
+ // Run: func(cmd *cobra.Command, args []string) { },
+
+ SilenceUsage: true,
+}
+
+// Execute adds all child commands to the root command sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+ log.SetFlags(0)
+ // Three categories of commands:
+ // - normal
+ // - user defined
+ // - help
+ // For the latter two, we need to use the default loading.
+ defer func() {
+ switch err := recover(); err {
+ case nil:
+ case panicSentinel:
+ log.Fatal(err)
+ os.Exit(1)
+ default:
+ panic(err)
+ }
+ // We use panic to escape, instead of os.Exit
+ }()
+ if args := os.Args[1:]; len(args) >= 1 && args[0] != "help" {
+ // TODO: for now we only allow one instance. Eventually, we can allow
+ // more if they all belong to the same package and we merge them
+ // before computing commands.
+ if cmd, _, err := RootCmd.Find(args); err != nil || cmd == nil {
+ tools := buildTools(RootCmd, args[1:])
+ addCustom(RootCmd, commandSection, args[0], tools)
+ }
+
+ type subSpec struct {
+ name string
+ cmd *cobra.Command
+ }
+ sub := map[string]subSpec{
+ "cmd": {commandSection, cmdCmd},
+ // "serve": {"server", nil},
+ // "fix": {"fix", nil},
+ }
+ if sub, ok := sub[args[0]]; ok && len(args) >= 2 {
+ args = args[1:]
+ if len(args) == 0 {
+ tools := buildTools(RootCmd, args)
+ // list available commands
+ commands := tools.Lookup(sub.name)
+ i, err := commands.Fields()
+ must(err)
+ for i.Next() {
+ _, _ = addCustom(sub.cmd, sub.name, i.Label(), tools)
+ }
+ return // TODO: will this trigger the help?
+ }
+ tools := buildTools(RootCmd, args[1:])
+ _, err := addCustom(sub.cmd, sub.name, args[0], tools)
+ if err != nil {
+ log.Printf("%s %q is not defined", sub.name, args[0])
+ exit()
+ }
+ }
+ }
+ if err := RootCmd.Execute(); err != nil {
+ // log.Fatal(err)
+ os.Exit(1)
+ }
+}
+
+var panicSentinel = "terminating because of errors"
+
+func must(err error) {
+ if err != nil {
+ log.Print(err)
+ exit()
+ }
+}
+
+func exit() { panic(panicSentinel) }
+
+func init() {
+ cobra.OnInitialize(initConfig)
+
+ RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cue)")
+ RootCmd.PersistentFlags().Bool("root", false, "load a CUE package from its root")
+}
+
+var (
+ fDebug = RootCmd.PersistentFlags().Bool("debug", false, "give detailed error info")
+ fTrace = RootCmd.PersistentFlags().Bool("trace", false, "trace computation")
+ fDryrun = RootCmd.PersistentFlags().BoolP("dryrun", "n", false, "only run simulation")
+ fPackage = RootCmd.PersistentFlags().StringP("package", "p", "", "CUE package to evaluate")
+ fSimplify = RootCmd.PersistentFlags().BoolP("simplify", "s", false, "simplify output")
+)
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+ if cfgFile != "" {
+ // Use config file from the flag.
+ viper.SetConfigFile(cfgFile)
+ } else {
+ // Find home directory.
+ home, err := homedir.Dir()
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+
+ // Search config in home directory with name ".cue" (without extension).
+ viper.AddConfigPath(home)
+ viper.SetConfigName(".cue")
+ }
+
+ viper.AutomaticEnv() // read in environment variables that match
+
+ // If a config file is found, read it in.
+ if err := viper.ReadInConfig(); err == nil {
+ fmt.Println("Using config file:", viper.ConfigFileUsed())
+ }
+}
diff --git a/cmd/cue/cmd/testdata/hello/cmd_echo.out b/cmd/cue/cmd/testdata/hello/cmd_echo.out
new file mode 100644
index 0000000..ea2fd5c
--- /dev/null
+++ b/cmd/cue/cmd/testdata/hello/cmd_echo.out
@@ -0,0 +1,2 @@
+Hello World!
+
diff --git a/cmd/cue/cmd/testdata/hello/data.cue b/cmd/cue/cmd/testdata/hello/data.cue
new file mode 100644
index 0000000..e226892
--- /dev/null
+++ b/cmd/cue/cmd/testdata/hello/data.cue
@@ -0,0 +1,3 @@
+package hello
+
+who: "World"
\ No newline at end of file
diff --git a/cmd/cue/cmd/testdata/hello/eval.out b/cmd/cue/cmd/testdata/hello/eval.out
new file mode 100644
index 0000000..8af3d0f
--- /dev/null
+++ b/cmd/cue/cmd/testdata/hello/eval.out
@@ -0,0 +1,4 @@
+{
+ who: "World"
+ message: "Hello World!"
+}
diff --git a/cmd/cue/cmd/testdata/hello/hello.cue b/cmd/cue/cmd/testdata/hello/hello.cue
new file mode 100644
index 0000000..f8f8654
--- /dev/null
+++ b/cmd/cue/cmd/testdata/hello/hello.cue
@@ -0,0 +1,3 @@
+package hello
+
+message: "Hello \(who)!" // who declared in data.cue
diff --git a/cmd/cue/cmd/testdata/hello/hello_tool.cue b/cmd/cue/cmd/testdata/hello/hello_tool.cue
new file mode 100644
index 0000000..b3a6f2d
--- /dev/null
+++ b/cmd/cue/cmd/testdata/hello/hello_tool.cue
@@ -0,0 +1,14 @@
+package hello
+
+command echo: {
+ task echo: {
+ kind: "exec"
+ cmd: "echo \(message)"
+ stdout: string
+ }
+
+ task display: {
+ kind: "print"
+ text: task.echo.stdout
+ }
+}
\ No newline at end of file
diff --git a/cmd/cue/cmd/testdata/import/import_files.out b/cmd/cue/cmd/testdata/import/import_files.out
new file mode 100644
index 0000000..df6d24d
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/import_files.out
@@ -0,0 +1,8 @@
+kind: "Service"
+name: "booster"
+kind: "Deployment"
+name: "booster"
+replicas: 1
+kind: "Service"
+name: "supplement"
+json: "[1, 2]"
diff --git a/cmd/cue/cmd/testdata/import/import_hoiststr.out b/cmd/cue/cmd/testdata/import/import_hoiststr.out
new file mode 100644
index 0000000..d01566f
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/import_hoiststr.out
@@ -0,0 +1,19 @@
+import _json "encoding/json"
+
+service: {
+ booster: [{
+ kind: "Service"
+ name: "booster"
+ }]
+ supplement: [{
+ kind: "Service"
+ name: "supplement"
+ json: _json.Marshal(cue_json)
+ cue_json = [1, 2]
+ }]
+}
+deployment booster: [{
+ kind: "Deployment"
+ name: "booster"
+ replicas: 1
+}]
diff --git a/cmd/cue/cmd/testdata/import/import_list.out b/cmd/cue/cmd/testdata/import/import_list.out
new file mode 100644
index 0000000..e071c0d
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/import_list.out
@@ -0,0 +1,13 @@
+service: [{
+ kind: "Service"
+ name: "booster"
+}, {
+ kind: "Service"
+ name: "supplement"
+ json: "[1, 2]"
+}]
+deployment: [{
+ kind: "Deployment"
+ name: "booster"
+ replicas: 1
+}]
diff --git a/cmd/cue/cmd/testdata/import/import_path.out b/cmd/cue/cmd/testdata/import/import_path.out
new file mode 100644
index 0000000..e59d747
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/import_path.out
@@ -0,0 +1,15 @@
+
+service booster: {
+ kind: "Service"
+ name: "booster"
+}
+deployment booster: {
+ kind: "Deployment"
+ name: "booster"
+ replicas: 1
+}
+service supplement: {
+ kind: "Service"
+ name: "supplement"
+ json: "[1, 2]"
+}
diff --git a/cmd/cue/cmd/testdata/import/services.cue b/cmd/cue/cmd/testdata/import/services.cue
new file mode 100644
index 0000000..e071c0d
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/services.cue
@@ -0,0 +1,13 @@
+service: [{
+ kind: "Service"
+ name: "booster"
+}, {
+ kind: "Service"
+ name: "supplement"
+ json: "[1, 2]"
+}]
+deployment: [{
+ kind: "Deployment"
+ name: "booster"
+ replicas: 1
+}]
diff --git a/cmd/cue/cmd/testdata/import/services.jsonl b/cmd/cue/cmd/testdata/import/services.jsonl
new file mode 100644
index 0000000..5527811
--- /dev/null
+++ b/cmd/cue/cmd/testdata/import/services.jsonl
@@ -0,0 +1,14 @@
+{
+ "kind": "Service",
+ "name": "booster"
+}
+{
+ "kind": "Deployment",
+ "name": "booster",
+ "replicas": 1
+}
+{
+ "kind": "Service",
+ "name": "supplement",
+ "json": "[1, 2]"
+}
\ No newline at end of file
diff --git a/cmd/cue/cmd/testdata/loaderr/loaderr.out b/cmd/cue/cmd/testdata/loaderr/loaderr.out
new file mode 100644
index 0000000..2921045
--- /dev/null
+++ b/cmd/cue/cmd/testdata/loaderr/loaderr.out
@@ -0,0 +1,3 @@
+cannot find package "non-existing":
+
+terminating because of errors
diff --git a/cmd/cue/cmd/testdata/tasks/cmd_baddisplay.out b/cmd/cue/cmd/testdata/tasks/cmd_baddisplay.out
new file mode 100644
index 0000000..e2dc4ef
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/cmd_baddisplay.out
@@ -0,0 +1,3 @@
+not of right kind (number vs string):
+ $CWD/testdata/tasks/task_tool.cue:23:9
+
diff --git a/cmd/cue/cmd/testdata/tasks/cmd_http.out b/cmd/cue/cmd/testdata/tasks/cmd_http.out
new file mode 100644
index 0000000..058b2c2
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/cmd_http.out
@@ -0,0 +1,2 @@
+{"data":"I'll be back!","when":"now"}
+
diff --git a/cmd/cue/cmd/testdata/tasks/cmd_run.out b/cmd/cue/cmd/testdata/tasks/cmd_run.out
new file mode 100644
index 0000000..47ee769
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/cmd_run.out
@@ -0,0 +1,2 @@
+Hello world!
+
diff --git a/cmd/cue/cmd/testdata/tasks/cmd_run_list.out b/cmd/cue/cmd/testdata/tasks/cmd_run_list.out
new file mode 100644
index 0000000..47ee769
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/cmd_run_list.out
@@ -0,0 +1,2 @@
+Hello world!
+
diff --git a/cmd/cue/cmd/testdata/tasks/eval.out b/cmd/cue/cmd/testdata/tasks/eval.out
new file mode 100644
index 0000000..8c85ba9
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/eval.out
@@ -0,0 +1,3 @@
+{
+ message: "Hello world!"
+}
diff --git a/cmd/cue/cmd/testdata/tasks/task.cue b/cmd/cue/cmd/testdata/tasks/task.cue
new file mode 100644
index 0000000..cb91d4f
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/task.cue
@@ -0,0 +1,3 @@
+package home
+
+message: "Hello world!"
diff --git a/cmd/cue/cmd/testdata/tasks/task_tool.cue b/cmd/cue/cmd/testdata/tasks/task_tool.cue
new file mode 100644
index 0000000..ce37bf1
--- /dev/null
+++ b/cmd/cue/cmd/testdata/tasks/task_tool.cue
@@ -0,0 +1,56 @@
+package home
+
+command run: runBase & {
+ task echo cmd: "echo \(message)"
+}
+
+command run_list: runBase & {
+ task echo cmd: ["echo", message]
+}
+
+// TODO: capture stdout and stderr for tests.
+command runRedirect: {
+ task echo: {
+ kind: "exec"
+ cmd: "echo \(message)"
+ stdout: null // should be automatic
+ }
+}
+
+command baddisplay: {
+ task display: {
+ kind: "print"
+ text: 42
+ }
+}
+
+command http: {
+ task testserver: {
+ kind: "testserver"
+ url: string
+ }
+ task http: {
+ kind: "http"
+ method: "POST"
+ url: task.testserver.url
+
+ request body: "I'll be back!"
+ response body: string // TODO: allow this to be a struct, parsing the body.
+ }
+ task print: {
+ kind: "print"
+ text: task.http.response.body
+ }
+}
+
+runBase: {
+ task echo: {
+ kind: "exec"
+ stdout: string
+ }
+
+ task display: {
+ kind: "print"
+ text: task.echo.stdout
+ }
+}
diff --git a/cmd/cue/cmd/testdata/trim/trim.cue b/cmd/cue/cmd/testdata/trim/trim.cue
new file mode 100644
index 0000000..816af3f
--- /dev/null
+++ b/cmd/cue/cmd/testdata/trim/trim.cue
@@ -0,0 +1,73 @@
+package trim
+
+foo <Name>: {
+ _value: string
+
+ a: 4
+ b: string
+ d: 8
+ e: "foo"
+ f: ">> \( _value ) <<"
+ n: 5
+
+ list: ["foo", 8.0]
+
+ struct: {a: 3.0}
+
+ sList: [{a: 8, b: string}, {a: 9, b: "foo" | string}]
+ rList: [{a: "a"}]
+ rcList: [{a: "a", c: b}]
+
+ t <Name>: {
+ x: 0..5
+ }
+}
+
+foo bar: {
+ _value: "here"
+
+ a: 4
+ b: "foo"
+ c: 45
+ e: string
+ f: ">> here <<"
+
+ // The template does not require that this field be an integer (it may be
+ // a float), and thus this field specified additional information and
+ // cannot be removed.
+ n: int
+
+ struct: {a: 3.0}
+
+ list: ["foo", float]
+
+ sList: [{a: 8, b: "foo"}, {b: "foo"}]
+ rList: [{a: string}]
+ rcList: [{a: "a", c: "foo"}]
+}
+
+foo baz: {}
+
+foo multipath: {
+ t <Name>: {
+ // Combined with the other template, we know the value must be 5 and
+ // thus the entry below can be eliminated.
+ x: 5..8
+ }
+
+ t u: {
+ x: 5
+ }
+}
+
+// TODO: top-level fields are currently not removed.
+group: {
+ comp [k]: v for k, v in foo
+
+ comp bar: {
+ a: 4
+ aa: 8 // new value
+ }
+
+ comp baz: {} // removed: fully implied by comprehension above
+}
\ No newline at end of file
diff --git a/cmd/cue/cmd/testdata/trim/trim.out b/cmd/cue/cmd/testdata/trim/trim.out
new file mode 100644
index 0000000..73f746e
--- /dev/null
+++ b/cmd/cue/cmd/testdata/trim/trim.out
@@ -0,0 +1,59 @@
+package trim
+
+foo <Name>: {
+ _value: string
+
+ a: 4
+ b: string
+ d: 8
+ e: "foo"
+ f: ">> \( _value) <<"
+ n: 5
+
+ list: ["foo", 8.0]
+
+ struct: {a: 3.0}
+
+ sList: [{a: 8, b: string}, {a: 9, b: "foo" | string}]
+ rList: [{a: "a"}]
+ rcList: [{a: "a", c: b}]
+
+ t <Name>: {
+ x: 0..5
+ }
+}
+
+foo bar: {
+ _value: "here"
+ b: "foo"
+ c: 45
+
+ // The template does not require that this field be an integer (it may be
+ // a float), and thus this field specified additional information and
+ // cannot be removed.
+ n: int
+
+ sList: [{b: "foo"}, {}]
+}
+
+foo baz: {}
+
+foo multipath: {
+ t <Name>: {
+ // Combined with the other template, we know the value must be 5 and
+ // thus the entry below can be eliminated.
+ x: 5..8
+ }
+
+ t u: {
+ }
+}
+
+// TODO: top-level fields are currently not removed.
+group: {
+ comp [k]: v for k, v in foo
+
+ comp bar: {
+ aa: 8 // new value
+ }
+}
diff --git a/cmd/cue/cmd/trim.go b/cmd/cue/cmd/trim.go
new file mode 100644
index 0000000..cac1c8d
--- /dev/null
+++ b/cmd/cue/cmd/trim.go
@@ -0,0 +1,551 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "strconv"
+ "strings"
+
+ "cuelang.org/go/cue"
+ "cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/build"
+ "cuelang.org/go/cue/format"
+ "cuelang.org/go/cue/load"
+ "cuelang.org/go/cue/parser"
+ "cuelang.org/go/cue/token"
+ "cuelang.org/go/internal"
+ "github.com/spf13/cobra"
+)
+
+// TODO:
+// - remove the limitations mentioned in the documentation
+// - implement verification post-processing as extra safety
+
+// trimCmd represents the trim command
+var trimCmd = &cobra.Command{
+ Use: "trim",
+ Short: "remove superfluous fields",
+ Long: `trim removes fields from structs that are already defined by a template
+
+A field, struct, or list is removed if it is implied by a template, a list type
+value, a comprehension or any other implied content. It will modify the files
+in place.
+
+Limitations
+Removal is on a best effort basis. Some caveats:
+- Fields in implied content may refer to fields within the struct in which
+ they are included, but are only resolved on a best-effort basis.
+- Disjunctions that contain structs in implied content cannot be used to
+ remove fields.
+- There is currently no verification step: manual verification is required.
+
+Examples:
+
+ $ cat <<EOF > foo.cue
+ light <Name>: {
+ room: string
+ brightnessOff: 0.0 | 0..100.0
+ brightnessOn: 100.0 | 0..100.0
+ }
+
+ light ceiling50: {
+ room: "MasterBedroom"
+ brightnessOn: 100 // this line
+ brightnessOff: 0 // and this line will be removed
+ }
+ EOF
+
+ $ cue trim foo.cue
+ $ cat foo.cue
+ light <Name>: {
+ room: string
+ brightnessOff: 0.0 | 0..100.0
+ brightnessOn: 100.0 | 0..100.0
+ }
+
+ light ceiling50: {
+ room: "MasterBedroom"
+ }
+
+It is guaranteed that the resulting files give the same output as before the
+removal.
+`,
+ RunE: runTrim,
+}
+
+func init() {
+ RootCmd.AddCommand(trimCmd)
+ fOut = trimCmd.Flags().StringP("out", "o", "", "alternative output or - for stdout")
+}
+
+var (
+ fOut *string
+)
+
+func runTrim(cmd *cobra.Command, args []string) error {
+ log.SetOutput(cmd.OutOrStderr())
+
+ ctxt := build.NewContext(build.ParseOptions(parser.ParseComments))
+ fset := ctxt.FileSet()
+ binst := load.Instances(args, &load.Config{Context: ctxt})
+
+ instances := cue.Build(binst)
+ for _, inst := range instances {
+ if err := inst.Err; err != nil {
+ return err
+ }
+ }
+
+ if *fOut != "" && *fOut != "-" {
+ switch _, err := os.Stat(*fOut); {
+ case os.IsNotExist(err):
+ case err == nil:
+ default:
+ return fmt.Errorf("error creating file: %v", err)
+ }
+ }
+
+ for i, inst := range binst {
+
+ gen := newTrimSet(fset)
+ for _, f := range inst.Files {
+ gen.markNodes(f)
+ }
+
+ root := instances[i].Lookup()
+ rm := gen.trim("root", root, cue.Value{}, root)
+
+ if *fDryrun {
+ continue
+ }
+
+ for _, f := range inst.Files {
+ filename := f.Filename
+
+ f.Decls = gen.trimDecls(f.Decls, rm, root, false)
+
+ opts := []format.Option{}
+ if *fSimplify {
+ opts = append(opts, format.Simplify())
+ }
+
+ var buf bytes.Buffer
+ err := format.Node(&buf, f, opts...)
+ if err != nil {
+ return fmt.Errorf("error formatting file: %v", err)
+ }
+
+ if *fOut == "-" {
+ _, err := io.Copy(cmd.OutOrStdout(), &buf)
+ if err != nil {
+ return err
+ }
+ continue
+ } else if *fOut != "" {
+ filename = *fOut
+ }
+
+ err = ioutil.WriteFile(filename, buf.Bytes(), 0644)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+type trimSet struct {
+ fset *token.FileSet
+ stack []string
+ exclude map[ast.Node]bool // don't remove fields marked here
+ alwaysGen map[ast.Node]bool // node is always from a generated source
+}
+
+func newTrimSet(fset *token.FileSet) *trimSet {
+ return &trimSet{
+ fset: fset,
+ exclude: map[ast.Node]bool{},
+ alwaysGen: map[ast.Node]bool{},
+ }
+}
+
+func (t *trimSet) path() string {
+ return strings.Join(t.stack[1:], " ")
+}
+
+func (t *trimSet) traceMsg(msg string) {
+ if *fTrace {
+ fmt.Print(t.path())
+ msg = strings.TrimRight(msg, "\n")
+ msg = strings.Replace(msg, "\n", "\n ", -1)
+ fmt.Printf(": %s\n", msg)
+ }
+}
+
+func (t *trimSet) markNodes(n ast.Node) {
+ ast.Walk(n, nil, func(n ast.Node) {
+ switch x := n.(type) {
+ case *ast.Ident:
+ if x.Node != nil {
+ t.exclude[x.Node] = true
+ }
+ if x.Scope != nil {
+ t.exclude[x.Scope] = true
+ }
+
+ case *ast.ListLit:
+ if x.Type != nil {
+ t.markAlwaysGen(x.Type)
+ }
+
+ case *ast.Field:
+ if _, ok := x.Label.(*ast.TemplateLabel); ok {
+ t.markAlwaysGen(x.Value)
+ }
+
+ case *ast.ListComprehension, *ast.ComprehensionDecl:
+ t.markAlwaysGen(x)
+ }
+ })
+}
+
+func (t *trimSet) markAlwaysGen(n ast.Node) {
+ ast.Walk(n, func(n ast.Node) bool {
+ t.alwaysGen[n] = true
+ return true
+ }, nil)
+}
+
+func (t *trimSet) canRemove(n ast.Node) bool {
+ return !t.exclude[n] && !t.alwaysGen[n]
+}
+
+func isDisjunctionOfStruct(n ast.Node) bool {
+ switch x := n.(type) {
+ case *ast.BinaryExpr:
+ if x.Op == token.DISJUNCTION {
+ return hasStruct(x.X) || hasStruct(x.Y)
+ }
+ }
+ return false
+}
+
+func hasStruct(n ast.Node) bool {
+ hasStruct := false
+ ast.Walk(n, func(n ast.Node) bool {
+ if _, ok := n.(*ast.StructLit); ok {
+ hasStruct = true
+ }
+ return !hasStruct
+ }, nil)
+ return hasStruct
+}
+
+// trim strips fields from structs that would otherwise be generated by implied
+// content, such as templates, comprehensions, and list types.
+//
+// The algorithm walks the tree with two values in parallel: one for the full
+// configuration, and one for implied content. For each node in the tree it
+// determines the value of the implied content and that of the full value
+// and strips any of the non-implied fields if it subsumes the implied ones.
+//
+// There are a few gotchas:
+// - Fields in the implied content may refer to fields in the complete config.
+// To support this, incomplete fields are detected and evaluated within the
+// configuration.
+// - Templates are instantiated as a result of the declaration of concrete
+// sibling fields. Such fields should not be removed even if the instantiated
+// template completely subsumes such fields as the reason to instantiate
+// the template will disappear with it.
+// - As the parallel structure is different, it may resolve to different
+// default values. There is no support yet for selecting defaults of a value
+// based on another value without doing a full unification. So for now we
+// skip any disjunction containing structs.
+//
+func (t *trimSet) trim(label string, v, m, scope cue.Value) (rmSet []ast.Node) {
+ saved := t.stack
+ t.stack = append(t.stack, label)
+ defer func() { t.stack = saved }()
+
+ vSplit := v.Split()
+
+ // At the moment disjunctions of structs are note supported. Detect them and
+ // punt.
+ // TODO: support disjunctions.
+ mSplit := m.Split()
+ for _, v := range mSplit {
+ if isDisjunctionOfStruct(v.Source()) {
+ return
+ }
+ }
+
+ switch v.Kind() {
+ case cue.StructKind:
+ // TODO: merge template preprocessing with that of fields.
+
+ // Only keep the good parts of the template.
+ // Incoming structs may be incomplete resulting in errors. It is safe
+ // to ignore these. If there is an actual error, it will manifest in
+ // the evaluation of v.
+ in := cue.Value{}
+ gen := []ast.Node{}
+ for _, v := range mSplit {
+ // TODO: consider resolving incomplete values within the current
+ // scope, as we do for fields.
+ if v.IsValid() {
+ in = in.Unify(v)
+ }
+ // Collect generated nodes.
+ gen = append(gen, v.Source())
+ }
+
+ // Identify generated components and unify them with the mixin value.
+ for _, v := range v.Split() {
+ if src := v.Source(); t.alwaysGen[src] {
+ if w := in.Unify(v); w.Err() == nil {
+ in = w
+ }
+ gen = append(gen, src)
+ }
+ }
+
+ // Build map of mixin fields.
+ valueMap := map[key]cue.Value{}
+ for mIter, _ := in.AllFields(); mIter.Next(); {
+ valueMap[iterKey(mIter)] = mIter.Value()
+ }
+
+ fn := v.Template()
+
+ // Process fields.
+ rm := []ast.Node{}
+ for iter, _ := v.AllFields(); iter.Next(); {
+ mSub := valueMap[iterKey(iter)]
+ if fn != nil {
+ mSub = mSub.Unify(fn(iter.Label()))
+ }
+
+ removed := t.trim(iter.Label(), iter.Value(), mSub, v)
+ rm = append(rm, removed...)
+ }
+
+ if *fTrace {
+ w := &bytes.Buffer{}
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "value: ", v)
+ if in.Exists() {
+ fmt.Fprintln(w, "mixed in: ", in)
+ }
+ for _, v := range vSplit {
+ status := "[]"
+ src := v.Source()
+ if inNodes(rmSet, src) {
+ status = "[re]"
+ } else if t.alwaysGen[src] {
+ status = "[i]"
+ }
+ fmt.Fprintf(w, " %4s %v: %v %T\n", status, v.Pos(), internal.DebugStr(src), src)
+ }
+
+ t.traceMsg(w.String())
+ }
+
+ // Remove fields from source.
+ for _, v := range vSplit {
+ if src := v.Source(); !t.alwaysGen[src] {
+ switch x := src.(type) {
+ case *ast.File:
+ // TODO: use in instead?
+ x.Decls = t.trimDecls(x.Decls, rm, m, fn != nil)
+
+ case *ast.StructLit:
+ x.Elts = t.trimDecls(x.Elts, rm, m, fn != nil)
+ if len(x.Elts) == 0 && m.Exists() && t.canRemove(src) && !inNodes(gen, src) {
+ rmSet = append(rmSet, src)
+ }
+
+ default:
+ if len(t.stack) == 1 {
+ // TODO: fix this hack to pass down the fields to remove
+ return rm
+ }
+ }
+ }
+ }
+
+ case cue.ListKind:
+ mIter, _ := m.List()
+ i := 0
+ rmElem := []ast.Node{}
+ for iter, _ := v.List(); iter.Next(); i++ {
+ mIter.Next()
+ rm := t.trim(strconv.Itoa(i), iter.Value(), mIter.Value(), scope)
+ rmElem = append(rmElem, rm...)
+ }
+
+ // Signal the removal of lists of which all elements have been marked
+ // for removal.
+ for _, v := range vSplit {
+ if src := v.Source(); !t.alwaysGen[src] {
+ l, ok := src.(*ast.ListLit)
+ if !ok {
+ break
+ }
+ rmList := true
+ iter, _ := v.List()
+ for i := 0; i < len(l.Elts) && iter.Next(); i++ {
+ if !inNodes(rmElem, l.Elts[i]) {
+ rmList = false
+ break
+ }
+ }
+ if rmList && m.Exists() && t.canRemove(src) {
+ rmSet = append(rmSet, src)
+ }
+ }
+ }
+ fallthrough
+
+ default:
+ // Collect generated nodes.
+ in := cue.Value{}
+ gen := []ast.Node{}
+ for _, v := range mSplit {
+ if v.IsValid() {
+ in = in.Unify(v)
+ }
+ // Collect generated nodes.
+ gen = append(gen, v.Source())
+ }
+
+ for _, v := range vSplit {
+ src := v.Source()
+ if t.alwaysGen[src] || inNodes(gen, src) {
+ if v.IsIncomplete() {
+ // The template has an expression that cannot be fully
+ // resolved. Attempt to complete the expression by
+ // evaluting it within the struct to which the template
+ // is applied.
+ expr := v.Syntax()
+ // TODO: this only resolves references contained in scope.
+ v = internal.EvalExpr(scope, expr).(cue.Value)
+ }
+ in = in.Unify(v)
+ gen = append(gen, src)
+ }
+ }
+
+ // Mark any subsumed part that is covered by generated config.
+ if in.Err() == nil && v.Subsumes(in) {
+ for _, v := range vSplit {
+ src := v.Source()
+ if t.canRemove(src) && !inNodes(gen, src) {
+ rmSet = append(rmSet, src)
+ }
+ }
+ }
+
+ if *fTrace {
+ w := &bytes.Buffer{}
+ if len(rmSet) > 0 {
+ fmt.Fprint(w, "field: SUBSUMED\n")
+ } else {
+ fmt.Fprint(w, "field: \n")
+ }
+ fmt.Fprintln(w, "value: ", v)
+ if in.Exists() {
+ fmt.Fprintln(w, "mixed in: ", in)
+ }
+ for _, v := range vSplit {
+ status := "["
+ if inNodes(gen, v.Source()) {
+ status += "i"
+ }
+ if inNodes(rmSet, v.Source()) {
+ status += "r"
+ }
+ status += "]"
+ src := v.Source()
+ fmt.Fprintf(w, " %4s %v: %v\n", status, v.Pos(), internal.DebugStr(src))
+ }
+
+ t.traceMsg(w.String())
+ }
+ }
+ return rmSet
+}
+
+func (t *trimSet) trimDecls(decls []ast.Decl, rm []ast.Node, m cue.Value, fromTemplate bool) []ast.Decl {
+ a := make([]ast.Decl, 0, len(decls))
+
+ for _, d := range decls {
+ if f, ok := d.(*ast.Field); ok {
+ label, _ := ast.LabelName(f.Label)
+ v := m.Lookup(label)
+ if inNodes(rm, f.Value) && (!fromTemplate || v.Exists()) {
+ continue
+ }
+ }
+ a = append(a, d)
+ }
+ return a
+}
+
+func inNodes(a []ast.Node, n ast.Node) bool {
+ for _, e := range a {
+ if e == n {
+ return true
+ }
+ }
+ return false
+}
+
+func deleteNode(a []ast.Node, n ast.Node) []ast.Node {
+ k := 0
+ for _, e := range a {
+ if e != n {
+ a[k] = e
+ k++
+ }
+ }
+ return a[:k]
+}
+
+type key struct {
+ label string
+ hidden bool
+}
+
+func iterKey(v cue.Iterator) key {
+ return key{v.Label(), v.IsHidden()}
+}
+
+func nodeKey(v *ast.Field) key {
+ if v == nil {
+ return key{}
+ }
+ name, _ := ast.LabelName(v.Label)
+ hidden := false
+ if ident, ok := v.Label.(*ast.Ident); ok &&
+ strings.HasPrefix(ident.Name, "_") {
+ hidden = true
+ }
+ return key{name, hidden}
+}
diff --git a/cmd/cue/cmd/trim_test.go b/cmd/cue/cmd/trim_test.go
new file mode 100644
index 0000000..7b23082
--- /dev/null
+++ b/cmd/cue/cmd/trim_test.go
@@ -0,0 +1,22 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import "testing"
+
+func TestTrim(t *testing.T) {
+ trimCmd.ParseFlags([]string{"-o", "-"})
+ runCommand(t, trimCmd.RunE, "trim")
+}
diff --git a/cmd/cue/main.go b/cmd/cue/main.go
new file mode 100644
index 0000000..2edd4b1
--- /dev/null
+++ b/cmd/cue/main.go
@@ -0,0 +1,21 @@
+// Copyright 2018 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "cuelang.org/go/cmd/cue/cmd"
+
+func main() {
+ cmd.Execute()
+}
diff --git a/go.mod b/go.mod
index e180d70..5e108d3 100644
--- a/go.mod
+++ b/go.mod
@@ -3,8 +3,12 @@
require (
github.com/cockroachdb/apd v1.1.0
github.com/ghodss/yaml v1.0.0
+ github.com/google/go-cmp v0.2.0
+ github.com/mitchellh/go-homedir v1.0.0
github.com/pkg/errors v0.8.0 // indirect
+ github.com/spf13/cobra v0.0.3
+ github.com/spf13/viper v1.3.1
golang.org/x/exp/errors v0.0.0-20181210123644-7d6377eee41f
+ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
golang.org/x/tools v0.0.0-20181210225255-6a3e9aa2ab77
- gopkg.in/yaml.v2 v2.2.2 // indirect
)
diff --git a/go.sum b/go.sum
index 5293807..7c0bae2 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,53 @@
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
+github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/exp/errors v0.0.0-20181210123644-7d6377eee41f h1:B/8yFg7PHSFdahc+fMB+RUy3if9GlZmexAbcdfCwREI=
golang.org/x/exp/errors v0.0.0-20181210123644-7d6377eee41f/go.mod h1:YgqsNsAu4fTvlab/7uiYK9LJrCIzKg/NiZUIH1/ayqo=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20181210225255-6a3e9aa2ab77 h1:s+6psEFi3o1QryeA/qyvUoVaHMCQkYVvZ0i2ZolwSJc=
golang.org/x/tools v0.0.0-20181210225255-6a3e9aa2ab77/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pkg/doc.go b/pkg/doc.go
index d4476d0..307e5cb 100644
--- a/pkg/doc.go
+++ b/pkg/doc.go
@@ -12,17 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Package pkg define CUE builtin packages.
+// Package pkg define CUE standard packages.
//
-// The CUE core packages are different, but closely mimic the Go core packages.
-// The types, values, and functions are defined as their Go equivalence and
-// mapped to CUE types.
+// Many of the standard packages are modeled after and generated from the Go
+// core packages. The types, values, and functions are defined as their Go
+// equivalence and mapped to CUE types.
//
// Beware that some packages are defined in lesser-precision types than are
// typically used in CUE and thus may lead to loss of precision.
//
-// All packages in these subdirectories are hermetic. That is, the results
-// do not change based on environment conditions. That is:
+// All packages except those defined in the tool subdirectory are hermetic,
+// that is depending only on a known set of inputs, and therefore can guarantee
+// reproducible results. That is:
//
// - no reading of files contents
// - no querying of the file system of any kind
@@ -31,6 +32,7 @@
// - only reproduceable random generators
//
// Hermetic configurations allow for fast and advanced analysis that otherwise
-// would not be possible or practical. The cue "cmd" command can be used to
-// mix in non-hermetic influences into configurations.
+// would not be possible or practical. The cue "cmd" command can be used to mix
+// in non-hermetic influences into configurations by using packages defined
+// in the tool subdirectory.
package pkg
