| // Copyright 2018 The CUE Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package cmd |
| |
| import ( |
| "bytes" |
| "io" |
| "os" |
| "path/filepath" |
| "regexp" |
| "strings" |
| |
| "github.com/spf13/pflag" |
| "golang.org/x/text/language" |
| "golang.org/x/text/message" |
| |
| "cuelang.org/go/cue" |
| "cuelang.org/go/cue/ast" |
| "cuelang.org/go/cue/build" |
| "cuelang.org/go/cue/errors" |
| "cuelang.org/go/cue/load" |
| "cuelang.org/go/cue/parser" |
| "cuelang.org/go/cue/token" |
| "cuelang.org/go/internal/encoding" |
| "cuelang.org/go/internal/filetypes" |
| "cuelang.org/go/internal/value" |
| ) |
| |
| // Disallow |
| // - block comments |
| // - old-style field comprehensions |
| // - space separator syntax |
| const syntaxVersion = -1000 + 100*2 + 1 |
| |
| var requestedVersion = os.Getenv("CUE_SYNTAX_OVERRIDE") |
| |
| var defaultConfig = config{ |
| loadCfg: &load.Config{ |
| ParseFile: func(name string, src interface{}) (*ast.File, error) { |
| version := syntaxVersion |
| if requestedVersion != "" { |
| switch { |
| case strings.HasPrefix(requestedVersion, "v0.1"): |
| version = -1000 + 100 |
| } |
| } |
| return parser.ParseFile(name, src, |
| parser.FromVersion(version), |
| parser.ParseComments, |
| ) |
| }, |
| }, |
| } |
| |
| var runtime = &cue.Runtime{} |
| |
| var inTest = false |
| |
| func exitIfErr(cmd *Command, inst *cue.Instance, err error, fatal bool) { |
| exitOnErr(cmd, err, fatal) |
| } |
| |
| func getLang() language.Tag { |
| loc := os.Getenv("LC_ALL") |
| if loc == "" { |
| loc = os.Getenv("LANG") |
| } |
| loc = strings.Split(loc, ".")[0] |
| return language.Make(loc) |
| } |
| |
| func exitOnErr(cmd *Command, err error, fatal bool) { |
| if err == nil { |
| return |
| } |
| |
| // Link x/text as our localizer. |
| p := message.NewPrinter(getLang()) |
| format := func(w io.Writer, format string, args ...interface{}) { |
| p.Fprintf(w, format, args...) |
| } |
| |
| cwd, _ := os.Getwd() |
| |
| w := &bytes.Buffer{} |
| errors.Print(w, err, &errors.Config{ |
| Format: format, |
| Cwd: cwd, |
| ToSlash: inTest, |
| }) |
| |
| b := w.Bytes() |
| _, _ = cmd.Stderr().Write(b) |
| if fatal { |
| exit() |
| } |
| } |
| |
| func loadFromArgs(cmd *Command, args []string, cfg *load.Config) []*build.Instance { |
| binst := load.Instances(args, cfg) |
| if len(binst) == 0 { |
| return nil |
| } |
| |
| return binst |
| } |
| |
| // A buildPlan defines what should be done based on command line |
| // arguments and flags. |
| // |
| // TODO: allow --merge/-m to mix in other packages. |
| type buildPlan struct { |
| cmd *Command |
| insts []*build.Instance |
| |
| // instance is a pre-compiled instance, which exists if value files are |
| // being processed, which may require a schema to decode. |
| instance *cue.Instance |
| |
| cfg *config |
| |
| // If orphanFiles are mixed with CUE files and/or if placement flags are used, |
| // the instance is also included in insts. |
| importing bool |
| mergeData bool // do not merge individual data files. |
| orphaned []*decoderInfo |
| orphanInstance *build.Instance |
| // imported files are files that were orphaned in the build instance, but |
| // were placed in the instance by using one the --files, --list or --path |
| // flags. |
| imported []*ast.File |
| |
| expressions []ast.Expr // only evaluate these expressions within results |
| schema ast.Expr // selects schema in instance for orphaned values |
| |
| // orphan placement flags. |
| perFile bool |
| useList bool |
| path []ast.Label |
| useContext bool |
| |
| // outFile defines the file to output to. Default is CUE stdout. |
| outFile *build.File |
| |
| encConfig *encoding.Config |
| } |
| |
| // instances iterates either over a list of instances, or a list of |
| // data files. In the latter case, there must be either 0 or 1 other |
| // instance, with which the data instance may be merged. |
| func (b *buildPlan) instances() iterator { |
| var i iterator |
| switch { |
| case len(b.orphaned) > 0: |
| i = newStreamingIterator(b) |
| case len(b.insts) > 0: |
| i = &instanceIterator{ |
| inst: b.instance, |
| a: buildInstances(b.cmd, b.insts), |
| i: -1, |
| } |
| default: |
| i = &instanceIterator{ |
| inst: b.instance, |
| a: []*cue.Instance{b.instance}, |
| i: -1, |
| } |
| b.instance = nil |
| } |
| if len(b.expressions) > 0 { |
| return &expressionIter{ |
| iter: i, |
| expr: b.expressions, |
| i: len(b.expressions), |
| } |
| } |
| return i |
| } |
| |
| type iterator interface { |
| scan() bool |
| value() cue.Value |
| instance() *cue.Instance // may return nil |
| file() *ast.File // may return nil |
| err() error |
| close() |
| id() string |
| } |
| |
| type instanceIterator struct { |
| inst *cue.Instance |
| a []*cue.Instance |
| i int |
| e error |
| } |
| |
| func (i *instanceIterator) scan() bool { |
| i.i++ |
| return i.i < len(i.a) && i.e == nil |
| } |
| |
| func (i *instanceIterator) close() {} |
| func (i *instanceIterator) err() error { return i.e } |
| func (i *instanceIterator) value() cue.Value { |
| v := i.a[i.i].Value() |
| if i.inst != nil { |
| v = v.Unify(i.inst.Value()) |
| } |
| return v |
| } |
| func (i *instanceIterator) instance() *cue.Instance { |
| if i.i >= len(i.a) { |
| return nil |
| } |
| return i.a[i.i] |
| } |
| func (i *instanceIterator) file() *ast.File { return nil } |
| func (i *instanceIterator) id() string { return i.a[i.i].Dir } |
| |
| type streamingIterator struct { |
| r *cue.Runtime |
| b *buildPlan |
| cfg *encoding.Config |
| a []*decoderInfo |
| dec *encoding.Decoder |
| v cue.Value |
| f *ast.File |
| e error |
| } |
| |
| func newStreamingIterator(b *buildPlan) *streamingIterator { |
| i := &streamingIterator{ |
| cfg: b.encConfig, |
| a: b.orphaned, |
| b: b, |
| } |
| |
| // TODO: use orphanedSchema |
| i.r = &cue.Runtime{} |
| if v := b.encConfig.Schema; v.Exists() { |
| i.r = value.ConvertToRuntime(v.Context()) |
| } |
| |
| return i |
| } |
| |
| func (i *streamingIterator) file() *ast.File { return i.f } |
| func (i *streamingIterator) value() cue.Value { return i.v } |
| func (i *streamingIterator) instance() *cue.Instance { return nil } |
| |
| func (i *streamingIterator) id() string { |
| return "" |
| } |
| |
| func (i *streamingIterator) scan() bool { |
| if i.e != nil { |
| return false |
| } |
| |
| // advance to next value |
| if i.dec != nil && !i.dec.Done() { |
| i.dec.Next() |
| } |
| |
| // advance to next stream if necessary |
| for i.dec == nil || i.dec.Done() { |
| if i.dec != nil { |
| i.dec.Close() |
| i.dec = nil |
| } |
| if len(i.a) == 0 { |
| return false |
| } |
| |
| i.dec = i.a[0].dec(i.b) |
| if i.e = i.dec.Err(); i.e != nil { |
| return false |
| } |
| i.a = i.a[1:] |
| } |
| |
| // compose value |
| i.f = i.dec.File() |
| inst, err := i.r.CompileFile(i.f) |
| if err != nil { |
| i.e = err |
| return false |
| } |
| i.v = inst.Value() |
| if schema := i.b.encConfig.Schema; schema.Exists() { |
| i.e = schema.Err() |
| if i.e == nil { |
| i.v = i.v.Unify(schema) // TODO(required fields): don't merge in schema |
| i.e = i.v.Err() |
| } |
| i.f = nil |
| } |
| return i.e == nil |
| } |
| |
| func (i *streamingIterator) close() { |
| if i.dec != nil { |
| i.dec.Close() |
| i.dec = nil |
| } |
| } |
| |
| func (i *streamingIterator) err() error { |
| if i.dec != nil { |
| if err := i.dec.Err(); err != nil { |
| return err |
| } |
| } |
| return i.e |
| } |
| |
| type expressionIter struct { |
| iter iterator |
| expr []ast.Expr |
| i int |
| } |
| |
| func (i *expressionIter) err() error { return i.iter.err() } |
| func (i *expressionIter) close() { i.iter.close() } |
| func (i *expressionIter) id() string { return i.iter.id() } |
| |
| func (i *expressionIter) scan() bool { |
| i.i++ |
| if i.i < len(i.expr) { |
| return true |
| } |
| if !i.iter.scan() { |
| return false |
| } |
| i.i = 0 |
| return true |
| } |
| |
| func (i *expressionIter) file() *ast.File { return nil } |
| func (i *expressionIter) instance() *cue.Instance { return nil } |
| |
| func (i *expressionIter) value() cue.Value { |
| if len(i.expr) == 0 { |
| return i.iter.value() |
| } |
| v := i.iter.value() |
| path := "" |
| if inst := i.iter.instance(); inst != nil { |
| path = inst.ID() |
| } |
| return v.Context().BuildExpr(i.expr[i.i], |
| cue.Scope(v), |
| cue.InferBuiltins(true), |
| cue.ImportPath(path), |
| ) |
| } |
| |
| type config struct { |
| outMode filetypes.Mode |
| |
| fileFilter string |
| reFile *regexp.Regexp |
| encoding build.Encoding |
| interpretation build.Interpretation |
| |
| overrideDefault bool |
| |
| noMerge bool // do not merge individual data files. |
| |
| loadCfg *load.Config |
| } |
| |
| func newBuildPlan(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) { |
| if cfg == nil { |
| cfg = &defaultConfig |
| } |
| if cfg.loadCfg == nil { |
| cfg.loadCfg = defaultConfig.loadCfg |
| } |
| cfg.loadCfg.Stdin = cmd.InOrStdin() |
| |
| p = &buildPlan{cfg: cfg, cmd: cmd, importing: cfg.loadCfg.DataFiles} |
| |
| if err := p.parseFlags(); err != nil { |
| return nil, err |
| } |
| re, err := regexp.Compile(p.cfg.fileFilter) |
| if err != nil { |
| return nil, err |
| } |
| cfg.reFile = re |
| |
| if err := setTags(cmd.Flags(), cfg.loadCfg); err != nil { |
| return nil, err |
| } |
| |
| return p, nil |
| } |
| |
| func (p *buildPlan) matchFile(file string) bool { |
| return p.cfg.reFile.MatchString(file) |
| } |
| |
| func setTags(f *pflag.FlagSet, cfg *load.Config) error { |
| tags, _ := f.GetStringArray(string(flagInject)) |
| cfg.Tags = tags |
| if b, _ := f.GetBool(string(flagInjectVars)); b { |
| cfg.TagVars = load.DefaultTagVars() |
| } |
| return nil |
| } |
| |
| type decoderInfo struct { |
| file *build.File |
| d *encoding.Decoder // may be nil if delayed |
| } |
| |
| func (d *decoderInfo) dec(b *buildPlan) *encoding.Decoder { |
| if d.d == nil { |
| d.d = encoding.NewDecoder(d.file, b.encConfig) |
| } |
| return d.d |
| } |
| |
| func (d *decoderInfo) close() { |
| if d.d != nil { |
| d.d.Close() |
| } |
| } |
| |
| // getDecoders takes the orphaned files of the given instance and splits them in |
| // schemas and values, saving the build.File and encoding.Decoder in the |
| // returned slices. It is up to the caller to Close any of the decoders that are |
| // returned. |
| func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderInfo, err error) { |
| files := b.OrphanedFiles |
| if p.cfg.overrideDefault { |
| files = append(files, b.UnknownFiles...) |
| } |
| for _, f := range files { |
| if !b.User && !p.matchFile(f.Filename) { |
| continue |
| } |
| if p.cfg.overrideDefault { |
| f.Encoding = p.cfg.encoding |
| f.Interpretation = p.cfg.interpretation |
| } |
| switch f.Encoding { |
| case build.Protobuf, build.YAML, build.JSON, build.JSONL, |
| build.Text, build.Binary: |
| if f.Interpretation == build.ProtobufJSON { |
| // Need a schema. |
| values = append(values, &decoderInfo{f, nil}) |
| continue |
| } |
| case build.TextProto: |
| if p.importing { |
| return schemas, values, errors.Newf(token.NoPos, |
| "cannot import textproto files") |
| } |
| // Needs to be decoded after any schema. |
| values = append(values, &decoderInfo{f, nil}) |
| continue |
| default: |
| return schemas, values, errors.Newf(token.NoPos, |
| "unsupported encoding %q", f.Encoding) |
| } |
| |
| // We add the module root to the path if there is a module defined. |
| c := *p.encConfig |
| if b.Module != "" { |
| c.ProtoPath = append(c.ProtoPath, b.Root) |
| } |
| d := encoding.NewDecoder(f, &c) |
| |
| fi, err := filetypes.FromFile(f, p.cfg.outMode) |
| if err != nil { |
| return schemas, values, err |
| } |
| switch { |
| // case !fi.Schema: // TODO: value/schema/auto |
| // values = append(values, d) |
| case fi.Form != build.Schema && fi.Form != build.Final: |
| values = append(values, &decoderInfo{f, d}) |
| |
| case f.Interpretation != build.Auto: |
| schemas = append(schemas, &decoderInfo{f, d}) |
| |
| case d.Interpretation() == "": |
| values = append(values, &decoderInfo{f, d}) |
| |
| default: |
| schemas = append(schemas, &decoderInfo{f, d}) |
| } |
| } |
| return schemas, values, nil |
| } |
| |
| // importFiles imports orphan files for existing instances. Note that during |
| // import, both schemas and non-schemas are placed (TODO: should we allow schema |
| // mode here as well? It seems that the existing package should have enough |
| // typing to allow for schemas). |
| // |
| // It is a separate call to allow closing decoders between processing each |
| // package. |
| func (p *buildPlan) importFiles(b *build.Instance) error { |
| // TODO: assume textproto is imported at top-level or just ignore them. |
| |
| schemas, values, err := p.getDecoders(b) |
| for _, d := range append(schemas, values...) { |
| defer d.close() |
| } |
| if err != nil { |
| return err |
| } |
| return p.placeOrphans(b, append(schemas, values...)) |
| } |
| |
| func parseArgs(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) { |
| p, err = newBuildPlan(cmd, args, cfg) |
| if err != nil { |
| return nil, err |
| } |
| |
| builds := loadFromArgs(cmd, args, cfg.loadCfg) |
| if builds == nil { |
| return nil, errors.Newf(token.NoPos, "invalid args") |
| } |
| |
| if err := p.parsePlacementFlags(); err != nil { |
| return nil, err |
| } |
| |
| for _, b := range builds { |
| if b.Err != nil { |
| return nil, b.Err |
| } |
| switch { |
| case !b.User: |
| if p.importing { |
| if err := p.importFiles(b); err != nil { |
| return nil, err |
| } |
| } |
| p.insts = append(p.insts, b) |
| |
| case p.orphanInstance != nil: |
| return nil, errors.Newf(token.NoPos, |
| "builds contain two file packages") |
| |
| default: |
| p.orphanInstance = b |
| } |
| } |
| |
| if len(p.insts) == 0 && flagGlob.String(p.cmd) != "" { |
| return nil, errors.Newf(token.NoPos, |
| "use of -n/--name flag without a directory") |
| } |
| |
| if b := p.orphanInstance; b != nil { |
| schemas, values, err := p.getDecoders(b) |
| for _, d := range append(schemas, values...) { |
| defer d.close() |
| } |
| if err != nil { |
| return nil, err |
| } |
| |
| if values == nil { |
| values, schemas = schemas, values |
| } |
| |
| for _, di := range schemas { |
| d := di.dec(p) |
| for ; !d.Done(); d.Next() { |
| if err := b.AddSyntax(d.File()); err != nil { |
| return nil, err |
| } |
| } |
| if err := d.Err(); err != nil { |
| return nil, err |
| } |
| } |
| |
| if len(p.insts) > 1 && p.schema != nil { |
| return nil, errors.Newf(token.NoPos, |
| "cannot use --schema/-d with flag more than one schema") |
| } |
| |
| var schema *build.Instance |
| switch n := len(p.insts); n { |
| default: |
| return nil, errors.Newf(token.NoPos, |
| "too many packages defined (%d) in combination with files", n) |
| case 1: |
| if len(schemas) > 0 { |
| return nil, errors.Newf(token.NoPos, |
| "cannot combine packages with individual schema files") |
| } |
| schema = p.insts[0] |
| p.insts = nil |
| |
| case 0: |
| bb := *b |
| schema = &bb |
| b.BuildFiles = nil |
| b.Files = nil |
| } |
| |
| if schema != nil && len(schema.Files) > 0 { |
| inst := buildInstances(p.cmd, []*build.Instance{schema})[0] |
| |
| if inst.Err != nil { |
| return nil, err |
| } |
| p.instance = inst |
| p.encConfig.Schema = inst.Value() |
| if p.schema != nil { |
| v := inst.Eval(p.schema) |
| if err := v.Err(); err != nil { |
| return nil, err |
| } |
| p.encConfig.Schema = v |
| } |
| } else if p.schema != nil { |
| return nil, errors.Newf(token.NoPos, |
| "-d/--schema flag specified without a schema") |
| } |
| |
| switch { |
| default: |
| fallthrough |
| |
| case p.schema != nil: |
| p.orphaned = values |
| |
| case p.mergeData, p.usePlacement(), p.importing: |
| if err = p.placeOrphans(b, values); err != nil { |
| return nil, err |
| } |
| |
| } |
| |
| if len(b.Files) > 0 { |
| p.insts = append(p.insts, b) |
| } |
| } |
| |
| if len(p.expressions) > 1 { |
| p.encConfig.Stream = true |
| } |
| return p, nil |
| } |
| |
| func (b *buildPlan) parseFlags() (err error) { |
| b.mergeData = !b.cfg.noMerge && flagMerge.Bool(b.cmd) |
| |
| out := flagOut.String(b.cmd) |
| outFile := flagOutFile.String(b.cmd) |
| |
| if strings.Contains(out, ":") && strings.Contains(outFile, ":") { |
| return errors.Newf(token.NoPos, |
| "cannot specify qualifier in both --out and --outfile") |
| } |
| if outFile == "" { |
| outFile = "-" |
| } |
| if out != "" { |
| outFile = out + ":" + outFile |
| } |
| b.outFile, err = filetypes.ParseFile(outFile, b.cfg.outMode) |
| if err != nil { |
| return err |
| } |
| |
| for _, e := range flagExpression.StringArray(b.cmd) { |
| expr, err := parser.ParseExpr("--expression", e) |
| if err != nil { |
| return err |
| } |
| b.expressions = append(b.expressions, expr) |
| } |
| if s := flagSchema.String(b.cmd); s != "" { |
| b.schema, err = parser.ParseExpr("--schema", s) |
| if err != nil { |
| return err |
| } |
| } |
| if s := flagGlob.String(b.cmd); s != "" { |
| // Set a default file filter to only include json and yaml files |
| b.cfg.fileFilter = s |
| } |
| b.encConfig = &encoding.Config{ |
| Force: flagForce.Bool(b.cmd), |
| Mode: b.cfg.outMode, |
| Stdin: b.cmd.InOrStdin(), |
| Stdout: b.cmd.OutOrStdout(), |
| ProtoPath: flagProtoPath.StringArray(b.cmd), |
| AllErrors: flagAllErrors.Bool(b.cmd), |
| PkgName: flagPackage.String(b.cmd), |
| Strict: flagStrict.Bool(b.cmd), |
| } |
| return nil |
| } |
| |
| func buildInstances(cmd *Command, binst []*build.Instance) []*cue.Instance { |
| // TODO: |
| // If there are no files and User is true, then use those? |
| // Always use all files in user mode? |
| instances := cue.Build(binst) |
| for _, inst := range instances { |
| // TODO: consider merging errors of multiple files, but ensure |
| // duplicates are removed. |
| exitIfErr(cmd, inst, inst.Err, true) |
| } |
| |
| if flagIgnore.Bool(cmd) { |
| return instances |
| } |
| |
| // TODO check errors after the fact in case of ignore. |
| for _, inst := range instances { |
| // TODO: consider merging errors of multiple files, but ensure |
| // duplicates are removed. |
| exitIfErr(cmd, inst, inst.Value().Validate(), !flagIgnore.Bool(cmd)) |
| } |
| return instances |
| } |
| |
| func buildToolInstances(cmd *Command, binst []*build.Instance) ([]*cue.Instance, error) { |
| instances := cue.Build(binst) |
| for _, inst := range instances { |
| if inst.Err != nil { |
| return nil, inst.Err |
| } |
| } |
| |
| // TODO check errors after the fact in case of ignore. |
| for _, inst := range instances { |
| if err := inst.Value().Validate(); err != nil { |
| return nil, err |
| } |
| } |
| return instances, nil |
| } |
| |
| func buildTools(cmd *Command, args []string) (*cue.Instance, error) { |
| |
| cfg := &load.Config{ |
| Tools: true, |
| } |
| f := cmd.cmd.Flags() |
| if err := setTags(f, cfg); err != nil { |
| return nil, err |
| } |
| |
| binst := loadFromArgs(cmd, args, cfg) |
| if len(binst) == 0 { |
| return nil, nil |
| } |
| included := map[string]bool{} |
| |
| ti := binst[0].Context().NewInstance(binst[0].Root, nil) |
| for _, inst := range binst { |
| k := 0 |
| for _, f := range inst.Files { |
| if strings.HasSuffix(f.Filename, "_tool.cue") { |
| if !included[f.Filename] { |
| _ = ti.AddSyntax(f) |
| included[f.Filename] = true |
| } |
| continue |
| } |
| inst.Files[k] = f |
| k++ |
| } |
| inst.Files = inst.Files[:k] |
| } |
| |
| insts, err := buildToolInstances(cmd, binst) |
| if err != nil { |
| return nil, err |
| } |
| |
| inst := insts[0] |
| if len(insts) > 1 { |
| inst = cue.Merge(insts...) |
| } |
| |
| r := value.ConvertToRuntime(inst.Value().Context()) |
| for _, b := range binst { |
| for _, i := range b.Imports { |
| if _, err := r.Build(i); err != nil { |
| return nil, err |
| } |
| } |
| } |
| |
| // Set path equal to the package from which it is loading. |
| ti.ImportPath = binst[0].ImportPath |
| |
| inst = inst.Build(ti) |
| return inst, inst.Err |
| } |
| |
| func shortFile(root string, f *build.File) string { |
| dir, _ := filepath.Rel(root, f.Filename) |
| if dir == "" { |
| return f.Filename |
| } |
| if !filepath.IsAbs(dir) { |
| dir = "." + string(filepath.Separator) + dir |
| } |
| return dir |
| } |