blob: 31c2f64539eb5f8dc617f0d83d963a6e58231588 [file] [log] [blame]
// Copyright 2018 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"bytes"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/pflag"
"golang.org/x/text/language"
"golang.org/x/text/message"
"cuelang.org/go/cue"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/build"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/load"
"cuelang.org/go/cue/parser"
"cuelang.org/go/cue/token"
"cuelang.org/go/internal/encoding"
"cuelang.org/go/internal/filetypes"
"cuelang.org/go/internal/value"
)
// Disallow
// - block comments
// - old-style field comprehensions
// - space separator syntax
const syntaxVersion = -1000 + 100*2 + 1
var requestedVersion = os.Getenv("CUE_SYNTAX_OVERRIDE")
var defaultConfig = config{
loadCfg: &load.Config{
ParseFile: func(name string, src interface{}) (*ast.File, error) {
version := syntaxVersion
if requestedVersion != "" {
switch {
case strings.HasPrefix(requestedVersion, "v0.1"):
version = -1000 + 100
}
}
return parser.ParseFile(name, src,
parser.FromVersion(version),
parser.ParseComments,
)
},
},
}
var runtime = &cue.Runtime{}
var inTest = false
func exitIfErr(cmd *Command, inst *cue.Instance, err error, fatal bool) {
exitOnErr(cmd, err, fatal)
}
func getLang() language.Tag {
loc := os.Getenv("LC_ALL")
if loc == "" {
loc = os.Getenv("LANG")
}
loc = strings.Split(loc, ".")[0]
return language.Make(loc)
}
func exitOnErr(cmd *Command, err error, fatal bool) {
if err == nil {
return
}
// Link x/text as our localizer.
p := message.NewPrinter(getLang())
format := func(w io.Writer, format string, args ...interface{}) {
p.Fprintf(w, format, args...)
}
cwd, _ := os.Getwd()
w := &bytes.Buffer{}
errors.Print(w, err, &errors.Config{
Format: format,
Cwd: cwd,
ToSlash: inTest,
})
b := w.Bytes()
_, _ = cmd.Stderr().Write(b)
if fatal {
exit()
}
}
func loadFromArgs(cmd *Command, args []string, cfg *load.Config) []*build.Instance {
binst := load.Instances(args, cfg)
if len(binst) == 0 {
return nil
}
return binst
}
// A buildPlan defines what should be done based on command line
// arguments and flags.
//
// TODO: allow --merge/-m to mix in other packages.
type buildPlan struct {
cmd *Command
insts []*build.Instance
// instance is a pre-compiled instance, which exists if value files are
// being processed, which may require a schema to decode.
instance *cue.Instance
cfg *config
// If orphanFiles are mixed with CUE files and/or if placement flags are used,
// the instance is also included in insts.
importing bool
mergeData bool // do not merge individual data files.
orphaned []*decoderInfo
orphanInstance *build.Instance
// imported files are files that were orphaned in the build instance, but
// were placed in the instance by using one the --files, --list or --path
// flags.
imported []*ast.File
expressions []ast.Expr // only evaluate these expressions within results
schema ast.Expr // selects schema in instance for orphaned values
// orphan placement flags.
perFile bool
useList bool
path []ast.Label
useContext bool
// outFile defines the file to output to. Default is CUE stdout.
outFile *build.File
encConfig *encoding.Config
}
// instances iterates either over a list of instances, or a list of
// data files. In the latter case, there must be either 0 or 1 other
// instance, with which the data instance may be merged.
func (b *buildPlan) instances() iterator {
var i iterator
switch {
case len(b.orphaned) > 0:
i = newStreamingIterator(b)
case len(b.insts) > 0:
i = &instanceIterator{
inst: b.instance,
a: buildInstances(b.cmd, b.insts),
i: -1,
}
default:
i = &instanceIterator{
inst: b.instance,
a: []*cue.Instance{b.instance},
i: -1,
}
b.instance = nil
}
if len(b.expressions) > 0 {
return &expressionIter{
iter: i,
expr: b.expressions,
i: len(b.expressions),
}
}
return i
}
type iterator interface {
scan() bool
value() cue.Value
instance() *cue.Instance // may return nil
file() *ast.File // may return nil
err() error
close()
id() string
}
type instanceIterator struct {
inst *cue.Instance
a []*cue.Instance
i int
e error
}
func (i *instanceIterator) scan() bool {
i.i++
return i.i < len(i.a) && i.e == nil
}
func (i *instanceIterator) close() {}
func (i *instanceIterator) err() error { return i.e }
func (i *instanceIterator) value() cue.Value {
v := i.a[i.i].Value()
if i.inst != nil {
v = v.Unify(i.inst.Value())
}
return v
}
func (i *instanceIterator) instance() *cue.Instance {
if i.i >= len(i.a) {
return nil
}
return i.a[i.i]
}
func (i *instanceIterator) file() *ast.File { return nil }
func (i *instanceIterator) id() string { return i.a[i.i].Dir }
type streamingIterator struct {
r *cue.Runtime
b *buildPlan
cfg *encoding.Config
a []*decoderInfo
dec *encoding.Decoder
v cue.Value
f *ast.File
e error
}
func newStreamingIterator(b *buildPlan) *streamingIterator {
i := &streamingIterator{
cfg: b.encConfig,
a: b.orphaned,
b: b,
}
// TODO: use orphanedSchema
i.r = &cue.Runtime{}
if v := b.encConfig.Schema; v.Exists() {
i.r = value.ConvertToRuntime(v.Context())
}
return i
}
func (i *streamingIterator) file() *ast.File { return i.f }
func (i *streamingIterator) value() cue.Value { return i.v }
func (i *streamingIterator) instance() *cue.Instance { return nil }
func (i *streamingIterator) id() string {
return ""
}
func (i *streamingIterator) scan() bool {
if i.e != nil {
return false
}
// advance to next value
if i.dec != nil && !i.dec.Done() {
i.dec.Next()
}
// advance to next stream if necessary
for i.dec == nil || i.dec.Done() {
if i.dec != nil {
i.dec.Close()
i.dec = nil
}
if len(i.a) == 0 {
return false
}
i.dec = i.a[0].dec(i.b)
if i.e = i.dec.Err(); i.e != nil {
return false
}
i.a = i.a[1:]
}
// compose value
i.f = i.dec.File()
inst, err := i.r.CompileFile(i.f)
if err != nil {
i.e = err
return false
}
i.v = inst.Value()
if schema := i.b.encConfig.Schema; schema.Exists() {
i.e = schema.Err()
if i.e == nil {
i.v = i.v.Unify(schema) // TODO(required fields): don't merge in schema
i.e = i.v.Err()
}
i.f = nil
}
return i.e == nil
}
func (i *streamingIterator) close() {
if i.dec != nil {
i.dec.Close()
i.dec = nil
}
}
func (i *streamingIterator) err() error {
if i.dec != nil {
if err := i.dec.Err(); err != nil {
return err
}
}
return i.e
}
type expressionIter struct {
iter iterator
expr []ast.Expr
i int
}
func (i *expressionIter) err() error { return i.iter.err() }
func (i *expressionIter) close() { i.iter.close() }
func (i *expressionIter) id() string { return i.iter.id() }
func (i *expressionIter) scan() bool {
i.i++
if i.i < len(i.expr) {
return true
}
if !i.iter.scan() {
return false
}
i.i = 0
return true
}
func (i *expressionIter) file() *ast.File { return nil }
func (i *expressionIter) instance() *cue.Instance { return nil }
func (i *expressionIter) value() cue.Value {
if len(i.expr) == 0 {
return i.iter.value()
}
v := i.iter.value()
path := ""
if inst := i.iter.instance(); inst != nil {
path = inst.ID()
}
return v.Context().BuildExpr(i.expr[i.i],
cue.Scope(v),
cue.InferBuiltins(true),
cue.ImportPath(path),
)
}
type config struct {
outMode filetypes.Mode
fileFilter string
reFile *regexp.Regexp
encoding build.Encoding
interpretation build.Interpretation
overrideDefault bool
noMerge bool // do not merge individual data files.
loadCfg *load.Config
}
func newBuildPlan(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) {
if cfg == nil {
cfg = &defaultConfig
}
if cfg.loadCfg == nil {
cfg.loadCfg = defaultConfig.loadCfg
}
cfg.loadCfg.Stdin = cmd.InOrStdin()
p = &buildPlan{cfg: cfg, cmd: cmd, importing: cfg.loadCfg.DataFiles}
if err := p.parseFlags(); err != nil {
return nil, err
}
re, err := regexp.Compile(p.cfg.fileFilter)
if err != nil {
return nil, err
}
cfg.reFile = re
if err := setTags(cmd.Flags(), cfg.loadCfg); err != nil {
return nil, err
}
return p, nil
}
func (p *buildPlan) matchFile(file string) bool {
return p.cfg.reFile.MatchString(file)
}
func setTags(f *pflag.FlagSet, cfg *load.Config) error {
tags, _ := f.GetStringArray(string(flagInject))
cfg.Tags = tags
if b, _ := f.GetBool(string(flagInjectVars)); b {
cfg.TagVars = load.DefaultTagVars()
}
return nil
}
type decoderInfo struct {
file *build.File
d *encoding.Decoder // may be nil if delayed
}
func (d *decoderInfo) dec(b *buildPlan) *encoding.Decoder {
if d.d == nil {
d.d = encoding.NewDecoder(d.file, b.encConfig)
}
return d.d
}
func (d *decoderInfo) close() {
if d.d != nil {
d.d.Close()
}
}
// getDecoders takes the orphaned files of the given instance and splits them in
// schemas and values, saving the build.File and encoding.Decoder in the
// returned slices. It is up to the caller to Close any of the decoders that are
// returned.
func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderInfo, err error) {
files := b.OrphanedFiles
if p.cfg.overrideDefault {
files = append(files, b.UnknownFiles...)
}
for _, f := range files {
if !b.User && !p.matchFile(f.Filename) {
continue
}
if p.cfg.overrideDefault {
f.Encoding = p.cfg.encoding
f.Interpretation = p.cfg.interpretation
}
switch f.Encoding {
case build.Protobuf, build.YAML, build.JSON, build.JSONL,
build.Text, build.Binary:
if f.Interpretation == build.ProtobufJSON {
// Need a schema.
values = append(values, &decoderInfo{f, nil})
continue
}
case build.TextProto:
if p.importing {
return schemas, values, errors.Newf(token.NoPos,
"cannot import textproto files")
}
// Needs to be decoded after any schema.
values = append(values, &decoderInfo{f, nil})
continue
default:
return schemas, values, errors.Newf(token.NoPos,
"unsupported encoding %q", f.Encoding)
}
// We add the module root to the path if there is a module defined.
c := *p.encConfig
if b.Module != "" {
c.ProtoPath = append(c.ProtoPath, b.Root)
}
d := encoding.NewDecoder(f, &c)
fi, err := filetypes.FromFile(f, p.cfg.outMode)
if err != nil {
return schemas, values, err
}
switch {
// case !fi.Schema: // TODO: value/schema/auto
// values = append(values, d)
case fi.Form != build.Schema && fi.Form != build.Final:
values = append(values, &decoderInfo{f, d})
case f.Interpretation != build.Auto:
schemas = append(schemas, &decoderInfo{f, d})
case d.Interpretation() == "":
values = append(values, &decoderInfo{f, d})
default:
schemas = append(schemas, &decoderInfo{f, d})
}
}
return schemas, values, nil
}
// importFiles imports orphan files for existing instances. Note that during
// import, both schemas and non-schemas are placed (TODO: should we allow schema
// mode here as well? It seems that the existing package should have enough
// typing to allow for schemas).
//
// It is a separate call to allow closing decoders between processing each
// package.
func (p *buildPlan) importFiles(b *build.Instance) error {
// TODO: assume textproto is imported at top-level or just ignore them.
schemas, values, err := p.getDecoders(b)
for _, d := range append(schemas, values...) {
defer d.close()
}
if err != nil {
return err
}
return p.placeOrphans(b, append(schemas, values...))
}
func parseArgs(cmd *Command, args []string, cfg *config) (p *buildPlan, err error) {
p, err = newBuildPlan(cmd, args, cfg)
if err != nil {
return nil, err
}
builds := loadFromArgs(cmd, args, cfg.loadCfg)
if builds == nil {
return nil, errors.Newf(token.NoPos, "invalid args")
}
if err := p.parsePlacementFlags(); err != nil {
return nil, err
}
for _, b := range builds {
if b.Err != nil {
return nil, b.Err
}
switch {
case !b.User:
if p.importing {
if err := p.importFiles(b); err != nil {
return nil, err
}
}
p.insts = append(p.insts, b)
case p.orphanInstance != nil:
return nil, errors.Newf(token.NoPos,
"builds contain two file packages")
default:
p.orphanInstance = b
}
}
if len(p.insts) == 0 && flagGlob.String(p.cmd) != "" {
return nil, errors.Newf(token.NoPos,
"use of -n/--name flag without a directory")
}
if b := p.orphanInstance; b != nil {
schemas, values, err := p.getDecoders(b)
for _, d := range append(schemas, values...) {
defer d.close()
}
if err != nil {
return nil, err
}
if values == nil {
values, schemas = schemas, values
}
for _, di := range schemas {
d := di.dec(p)
for ; !d.Done(); d.Next() {
if err := b.AddSyntax(d.File()); err != nil {
return nil, err
}
}
if err := d.Err(); err != nil {
return nil, err
}
}
if len(p.insts) > 1 && p.schema != nil {
return nil, errors.Newf(token.NoPos,
"cannot use --schema/-d with flag more than one schema")
}
var schema *build.Instance
switch n := len(p.insts); n {
default:
return nil, errors.Newf(token.NoPos,
"too many packages defined (%d) in combination with files", n)
case 1:
if len(schemas) > 0 {
return nil, errors.Newf(token.NoPos,
"cannot combine packages with individual schema files")
}
schema = p.insts[0]
p.insts = nil
case 0:
bb := *b
schema = &bb
b.BuildFiles = nil
b.Files = nil
}
if schema != nil && len(schema.Files) > 0 {
inst := buildInstances(p.cmd, []*build.Instance{schema})[0]
if inst.Err != nil {
return nil, err
}
p.instance = inst
p.encConfig.Schema = inst.Value()
if p.schema != nil {
v := inst.Eval(p.schema)
if err := v.Err(); err != nil {
return nil, err
}
p.encConfig.Schema = v
}
} else if p.schema != nil {
return nil, errors.Newf(token.NoPos,
"-d/--schema flag specified without a schema")
}
switch {
default:
fallthrough
case p.schema != nil:
p.orphaned = values
case p.mergeData, p.usePlacement(), p.importing:
if err = p.placeOrphans(b, values); err != nil {
return nil, err
}
}
if len(b.Files) > 0 {
p.insts = append(p.insts, b)
}
}
if len(p.expressions) > 1 {
p.encConfig.Stream = true
}
return p, nil
}
func (b *buildPlan) parseFlags() (err error) {
b.mergeData = !b.cfg.noMerge && flagMerge.Bool(b.cmd)
out := flagOut.String(b.cmd)
outFile := flagOutFile.String(b.cmd)
if strings.Contains(out, ":") && strings.Contains(outFile, ":") {
return errors.Newf(token.NoPos,
"cannot specify qualifier in both --out and --outfile")
}
if outFile == "" {
outFile = "-"
}
if out != "" {
outFile = out + ":" + outFile
}
b.outFile, err = filetypes.ParseFile(outFile, b.cfg.outMode)
if err != nil {
return err
}
for _, e := range flagExpression.StringArray(b.cmd) {
expr, err := parser.ParseExpr("--expression", e)
if err != nil {
return err
}
b.expressions = append(b.expressions, expr)
}
if s := flagSchema.String(b.cmd); s != "" {
b.schema, err = parser.ParseExpr("--schema", s)
if err != nil {
return err
}
}
if s := flagGlob.String(b.cmd); s != "" {
// Set a default file filter to only include json and yaml files
b.cfg.fileFilter = s
}
b.encConfig = &encoding.Config{
Force: flagForce.Bool(b.cmd),
Mode: b.cfg.outMode,
Stdin: b.cmd.InOrStdin(),
Stdout: b.cmd.OutOrStdout(),
ProtoPath: flagProtoPath.StringArray(b.cmd),
AllErrors: flagAllErrors.Bool(b.cmd),
PkgName: flagPackage.String(b.cmd),
Strict: flagStrict.Bool(b.cmd),
}
return nil
}
func buildInstances(cmd *Command, binst []*build.Instance) []*cue.Instance {
// TODO:
// If there are no files and User is true, then use those?
// Always use all files in user mode?
instances := cue.Build(binst)
for _, inst := range instances {
// TODO: consider merging errors of multiple files, but ensure
// duplicates are removed.
exitIfErr(cmd, inst, inst.Err, true)
}
if flagIgnore.Bool(cmd) {
return instances
}
// TODO check errors after the fact in case of ignore.
for _, inst := range instances {
// TODO: consider merging errors of multiple files, but ensure
// duplicates are removed.
exitIfErr(cmd, inst, inst.Value().Validate(), !flagIgnore.Bool(cmd))
}
return instances
}
func buildToolInstances(cmd *Command, binst []*build.Instance) ([]*cue.Instance, error) {
instances := cue.Build(binst)
for _, inst := range instances {
if inst.Err != nil {
return nil, inst.Err
}
}
// TODO check errors after the fact in case of ignore.
for _, inst := range instances {
if err := inst.Value().Validate(); err != nil {
return nil, err
}
}
return instances, nil
}
func buildTools(cmd *Command, args []string) (*cue.Instance, error) {
cfg := &load.Config{
Tools: true,
}
f := cmd.cmd.Flags()
if err := setTags(f, cfg); err != nil {
return nil, err
}
binst := loadFromArgs(cmd, args, cfg)
if len(binst) == 0 {
return nil, nil
}
included := map[string]bool{}
ti := binst[0].Context().NewInstance(binst[0].Root, nil)
for _, inst := range binst {
k := 0
for _, f := range inst.Files {
if strings.HasSuffix(f.Filename, "_tool.cue") {
if !included[f.Filename] {
_ = ti.AddSyntax(f)
included[f.Filename] = true
}
continue
}
inst.Files[k] = f
k++
}
inst.Files = inst.Files[:k]
}
insts, err := buildToolInstances(cmd, binst)
if err != nil {
return nil, err
}
inst := insts[0]
if len(insts) > 1 {
inst = cue.Merge(insts...)
}
r := value.ConvertToRuntime(inst.Value().Context())
for _, b := range binst {
for _, i := range b.Imports {
if _, err := r.Build(i); err != nil {
return nil, err
}
}
}
// Set path equal to the package from which it is loading.
ti.ImportPath = binst[0].ImportPath
inst = inst.Build(ti)
return inst, inst.Err
}
func shortFile(root string, f *build.File) string {
dir, _ := filepath.Rel(root, f.Filename)
if dir == "" {
return f.Filename
}
if !filepath.IsAbs(dir) {
dir = "." + string(filepath.Separator) + dir
}
return dir
}