cue/load: use build.File to support "-"
Added test to test high-level usage for the eval command.
Change-Id: I00541817ef1a6a30580d27a0c2ccd3685602e846
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/5223
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/cmd/cue/cmd/common.go b/cmd/cue/cmd/common.go
index b1abf47..7e0a220 100644
--- a/cmd/cue/cmd/common.go
+++ b/cmd/cue/cmd/common.go
@@ -301,6 +301,8 @@
if cfg == nil {
cfg = defaultConfig
}
+ cfg.Stdin = stdin
+
builds := loadFromArgs(cmd, args, cfg)
if builds == nil {
return nil, errors.Newf(token.NoPos, "invalid args")
diff --git a/cmd/cue/cmd/testdata/script/eval_stdin.txt b/cmd/cue/cmd/testdata/script/eval_stdin.txt
new file mode 100644
index 0000000..18d07eb
--- /dev/null
+++ b/cmd/cue/cmd/testdata/script/eval_stdin.txt
@@ -0,0 +1,12 @@
+stdin stdin.cue
+cue eval t.cue -
+cmp stdout expect-stdout
+
+-- stdin.cue --
+foo:3
+-- t.cue --
+foo: int
+bar: 3
+-- expect-stdout --
+foo: 3
+bar: 3
diff --git a/cmd/cue/cmd/testdata/script/issue174.txt b/cmd/cue/cmd/testdata/script/issue174.txt
new file mode 100644
index 0000000..a611481
--- /dev/null
+++ b/cmd/cue/cmd/testdata/script/issue174.txt
@@ -0,0 +1,8 @@
+! cue export ./issue174
+cmp stderr expect-stderr
+-- expect-stderr --
+build constraints exclude all CUE files in ./issue174 (ignored: issue174/issue174.cue)
+-- issue174/issue174.cue --
+import 'foo'
+
+a: 1
\ No newline at end of file
diff --git a/cue/load/config.go b/cue/load/config.go
index 51b1faf..95d883e 100644
--- a/cue/load/config.go
+++ b/cue/load/config.go
@@ -15,6 +15,7 @@
package load
import (
+ "io"
"os"
pathpkg "path"
"path/filepath"
@@ -178,17 +179,25 @@
// If the file with the given path already exists, the parser will use the
// alternative file contents provided by the map.
//
- // Overlays provide incomplete support for when a given file doesn't
- // already exist on disk. See the package doc above for more details.
- //
// If the value must be of type string, []byte, io.Reader, or *ast.File.
Overlay map[string]Source
+ // Stdin defines an alternative for os.Stdin for the file "-". When used,
+ // the corresponding build.File will be associated with the full buffer.
+ Stdin io.Reader
+
fileSystem
loadFunc build.LoadFunc
}
+func (c *Config) stdin() io.Reader {
+ if c.Stdin == nil {
+ return os.Stdin
+ }
+ return c.Stdin
+}
+
func (c *Config) newInstance(pos token.Pos, p importPath) *build.Instance {
dir, name, err := c.absDirFromImportPath(pos, p)
i := c.Context.NewInstance(dir, c.loadFunc)
diff --git a/cue/load/fs.go b/cue/load/fs.go
index 082efbb..b269f00 100644
--- a/cue/load/fs.go
+++ b/cue/load/fs.go
@@ -16,7 +16,6 @@
import (
"bytes"
- "go/ast"
"io"
"io/ioutil"
"os"
@@ -25,6 +24,7 @@
"strings"
"time"
+ "cuelang.org/go/cue/ast"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/token"
)
@@ -73,13 +73,14 @@
dir, base := filepath.Split(filename)
m := fs.getDir(dir, true)
- b, err := src.contents()
+ b, file, err := src.contents()
if err != nil {
return err
}
m[base] = &overlayFile{
basename: base,
contents: b,
+ file: file,
modtime: time.Now(),
}
diff --git a/cue/load/import.go b/cue/load/import.go
index 3839ad7..ab045ce 100644
--- a/cue/load/import.go
+++ b/cue/load/import.go
@@ -238,27 +238,27 @@
func rewriteFiles(p *build.Instance, root string, isLocal bool) {
p.Root = root
- for i, path := range p.CUEFiles {
- p.CUEFiles[i] = normPrefix(root, path, isLocal)
- sortParentsFirst(p.CUEFiles)
- }
- for i, path := range p.TestCUEFiles {
- p.TestCUEFiles[i] = normPrefix(root, path, isLocal)
- sortParentsFirst(p.TestCUEFiles)
- }
- for i, path := range p.ToolCUEFiles {
- p.ToolCUEFiles[i] = normPrefix(root, path, isLocal)
- sortParentsFirst(p.ToolCUEFiles)
- }
- for i, path := range p.IgnoredCUEFiles {
+
+ normalizeFilenames(root, p.CUEFiles, isLocal)
+ normalizeFilenames(root, p.TestCUEFiles, isLocal)
+ normalizeFilenames(root, p.ToolCUEFiles, isLocal)
+ normalizeFilenames(root, p.IgnoredCUEFiles, isLocal)
+ normalizeFilenames(root, p.InvalidCUEFiles, isLocal)
+
+ normalizeFiles(p.BuildFiles)
+ normalizeFiles(p.IgnoredFiles)
+ normalizeFiles(p.OrphanedFiles)
+ normalizeFiles(p.InvalidFiles)
+ normalizeFiles(p.UnknownFiles)
+}
+
+func normalizeFilenames(root string, a []string, isLocal bool) {
+ for i, path := range a {
if strings.HasPrefix(path, root) {
- p.IgnoredCUEFiles[i] = normPrefix(root, path, isLocal)
+ a[i] = normPrefix(root, path, isLocal)
}
}
- for i, path := range p.InvalidCUEFiles {
- p.InvalidCUEFiles[i] = normPrefix(root, path, isLocal)
- sortParentsFirst(p.InvalidCUEFiles)
- }
+ sortParentsFirst(a)
}
func sortParentsFirst(s []string) {
@@ -267,6 +267,12 @@
})
}
+func normalizeFiles(a []*build.File) {
+ sort.Slice(a, func(i, j int) bool {
+ return len(filepath.Dir(a[i].Filename)) < len(filepath.Dir(a[j].Filename))
+ })
+}
+
type fileProcessor struct {
firstFile string
firstCommentFile string
@@ -322,15 +328,15 @@
}
func (fp *fileProcessor) add(pos token.Pos, root string, file *build.File, mode importMode) (added bool) {
- path := file.Filename
- fullPath := path
- if !filepath.IsAbs(path) {
- fullPath = filepath.Join(root, path)
+ fullPath := file.Filename
+ if fullPath != "-" {
+ if !filepath.IsAbs(fullPath) {
+ fullPath = filepath.Join(root, fullPath)
+ }
}
file.Filename = fullPath
- name := filepath.Base(fullPath)
- dir := filepath.Dir(fullPath)
+ base := filepath.Base(fullPath)
p := fp.pkg
@@ -341,7 +347,7 @@
return true
}
- match, data, filename, err := matchFile(fp.c, dir, name, true, fp.allFiles, fp.allTags)
+ match, data, err := matchFile(fp.c, file, true, fp.allFiles, fp.allTags)
if err != nil {
return badFile(err)
}
@@ -356,7 +362,7 @@
return false // don't mark as added
}
- pf, perr := parser.ParseFile(filename, data, parser.ImportsOnly, parser.ParseComments)
+ pf, perr := parser.ParseFile(fullPath, data, parser.ImportsOnly, parser.ParseComments)
if perr != nil {
badFile(errors.Promote(perr, "add failed"))
return true
@@ -371,7 +377,7 @@
if p.PkgName == "" {
p.PkgName = pkg
- fp.firstFile = name
+ fp.firstFile = base
} else if pkg != p.PkgName {
if fp.ignoreOther {
p.IgnoredCUEFiles = append(p.IgnoredCUEFiles, fullPath)
@@ -381,24 +387,24 @@
return badFile(&MultiplePackageError{
Dir: p.Dir,
Packages: []string{p.PkgName, pkg},
- Files: []string{fp.firstFile, name},
+ Files: []string{fp.firstFile, base},
})
}
- isTest := strings.HasSuffix(name, "_test"+cueSuffix)
- isTool := strings.HasSuffix(name, "_tool"+cueSuffix)
+ isTest := strings.HasSuffix(base, "_test"+cueSuffix)
+ isTool := strings.HasSuffix(base, "_tool"+cueSuffix)
if mode&importComment != 0 {
qcom, line := findimportComment(data)
if line != 0 {
com, err := strconv.Unquote(qcom)
if err != nil {
- badFile(errors.Newf(pos, "%s:%d: cannot parse import comment", filename, line))
+ badFile(errors.Newf(pos, "%s:%d: cannot parse import comment", fullPath, line))
} else if p.ImportComment == "" {
p.ImportComment = com
- fp.firstCommentFile = name
+ fp.firstCommentFile = base
} else if p.ImportComment != com {
- badFile(errors.Newf(pos, "found import comments %q (%s) and %q (%s) in %s", p.ImportComment, fp.firstCommentFile, com, name, p.Dir))
+ badFile(errors.Newf(pos, "found import comments %q (%s) and %q (%s) in %s", p.ImportComment, fp.firstCommentFile, com, base, p.Dir))
}
}
}
@@ -414,7 +420,7 @@
if err != nil {
badFile(errors.Newf(
spec.Path.Pos(),
- "%s: parser returned invalid quoted string: <%s>", filename, quoted,
+ "%s: parser returned invalid quoted string: <%s>", fullPath, quoted,
))
}
if !isTest || fp.c.Tests {
diff --git a/cue/load/loader.go b/cue/load/loader.go
index 7ac0d01..74a9daa 100644
--- a/cue/load/loader.go
+++ b/cue/load/loader.go
@@ -29,6 +29,7 @@
"cuelang.org/go/cue/build"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/token"
+ "cuelang.org/go/internal/encoding"
"cuelang.org/go/internal/filetypes"
"golang.org/x/xerrors"
)
@@ -149,39 +150,37 @@
// ModInit() // TODO: support modules
pkg := l.cfg.Context.NewInstance(cfg.Dir, l.loadFunc())
+ _, err := filepath.Abs(cfg.Dir)
+ if err != nil {
+ return cfg.newErrInstance(pos, toImportPath(cfg.Dir),
+ errors.Wrapf(err, pos, "could not convert '%s' to absolute path", cfg.Dir))
+ }
+
for _, bf := range files {
f := bf.Filename
- if cfg.isDir(f) {
+ if f == "-" {
+ continue
+ }
+ if !filepath.IsAbs(f) {
+ f = filepath.Join(cfg.Dir, f)
+ }
+ fi, err := cfg.fileSystem.stat(f)
+ if err != nil {
+ return cfg.newErrInstance(pos, toImportPath(f),
+ errors.Wrapf(err, pos, "could not find file"))
+ }
+ if fi.IsDir() {
return cfg.newErrInstance(token.NoPos, toImportPath(f),
errors.Newf(pos, "file is a directory %v", f))
}
}
- // TODO: add fields directly?
fp := newFileProcessor(cfg, pkg)
for _, file := range files {
- path := file.Filename
- if !filepath.IsAbs(path) {
- path = filepath.Join(cfg.Dir, path)
- }
- fi, err := cfg.fileSystem.stat(path)
- if err != nil {
- return cfg.newErrInstance(pos, toImportPath(path),
- errors.Wrapf(err, pos, "could not find dir %s", path))
- }
- if fi.IsDir() {
- return cfg.newErrInstance(pos, toImportPath(path),
- errors.Newf(pos, "%s is a directory, should be a CUE file", file.Filename))
- }
fp.add(pos, cfg.Dir, file, allowAnonymous)
}
// TODO: ModImportFromFiles(files)
- _, err := filepath.Abs(cfg.Dir)
- if err != nil {
- return cfg.newErrInstance(pos, toImportPath(cfg.Dir),
- errors.Wrapf(err, pos, "could convert '%s' to absolute path", cfg.Dir))
- }
pkg.Dir = cfg.Dir
rewriteFiles(pkg, pkg.Dir, true)
for _, err := range errors.Errors(fp.finalize()) { // ImportDir(&ctxt, dir, 0)
@@ -211,19 +210,15 @@
}
func (l *loader) addFiles(dir string, p *build.Instance) {
- files := p.CUEFiles
- fs := &l.cfg.fileSystem
-
- for _, f := range files {
- if !fs.isAbsPath(f) {
- f = fs.joinPath(dir, f)
+ for _, f := range p.BuildFiles {
+ d := encoding.NewDecoder(f, &encoding.Config{Stdin: l.cfg.stdin()})
+ for ; !d.Done(); d.Next() {
+ _ = p.AddSyntax(d.File())
}
- r, err := fs.openFile(f)
- if err != nil {
- p.ReportError(err)
+ if err := d.Err(); err != nil {
+ p.ReportError(errors.Promote(err, "load"))
}
-
- _ = p.AddFile(f, r)
+ d.Close()
}
}
diff --git a/cue/load/loader_test.go b/cue/load/loader_test.go
index b0090d8..524333a 100644
--- a/cue/load/loader_test.go
+++ b/cue/load/loader_test.go
@@ -154,7 +154,7 @@
cfg: dirCfg,
args: args("example.org/test/hello:nonexist"),
want: `
-err: build constraints exclude all CUE files in example.org/test/hello:nonexist (ignored: hello/test.cue, anon.cue, test.cue)
+err: build constraints exclude all CUE files in example.org/test/hello:nonexist (ignored: anon.cue, test.cue, hello/test.cue)
path: example.org/test/hello:nonexist
module: example.org/test
root: $CWD/testdata
@@ -183,7 +183,18 @@
dir: $CWD/testdata
display:command-line-arguments
files:
- $CWD/testdata/anon.cue`,
+ $CWD/testdata/anon.cue`,
+ }, {
+ cfg: dirCfg,
+ args: args("-"),
+ want: `
+path: ""
+module: ""
+root: $CWD/testdata
+dir: $CWD/testdata
+display:command-line-arguments
+files:
+ -`,
}, {
// NOTE: dir should probably be set to $CWD/testdata, but either way.
cfg: dirCfg,
diff --git a/cue/load/match.go b/cue/load/match.go
index cb61e07..e357996 100644
--- a/cue/load/match.go
+++ b/cue/load/match.go
@@ -16,12 +16,15 @@
import (
"bytes"
- "path"
+ "io/ioutil"
+ "path/filepath"
"strings"
"unicode"
+ "cuelang.org/go/cue/build"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/token"
+ "cuelang.org/go/internal/filetypes"
)
// matchFileTest reports whether the file with the given name in the given directory
@@ -31,7 +34,11 @@
// matchFileTest considers the name of the file and may use cfg.Build.OpenFile to
// read some or all of the file's content.
func matchFileTest(cfg *Config, dir, name string) (match bool, err error) {
- match, _, _, err = matchFile(cfg, dir, name, false, false, nil)
+ file, err := filetypes.ParseFile(filepath.Join(dir, name), filetypes.Input)
+ if err != nil {
+ return false, nil
+ }
+ match, _, err = matchFile(cfg, file, false, false, nil)
return
}
@@ -43,38 +50,49 @@
// considers text until the first non-comment.
// If allTags is non-nil, matchFile records any encountered build tag
// by setting allTags[tag] = true.
-func matchFile(cfg *Config, dir, name string, returnImports, allFiles bool, allTags map[string]bool) (match bool, data []byte, filename string, err errors.Error) {
- if strings.HasPrefix(name, "_") {
+func matchFile(cfg *Config, file *build.File, returnImports, allFiles bool, allTags map[string]bool) (match bool, data []byte, err errors.Error) {
+ if fi := cfg.fileSystem.getOverlay(file.Filename); fi != nil {
+ if fi.file != nil {
+ file.Source = fi.file
+ } else {
+ file.Source = fi.contents
+ }
+ }
+
+ if file.Encoding != build.CUE {
return
}
+
+ if file.Filename == "-" {
+ b, err2 := ioutil.ReadAll(cfg.stdin())
+ if err2 != nil {
+ err = errors.Newf(token.NoPos, "read stdin: %v", err)
+ return
+ }
+ file.Source = b
+ data = b
+ match = true // don't check shouldBuild for stdin
+ return
+ }
+
+ name := filepath.Base(file.Filename)
if !cfg.filesMode && strings.HasPrefix(name, ".") {
return
}
- ext := path.Ext(name)
-
- switch ext {
- case cueSuffix:
- // tentatively okay - read to make sure
- default:
- // skip
+ if strings.HasPrefix(name, "_") {
return
}
- filename = cfg.fileSystem.joinPath(dir, name)
- f, err := cfg.fileSystem.openFile(filename)
+ f, err := cfg.fileSystem.openFile(file.Filename)
if err != nil {
return
}
- if strings.HasSuffix(filename, cueSuffix) {
- data, err = readImports(f, false, nil)
- } else {
- data, err = readComments(f)
- }
+ data, err = readImports(f, false, nil)
f.Close()
if err != nil {
- err = errors.Newf(token.NoPos, "read %s: %v", filename, err)
+ err = errors.Newf(token.NoPos, "read %s: %v", file.Filename, err)
return
}
diff --git a/cue/load/source.go b/cue/load/source.go
index 9514f2a..0485585 100644
--- a/cue/load/source.go
+++ b/cue/load/source.go
@@ -21,7 +21,7 @@
// A Source represents file contents.
type Source interface {
- contents() ([]byte, error)
+ contents() ([]byte, *ast.File, error)
}
// FromString creates a Source from the given string.
@@ -43,18 +43,21 @@
type stringSource string
-func (s stringSource) contents() ([]byte, error) {
- return []byte(s), nil
+func (s stringSource) contents() ([]byte, *ast.File, error) {
+ return []byte(s), nil, nil
}
type bytesSource []byte
-func (s bytesSource) contents() ([]byte, error) {
- return []byte(s), nil
+func (s bytesSource) contents() ([]byte, *ast.File, error) {
+ return []byte(s), nil, nil
}
type fileSource ast.File
-func (s *fileSource) contents() ([]byte, error) {
- return format.Node((*ast.File)(s))
+func (s *fileSource) contents() ([]byte, *ast.File, error) {
+ f := (*ast.File)(s)
+ // TODO: wasteful formatting, but needed for now.
+ b, err := format.Node(f)
+ return b, f, err
}
diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go
index 22871ea..178d506 100644
--- a/internal/encoding/encoding.go
+++ b/internal/encoding/encoding.go
@@ -115,18 +115,24 @@
// type of f must be a data type, but does not have to be an encoding that
// can stream. stdin is used in case the file is "-".
func NewDecoder(f *build.File, cfg *Config) *Decoder {
- r, err := reader(f, cfg.Stdin)
- i := &Decoder{
- closer: r,
- err: err,
- filename: f.Filename,
- next: func() (ast.Expr, error) {
- if err == nil {
- err = io.EOF
- }
- return nil, io.EOF
- },
+ i := &Decoder{filename: f.Filename}
+ i.next = func() (ast.Expr, error) {
+ if i.err != nil {
+ return nil, i.err
+ }
+ return nil, io.EOF
}
+
+ if f, ok := f.Source.(*ast.File); ok {
+ i.file = f
+ i.closer = ioutil.NopCloser(strings.NewReader(""))
+ // TODO: verify input format for CUE.
+ return i
+ }
+
+ r, err := reader(f, cfg.Stdin)
+ i.closer = r
+ i.err = err
if err != nil {
return i
}
