cue/cmd/cue: hoist placement logic from import
Change-Id: I81e5a9881f5883c38f77b72a454010dbb5f191d7
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/5026
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/cmd/cue/cmd/common.go b/cmd/cue/cmd/common.go
index 976f9a0..26a1f74 100644
--- a/cmd/cue/cmd/common.go
+++ b/cmd/cue/cmd/common.go
@@ -16,8 +16,11 @@
import (
"bytes"
+ "fmt"
"io"
"os"
+ "path/filepath"
+ "regexp"
"strings"
"testing"
@@ -120,9 +123,14 @@
// If orphanFiles are mixed with CUE files and/or if placement flags are used,
// the instance is also included in insts.
- orphanedData []*build.File
- orphanedSchema []*build.File
- orphanInstance *build.Instance
+ forceOrphanProcessing bool
+ orphanedData []*build.File
+ orphanedSchema []*build.File
+ orphanInstance *build.Instance
+ // imported files are files that were orphaned in the build instance, but
+ // were placed in the instance by using one the --files, --list or --path
+ // flags.
+ imported []*ast.File
expressions []ast.Expr // only evaluate these expressions within results
schema ast.Expr // selects schema in instance for orphaned values
@@ -138,7 +146,7 @@
return buildInstances(b.cmd, b.insts)
}
-func parseArgs(cmd *Command, args []string, cfg *load.Config) (*buildPlan, error) {
+func parseArgs(cmd *Command, args []string, cfg *load.Config) (p *buildPlan, err error) {
if cfg == nil {
cfg = defaultConfig
}
@@ -148,21 +156,30 @@
}
decorateInstances(cmd, flagTags.StringArray(cmd), builds)
- p := &buildPlan{cmd: cmd}
+ p = &buildPlan{cmd: cmd, forceOrphanProcessing: cfg.DataFiles}
if err := p.parseFlags(); err != nil {
return nil, err
}
for _, b := range builds {
+ var ok bool
+ if b.User || p.forceOrphanProcessing {
+ ok, err = p.placeOrphans(b)
+ if err != nil {
+ return nil, err
+ }
+ }
if !b.User {
p.insts = append(p.insts, b)
continue
}
-
if len(b.BuildFiles) > 0 {
p.insts = append(p.insts, b)
}
+ if ok {
+ continue
+ }
if len(b.OrphanedFiles) > 0 {
if p.orphanInstance != nil {
@@ -215,6 +232,100 @@
return nil
}
+func (b *buildPlan) placeOrphans(i *build.Instance) (ok bool, err error) {
+ var (
+ perFile = flagFiles.Bool(b.cmd)
+ useList = flagList.Bool(b.cmd)
+ path = flagPath.StringArray(b.cmd)
+ useContext = flagWithContext.Bool(b.cmd)
+ pkg = flagPackage.String(b.cmd)
+ match = flagGlob.String(b.cmd)
+ )
+ if !b.forceOrphanProcessing && !perFile && !useList && len(path) == 0 {
+ if useContext {
+ return false, fmt.Errorf(
+ "flag %q must be used with at least one of flag %q, %q, or %q",
+ flagWithContext, flagPath, flagList, flagFiles,
+ )
+ }
+ return false, err
+ }
+
+ if pkg == "" {
+ pkg = i.PkgName
+ } else if pkg != "" && i.PkgName != pkg && !flagForce.Bool(b.cmd) {
+ return false, fmt.Errorf(
+ "%q flag clashes with existing package name (%s vs %s)",
+ flagPackage, pkg, i.PkgName,
+ )
+ }
+
+ var files []*ast.File
+
+ re, err := regexp.Compile(match)
+ if err != nil {
+ return false, err
+ }
+
+ for _, f := range i.OrphanedFiles {
+ if !re.MatchString(filepath.Base(f.Filename)) {
+ return false, nil
+ }
+
+ d := encoding.NewDecoder(f, b.encConfig)
+ defer d.Close()
+
+ var objs []ast.Expr
+
+ for ; !d.Done(); d.Next() {
+ if expr := d.Expr(); expr != nil {
+ objs = append(objs, expr)
+ continue
+ }
+ f := d.File()
+ f.Filename = newName(d.Filename(), d.Index())
+ files = append(files, f)
+ }
+
+ if perFile {
+ for i, obj := range objs {
+ f, err := placeOrphans(b.cmd, d.Filename(), pkg, obj)
+ if err != nil {
+ return false, err
+ }
+ f.Filename = newName(d.Filename(), i)
+ files = append(files, f)
+ }
+ continue
+ }
+ if len(objs) > 1 && len(path) == 0 && useList {
+ return false, fmt.Errorf(
+ "%s, %s, or %s flag needed to handle multiple objects in file %s",
+ flagPath, flagList, flagFiles, f.Filename)
+ }
+
+ f, err := placeOrphans(b.cmd, d.Filename(), pkg, objs...)
+ if err != nil {
+ return false, err
+ }
+ f.Filename = newName(d.Filename(), 0)
+ files = append(files, f)
+ }
+
+ b.imported = append(b.imported, files...)
+ for _, f := range files {
+ if err := i.AddSyntax(f); err != nil {
+ return false, err
+ }
+ i.BuildFiles = append(i.BuildFiles, &build.File{
+ Filename: f.Filename,
+ Encoding: build.CUE,
+ Source: f,
+ })
+ }
+ return true, nil
+}
+
func (b *buildPlan) singleInstance() *cue.Instance {
var p *build.Instance
switch len(b.insts) {
@@ -241,8 +352,6 @@
exitIfErr(cmd, inst, inst.Err, true)
}
- decorateInstances(cmd, flagTags.StringArray(cmd), binst)
-
if flagIgnore.Bool(cmd) {
return instances
}
diff --git a/cmd/cue/cmd/import.go b/cmd/cue/cmd/import.go
index 015e43d..18a440a 100644
--- a/cmd/cue/cmd/import.go
+++ b/cmd/cue/cmd/import.go
@@ -16,31 +16,24 @@
import (
"fmt"
- "io"
"io/ioutil"
"os"
"path/filepath"
- "regexp"
"strconv"
"strings"
- "sync"
"unicode"
"github.com/spf13/cobra"
- "golang.org/x/sync/errgroup"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/ast/astutil"
- "cuelang.org/go/cue/build"
"cuelang.org/go/cue/format"
"cuelang.org/go/cue/literal"
"cuelang.org/go/cue/load"
"cuelang.org/go/cue/parser"
"cuelang.org/go/cue/token"
"cuelang.org/go/encoding/json"
- "cuelang.org/go/encoding/protobuf"
"cuelang.org/go/internal"
- "cuelang.org/go/internal/encoding"
"cuelang.org/go/internal/third_party/yaml"
)
@@ -231,7 +224,7 @@
cmd.Flags().BoolP(string(flagForce), "f", false, "force overwriting existing files")
cmd.Flags().Bool(string(flagDryrun), false, "only run simulation")
cmd.Flags().BoolP(string(flagRecursive), "R", false, "recursively parse string values")
- cmd.Flags().String("fix", "", "apply given fix")
+ cmd.Flags().String("fix", "", "apply given fix") // XXX
return cmd
}
@@ -245,131 +238,48 @@
// TODO: factor out rooting of orphaned files.
func runImport(cmd *Command, args []string) (err error) {
- var group errgroup.Group
-
- pkgFlag := flagPackage.String(cmd)
-
- done := map[string]bool{}
-
b, err := parseArgs(cmd, args, &load.Config{DataFiles: true})
if err != nil {
return err
}
- builds := b.insts
- if b.orphanInstance != nil {
- builds = append(builds, b.orphanInstance)
- }
- for _, pkg := range builds {
+
+ pkgFlag := flagPackage.String(cmd)
+ for _, pkg := range b.insts {
pkgName := pkgFlag
if pkgName == "" {
pkgName = pkg.PkgName
}
// TODO: allow if there is a unique package name.
- if pkgName == "" && len(builds) > 1 {
+ if pkgName == "" && len(b.insts) > 1 {
err = fmt.Errorf("must specify package name with the -p flag")
- break
- }
- if err = pkg.Err; err != nil {
- break
- }
- if done[pkg.Dir] {
- continue
- }
- done[pkg.Dir] = true
-
- for _, f := range pkg.OrphanedFiles {
- f := f // capture range var
- group.Go(func() error { return handleFile(b, pkgName, f) })
+ exitOnErr(cmd, err, true)
}
}
- err2 := group.Wait()
+ for _, f := range b.imported {
+ err := handleFile(b, f)
+ if err != nil {
+ return err
+ }
+ }
+
exitOnErr(cmd, err, true)
- exitOnErr(cmd, err2, true)
return nil
}
-func handleFile(b *buildPlan, pkg string, file *build.File) (err error) {
- filename := file.Filename
- // filter file names
- re, err := regexp.Compile(flagGlob.String(b.cmd))
- if err != nil {
- return err
- }
- if !re.MatchString(filepath.Base(filename)) {
- return nil
- }
-
- // TODO: consider unifying the two modes.
- var objs []ast.Expr
- i := encoding.NewDecoder(file, b.encConfig)
- defer i.Close()
- for ; !i.Done(); i.Next() {
- if expr := i.Expr(); expr != nil {
- objs = append(objs, expr)
- continue
- }
- if err := processFile(b.cmd, i.File()); err != nil {
- return err
- }
- }
-
- if len(objs) > 0 {
- if err := processStream(b.cmd, pkg, filename, objs); err != nil {
- return err
- }
- }
- return i.Err()
-}
-
-func processFile(cmd *Command, file *ast.File) (err error) {
- name := file.Filename + ".cue"
-
- b, err := format.Node(file)
- if err != nil {
- return err
- }
-
- return ioutil.WriteFile(name, b, 0644)
-}
-
-func processStream(cmd *Command, pkg, filename string, objs []ast.Expr) error {
- if flagFiles.Bool(cmd) {
- for i, f := range objs {
- err := combineExpressions(cmd, pkg, filename, i, f)
- if err != nil {
- return err
- }
- }
- return nil
- } else if len(objs) > 1 {
- if !flagList.Bool(cmd) && len(flagPath.StringArray(cmd)) == 0 && !flagFiles.Bool(cmd) {
- return fmt.Errorf("list, flag, or files flag needed to handle multiple objects in file %q", filename)
- }
- }
- return combineExpressions(cmd, pkg, filename, 0, objs...)
-}
-
-// TODO: implement a more fine-grained approach.
-var mutex sync.Mutex
-
-func combineExpressions(cmd *Command, pkg, filename string, idx int, objs ...ast.Expr) error {
- cueFile := newName(filename, idx)
-
- mutex.Lock()
- defer mutex.Unlock()
-
- if out := flagOut.String(cmd); out != "" {
+func handleFile(b *buildPlan, f *ast.File) (err error) {
+ cueFile := f.Filename
+ if out := flagOut.String(b.cmd); out != "" {
cueFile = out
}
if cueFile != "-" {
switch _, err := os.Stat(cueFile); {
case os.IsNotExist(err):
case err == nil:
- if !flagForce.Bool(cmd) {
+ if !flagForce.Bool(b.cmd) {
// TODO: mimic old behavior: write to stderr, but do not exit
// with error code. Consider what is best to do here.
- stderr := cmd.Command.OutOrStderr()
+ stderr := b.cmd.Command.OutOrStderr()
fmt.Fprintf(stderr, "skipping file %q: already exists\n", cueFile)
return nil
}
@@ -378,16 +288,15 @@
}
}
- f, err := placeOrphans(cmd, filename, pkg, objs)
- if err != nil {
- return err
- }
-
- if flagRecursive.Bool(cmd) {
+ if flagRecursive.Bool(b.cmd) {
h := hoister{fields: map[string]bool{}}
h.hoist(f)
}
+ return writeFile(b.cmd, f, cueFile)
+}
+
+func writeFile(cmd *Command, f *ast.File, cueFile string) error {
b, err := format.Node(f, format.Simplify())
if err != nil {
return fmt.Errorf("error formatting file: %v", err)
@@ -400,7 +309,7 @@
return ioutil.WriteFile(cueFile, b, 0644)
}
-func placeOrphans(cmd *Command, filename, pkg string, objs []ast.Expr) (*ast.File, error) {
+func placeOrphans(cmd *Command, filename, pkg string, objs ...ast.Expr) (*ast.File, error) {
f := &ast.File{}
index := newIndex()
@@ -534,10 +443,6 @@
return filename
}
-func handleProtoDef(cmd *Command, path string, r io.Reader) (f *ast.File, err error) {
- return protobuf.Extract(path, r, &protobuf.Config{Paths: flagProtoPath.StringArray(cmd)})
-}
-
type hoister struct {
fields map[string]bool
}
