cmd/cue/cmd/import.go - cue - Git at Google

 // Copyright 2018 The CUE Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package cmd

 import (
 	"fmt"
 	"io/ioutil"
 	"os"
 	"strings"
 	"unicode"

 	"github.com/spf13/cobra"

 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/ast/astutil"
 	"cuelang.org/go/cue/format"
 	"cuelang.org/go/cue/literal"
 	"cuelang.org/go/cue/load"
 	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/encoding/json"
 	"cuelang.org/go/internal/third_party/yaml"
 )

 func newImportCmd(c *Command) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "import",
 		Short: "convert other data formats to CUE files",
 		Long: `import converts other data formats, like JSON and YAML to CUE files

 The following file formats are currently supported:

   Format       Extensions
 	JSON       .json .jsonl .ndjson
 	YAML       .yaml .yml
 	protobuf   .proto

 Files can either be specified explicitly, or inferred from the
 specified packages. In either case, the file extension is
 replaced with .cue. It will fail if the file already exists by
 default. The -f flag overrides this.

 Examples:

   # Convert individual files:
   $ cue import foo.json bar.json  # create foo.yaml and bar.yaml

   # Convert all json files in the indicated directories:
   $ cue import ./... -type=json

 The "flags" help topic describes how to assign values to a
 specific path within a CUE namespace. Some examples of that

 Examples:

   $ cat <<EOF > foo.yaml
   kind: Service
   name: booster
   EOF

   # include the parsed file as an emit value:
   $ cue import foo.yaml
   $ cat foo.cue
   {
       kind: Service
       name: booster
   }

   # include the parsed file at the root of the CUE file:
   $ cue import -f foo.yaml
   $ cat foo.cue
   kind: Service
   name: booster

   # include the import config at the mystuff path
   $ cue import -f -l '"mystuff"' foo.yaml
   $ cat foo.cue
   myStuff: {
       kind: Service
       name: booster
   }

   # append another object to the input file
   $ cat <<EOF >> foo.yaml
   ---
   kind: Deployment
   name: booster
   replicas: 1

   # base the path values on the input
   $ cue import -f -l 'strings.ToLower(kind)' -l name foo.yaml
   $ cat foo.cue
   service: booster: {
       kind: "Service"
       name: "booster"
   }

   # base the path values on the input and file name
   $ cue import -f --with-context -l 'path.Base(filename)' -l data.kind foo.yaml
   $ cat foo.cue
   "foo.yaml": Service: {
       kind: "Service"
       name: "booster"
   }

   "foo.yaml": Deployment: {
       kind:     "Deployment"
       name:     "booster
       replicas: 1
   }

   # include all files as list elements
   $ cue import -f -list -foo.yaml
   $ cat foo.cue
   [{
       kind: "Service"
       name: "booster"
   }, {
       kind:     "Deployment"
       name:     "booster
       replicas: 1
   }]

   # collate files with the same path into a list
   $ cue import -f -list -l 'strings.ToLower(kind)' foo.yaml
   $ cat foo.cue
   service: [{
       kind: "Service"
       name: "booster"
   }
   deployment: [{
       kind:     "Deployment"
       name:     "booster
       replicas: 1
   }]


 Embedded data files

 The --recursive or -R flag enables the parsing of fields that are string
 representations of data formats themselves. A field that can be parsed is
 replaced with a call encoding the data from a structured form that is placed
 in a sibling field.

 It is also possible to recursively hoist data formats:

 Example:
   $ cat <<EOF > example.json
   "a": {
       "data": '{ "foo": 1, "bar": 2 }',
   }
   EOF

   $ cue import -R example.json
   $ cat example.cue
   import "encoding/json"

   a: {
       data: json.Encode(_data),
       _data = {
           foo: 1
           bar: 2
       }
   }
 `,
 		RunE: mkRunE(c, runImport),
 	}

 	addOutFlags(cmd.Flags(), false)
 	addOrphanFlags(cmd.Flags())

 	cmd.Flags().Bool(string(flagFiles), false, "split multiple entries into different files")
 	cmd.Flags().String(string(flagType), "", "only apply to files of this type")
 	cmd.Flags().BoolP(string(flagForce), "f", false, "force overwriting existing files")
 	cmd.Flags().Bool(string(flagDryrun), false, "only run simulation")
 	cmd.Flags().BoolP(string(flagRecursive), "R", false, "recursively parse string values")

 	return cmd
 }

 const (
 	flagFiles       flagName = "files"
 	flagProtoPath   flagName = "proto_path"
 	flagWithContext flagName = "with-context"
 )

 // TODO: factor out rooting of orphaned files.

 func runImport(cmd *Command, args []string) (err error) {
 	b, err := parseArgs(cmd, args, &load.Config{DataFiles: true})
 	if err != nil {
 		return err
 	}

 	pkgFlag := flagPackage.String(cmd)
 	for _, pkg := range b.insts {
 		pkgName := pkgFlag
 		if pkgName == "" {
 			pkgName = pkg.PkgName
 		}
 		// TODO: allow if there is a unique package name.
 		if pkgName == "" && len(b.insts) > 1 {
 			err = fmt.Errorf("must specify package name with the -p flag")
 			exitOnErr(cmd, err, true)
 		}
 	}

 	for _, f := range b.imported {
 		err := handleFile(b, f)
 		if err != nil {
 			return err
 		}
 	}

 	exitOnErr(cmd, err, true)
 	return nil
 }

 func handleFile(b *buildPlan, f *ast.File) (err error) {
 	cueFile := f.Filename
 	if out := flagOutFile.String(b.cmd); out != "" {
 		cueFile = out
 	}
 	if cueFile != "-" {
 		switch _, err := os.Stat(cueFile); {
 		case os.IsNotExist(err):
 		case err == nil:
 			if !flagForce.Bool(b.cmd) {
 				// TODO: mimic old behavior: write to stderr, but do not exit
 				// with error code. Consider what is best to do here.
 				stderr := b.cmd.Command.OutOrStderr()
 				fmt.Fprintf(stderr, "skipping file %q: already exists\n", cueFile)
 				return nil
 			}
 		default:
 			return fmt.Errorf("error creating file: %v", err)
 		}
 	}

 	if flagRecursive.Bool(b.cmd) {
 		h := hoister{fields: map[string]bool{}}
 		h.hoist(f)
 	}

 	return writeFile(b.cmd, f, cueFile)
 }

 func writeFile(cmd *Command, f *ast.File, cueFile string) error {
 	b, err := format.Node(f, format.Simplify())
 	if err != nil {
 		return fmt.Errorf("error formatting file: %v", err)
 	}

 	if cueFile == "-" {
 		_, err := cmd.OutOrStdout().Write(b)
 		return err
 	}
 	return ioutil.WriteFile(cueFile, b, 0644)
 }

 type hoister struct {
 	fields map[string]bool
 }

 func (h *hoister) hoist(f *ast.File) {
 	ast.Walk(f, nil, func(n ast.Node) {
 		name := ""
 		switch x := n.(type) {
 		case *ast.Field:
 			name, _, _ = ast.LabelName(x.Label)
 		case *ast.Alias:
 			name = x.Ident.Name
 		}
 		if name != "" {
 			h.fields[name] = true
 		}
 	})

 	_ = astutil.Apply(f, func(c astutil.Cursor) bool {
 		n := c.Node()
 		switch n.(type) {
 		case *ast.Comprehension:
 			return false
 		}
 		return true

 	}, func(c astutil.Cursor) bool {
 		switch f := c.Node().(type) {
 		case *ast.Field:
 			name, _, _ := ast.LabelName(f.Label)
 			if name == "" {
 				return false
 			}

 			lit, ok := f.Value.(*ast.BasicLit)
 			if !ok || lit.Kind != token.STRING {
 				return false
 			}

 			str, err := literal.Unquote(lit.Value)
 			if err != nil {
 				return false
 			}

 			expr, enc := tryParse(str)
 			if expr == nil {
 				return false
 			}

 			pkg := c.Import("encoding/" + enc)
 			if pkg == nil {
 				return false
 			}

 			// found a replacable string
 			dataField := h.uniqueName(name, "_", "cue_")

 			f.Value = ast.NewCall(
 				ast.NewSel(pkg, "Marshal"),
 				ast.NewIdent(dataField))

 			// TODO: use definitions instead
 			c.InsertAfter(astutil.ApplyRecursively(&ast.Alias{
 				Ident: ast.NewIdent(dataField),
 				Expr:  expr,
 			}))
 		}
 		return true
 	})
 }

 func tryParse(str string) (s ast.Expr, pkg string) {
 	b := []byte(str)
 	if json.Valid(b) {
 		expr, err := parser.ParseExpr("", b)
 		if err != nil {
 			// TODO: report error
 			return nil, ""
 		}
 		switch expr.(type) {
 		case *ast.StructLit, *ast.ListLit:
 		default:
 			return nil, ""
 		}
 		return expr, "json"
 	}

 	if expr, err := yaml.Unmarshal("", b); err == nil {
 		switch expr.(type) {
 		case *ast.StructLit, *ast.ListLit:
 		default:
 			return nil, ""
 		}
 		return expr, "yaml"
 	}

 	return nil, ""
 }

 func (h *hoister) uniqueName(base, prefix, typ string) string {
 	base = strings.Map(func(r rune) rune {
 		if unicode.In(r, unicode.L, unicode.N) {
 			return r
 		}
 		return '_'
 	}, base)

 	name := prefix + typ + base
 	for {
 		if !h.fields[name] {
 			h.fields[name] = true
 			return name
 		}
 		name = prefix + typ + base
 		typ += "x"
 	}
 }
	// Copyright 2018 The CUE Authors
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package cmd

	import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"unicode"

	"github.com/spf13/cobra"

	"cuelang.org/go/cue/ast"
	"cuelang.org/go/cue/ast/astutil"
	"cuelang.org/go/cue/format"
	"cuelang.org/go/cue/literal"
	"cuelang.org/go/cue/load"
	"cuelang.org/go/cue/parser"
	"cuelang.org/go/cue/token"
	"cuelang.org/go/encoding/json"
	"cuelang.org/go/internal/third_party/yaml"
	)

	func newImportCmd(c Command) cobra.Command {
	cmd := &cobra.Command{
	Use: "import",
	Short: "convert other data formats to CUE files",
	Long: `import converts other data formats, like JSON and YAML to CUE files

	The following file formats are currently supported:

	Format Extensions
	JSON .json .jsonl .ndjson
	YAML .yaml .yml
	protobuf .proto

	Files can either be specified explicitly, or inferred from the
	specified packages. In either case, the file extension is
	replaced with .cue. It will fail if the file already exists by
	default. The -f flag overrides this.

	Examples:

	# Convert individual files:
	$ cue import foo.json bar.json # create foo.yaml and bar.yaml

	# Convert all json files in the indicated directories:
	$ cue import ./... -type=json

	The "flags" help topic describes how to assign values to a
	specific path within a CUE namespace. Some examples of that

	Examples:

	$ cat <<EOF > foo.yaml
	kind: Service
	name: booster
	EOF

	# include the parsed file as an emit value:
	$ cue import foo.yaml
	$ cat foo.cue
	{
	kind: Service
	name: booster
	}

	# include the parsed file at the root of the CUE file:
	$ cue import -f foo.yaml
	$ cat foo.cue
	kind: Service
	name: booster

	# include the import config at the mystuff path
	$ cue import -f -l '"mystuff"' foo.yaml
	$ cat foo.cue
	myStuff: {
	kind: Service
	name: booster
	}

	# append another object to the input file
	$ cat <<EOF >> foo.yaml
	---
	kind: Deployment
	name: booster
	replicas: 1

	# base the path values on the input
	$ cue import -f -l 'strings.ToLower(kind)' -l name foo.yaml
	$ cat foo.cue
	service: booster: {
	kind: "Service"
	name: "booster"
	}

	# base the path values on the input and file name
	$ cue import -f --with-context -l 'path.Base(filename)' -l data.kind foo.yaml
	$ cat foo.cue
	"foo.yaml": Service: {
	kind: "Service"
	name: "booster"
	}

	"foo.yaml": Deployment: {
	kind: "Deployment"
	name: "booster
	replicas: 1
	}

	# include all files as list elements
	$ cue import -f -list -foo.yaml
	$ cat foo.cue
	[{
	kind: "Service"
	name: "booster"
	}, {
	kind: "Deployment"
	name: "booster
	replicas: 1
	}]

	# collate files with the same path into a list
	$ cue import -f -list -l 'strings.ToLower(kind)' foo.yaml
	$ cat foo.cue
	service: [{
	kind: "Service"
	name: "booster"
	}
	deployment: [{
	kind: "Deployment"
	name: "booster
	replicas: 1
	}]


	Embedded data files

	The --recursive or -R flag enables the parsing of fields that are string
	representations of data formats themselves. A field that can be parsed is
	replaced with a call encoding the data from a structured form that is placed
	in a sibling field.

	It is also possible to recursively hoist data formats:

	Example:
	$ cat <<EOF > example.json
	"a": {
	"data": '{ "foo": 1, "bar": 2 }',
	}
	EOF

	$ cue import -R example.json
	$ cat example.cue
	import "encoding/json"

	a: {
	data: json.Encode(_data),
	_data = {
	foo: 1
	bar: 2
	}
	}
	`,
	RunE: mkRunE(c, runImport),
	}

	addOutFlags(cmd.Flags(), false)
	addOrphanFlags(cmd.Flags())

	cmd.Flags().Bool(string(flagFiles), false, "split multiple entries into different files")
	cmd.Flags().String(string(flagType), "", "only apply to files of this type")
	cmd.Flags().BoolP(string(flagForce), "f", false, "force overwriting existing files")
	cmd.Flags().Bool(string(flagDryrun), false, "only run simulation")
	cmd.Flags().BoolP(string(flagRecursive), "R", false, "recursively parse string values")

	return cmd
	}

	const (
	flagFiles flagName = "files"
	flagProtoPath flagName = "proto_path"
	flagWithContext flagName = "with-context"
	)

	// TODO: factor out rooting of orphaned files.

	func runImport(cmd *Command, args []string) (err error) {
	b, err := parseArgs(cmd, args, &load.Config{DataFiles: true})
	if err != nil {
	return err
	}

	pkgFlag := flagPackage.String(cmd)
	for _, pkg := range b.insts {
	pkgName := pkgFlag
	if pkgName == "" {
	pkgName = pkg.PkgName
	}
	// TODO: allow if there is a unique package name.
	if pkgName == "" && len(b.insts) > 1 {
	err = fmt.Errorf("must specify package name with the -p flag")
	exitOnErr(cmd, err, true)
	}
	}

	for _, f := range b.imported {
	err := handleFile(b, f)
	if err != nil {
	return err
	}
	}

	exitOnErr(cmd, err, true)
	return nil
	}

	func handleFile(b buildPlan, f ast.File) (err error) {
	cueFile := f.Filename
	if out := flagOutFile.String(b.cmd); out != "" {
	cueFile = out
	}
	if cueFile != "-" {
	switch _, err := os.Stat(cueFile); {
	case os.IsNotExist(err):
	case err == nil:
	if !flagForce.Bool(b.cmd) {
	// TODO: mimic old behavior: write to stderr, but do not exit
	// with error code. Consider what is best to do here.
	stderr := b.cmd.Command.OutOrStderr()
	fmt.Fprintf(stderr, "skipping file %q: already exists\n", cueFile)
	return nil
	}
	default:
	return fmt.Errorf("error creating file: %v", err)
	}
	}

	if flagRecursive.Bool(b.cmd) {
	h := hoister{fields: map[string]bool{}}
	h.hoist(f)
	}

	return writeFile(b.cmd, f, cueFile)
	}

	func writeFile(cmd Command, f ast.File, cueFile string) error {
	b, err := format.Node(f, format.Simplify())
	if err != nil {
	return fmt.Errorf("error formatting file: %v", err)
	}

	if cueFile == "-" {
	_, err := cmd.OutOrStdout().Write(b)
	return err
	}
	return ioutil.WriteFile(cueFile, b, 0644)
	}

	type hoister struct {
	fields map[string]bool
	}

	func (h hoister) hoist(f ast.File) {
	ast.Walk(f, nil, func(n ast.Node) {
	name := ""
	switch x := n.(type) {
	case *ast.Field:
	name, _, _ = ast.LabelName(x.Label)
	case *ast.Alias:
	name = x.Ident.Name
	}
	if name != "" {
	h.fields[name] = true
	}
	})

	_ = astutil.Apply(f, func(c astutil.Cursor) bool {
	n := c.Node()
	switch n.(type) {
	case *ast.Comprehension:
	return false
	}
	return true

	}, func(c astutil.Cursor) bool {
	switch f := c.Node().(type) {
	case *ast.Field:
	name, _, _ := ast.LabelName(f.Label)
	if name == "" {
	return false
	}

	lit, ok := f.Value.(*ast.BasicLit)
	if !ok \|\| lit.Kind != token.STRING {
	return false
	}

	str, err := literal.Unquote(lit.Value)
	if err != nil {
	return false
	}

	expr, enc := tryParse(str)
	if expr == nil {
	return false
	}

	pkg := c.Import("encoding/" + enc)
	if pkg == nil {
	return false
	}

	// found a replacable string
	dataField := h.uniqueName(name, "_", "cue_")

	f.Value = ast.NewCall(
	ast.NewSel(pkg, "Marshal"),
	ast.NewIdent(dataField))

	// TODO: use definitions instead
	c.InsertAfter(astutil.ApplyRecursively(&ast.Alias{
	Ident: ast.NewIdent(dataField),
	Expr: expr,
	}))
	}
	return true
	})
	}

	func tryParse(str string) (s ast.Expr, pkg string) {
	b := []byte(str)
	if json.Valid(b) {
	expr, err := parser.ParseExpr("", b)
	if err != nil {
	// TODO: report error
	return nil, ""
	}
	switch expr.(type) {
	case ast.StructLit, ast.ListLit:
	default:
	return nil, ""
	}
	return expr, "json"
	}

	if expr, err := yaml.Unmarshal("", b); err == nil {
	switch expr.(type) {
	case ast.StructLit, ast.ListLit:
	default:
	return nil, ""
	}
	return expr, "yaml"
	}

	return nil, ""
	}

	func (h *hoister) uniqueName(base, prefix, typ string) string {
	base = strings.Map(func(r rune) rune {
	if unicode.In(r, unicode.L, unicode.N) {
	return r
	}
	return '_'
	}, base)

	name := prefix + typ + base
	for {
	if !h.fields[name] {
	h.fields[name] = true
	return name
	}
	name = prefix + typ + base
	typ += "x"
	}
	}