cue: implement parsing and formatting of definitions
- allow :: in addition to : in fields
- allow embedding in structs (not just top-level)
types are checked at compile time
- does not yet handle ExpressionLabels
Not handled yet by the interpreter itself.
Issue #40
Change-Id: I9bc8ef987fb423a31f9979972208c8924bf48399
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/2871
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/cue/ast/ast.go b/cue/ast/ast.go
index 4755321..deb5915 100644
--- a/cue/ast/ast.go
+++ b/cue/ast/ast.go
@@ -86,6 +86,7 @@
func (*BadDecl) declNode() {}
func (*EmbedDecl) declNode() {}
func (*Alias) declNode() {}
+func (*Ellipsis) declNode() {}
// Not technically declarations, but appearing at the same level.
func (*Package) declNode() {}
@@ -285,8 +286,10 @@
Label Label // must have at least one element.
Optional token.Pos
- // No colon: Value must be an StructLit with one field.
- Colon token.Pos
+ // No TokenPos: Value must be an StructLit with one field.
+ TokenPos token.Pos
+ Token token.Token // ':' or '::', ILLEGAL implies ':'
+
Value Expr // the value associated with this field.
Attrs []*Attribute
@@ -300,6 +303,9 @@
return d.Value.End()
}
+// TODO: make Alias a type of Field. This is possible now we have different
+// separator types.
+
// An Alias binds another field to the alias name in the current struct.
type Alias struct {
comments
@@ -419,8 +425,9 @@
// A ForClause node represents a for clause in a comprehension.
type ForClause struct {
comments
- For token.Pos
- Key *Ident // allow pattern matching?
+ For token.Pos
+ Key *Ident // allow pattern matching?
+ // TODO: change to Comma
Colon token.Pos
Value *Ident // allow pattern matching?
In token.Pos
diff --git a/cue/format/node.go b/cue/format/node.go
index 34b45f9..05bd943 100644
--- a/cue/format/node.go
+++ b/cue/format/node.go
@@ -50,6 +50,10 @@
return fmt.Errorf("cue/format: unsupported node type %T", node)
}
+func isRegularField(tok token.Token) bool {
+ return tok == token.ILLEGAL || tok == token.COLON
+}
+
// Helper functions for common node lists. They may be empty.
func (f *formatter) walkDeclList(list []ast.Decl) {
@@ -131,10 +135,12 @@
// shortcut single-element structs.
lastSize := len(f.labelBuf)
f.labelBuf = f.labelBuf[:0]
+ regular := isRegularField(n.Token)
first, opt := n.Label, n.Optional != token.NoPos
- // If the field has a valid position, we assume that an unspecified
- // Lbrace does not signal the intend to collapse fields.
- for n.Label.Pos().IsValid() || f.printer.cfg.simplify {
+
+ // If the label has a valid position, we assume that an unspecified
+ // Lbrace signals the intend to collapse fields.
+ for (n.Label.Pos().IsValid() || f.printer.cfg.simplify) && regular {
obj, ok := n.Value.(*ast.StructLit)
if !ok || len(obj.Elts) != 1 || (obj.Lbrace.IsValid() && !f.printer.cfg.simplify) || len(n.Attrs) > 0 {
break
@@ -156,6 +162,9 @@
if !ok || len(mem.Attrs) > 0 {
break
}
+ if !isRegularField(mem.Token) {
+ break
+ }
entry := labelEntry{mem.Label, mem.Optional != token.NoPos}
f.labelBuf = append(f.labelBuf, entry)
n = mem
@@ -164,6 +173,9 @@
if lastSize != len(f.labelBuf) {
f.print(formfeed)
}
+ if !regular && first.Pos().RelPos() < token.Newline {
+ f.print(newline, nooverride)
+ }
f.before(nil)
f.label(first, opt)
@@ -179,7 +191,11 @@
tab = blank
}
- f.print(n.Colon, token.COLON, tab)
+ if isRegularField(n.Token) {
+ f.print(n.TokenPos, token.COLON, tab)
+ } else {
+ f.print(blank, nooverride, n.Token, tab)
+ }
if n.Value != nil {
switch n.Value.(type) {
case *ast.ListComprehension, *ast.ListLit, *ast.StructLit:
@@ -250,6 +266,9 @@
f.expr(n.Expr)
f.print(newline, newsection, nooverride) // force newline
+ case *ast.Ellipsis:
+ f.ellipsis(n)
+
case *ast.Alias:
f.expr(n.Ident)
f.print(blank, n.Equal, token.BIND, blank)
@@ -340,6 +359,13 @@
}
}
+func (f *formatter) ellipsis(x *ast.Ellipsis) {
+ f.print(x.Ellipsis, token.ELLIPSIS)
+ if x.Type != nil && !isTop(x.Type) {
+ f.expr(x.Type)
+ }
+}
+
func (f *formatter) expr(x ast.Expr) {
const depth = 1
f.expr1(x, token.LowestPrec, depth)
@@ -465,10 +491,7 @@
f.print(noblank, x.Rbrack, token.RBRACK)
case *ast.Ellipsis:
- f.print(x.Ellipsis, token.ELLIPSIS)
- if x.Type != nil && !isTop(x.Type) {
- f.expr(x.Type)
- }
+ f.ellipsis(x)
case *ast.ListComprehension:
f.print(x.Lbrack, token.LBRACK, blank, indent)
diff --git a/cue/format/node_test.go b/cue/format/node_test.go
new file mode 100644
index 0000000..12a4892
--- /dev/null
+++ b/cue/format/node_test.go
@@ -0,0 +1,62 @@
+// Copyright 2019 CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package format
+
+import (
+ "strings"
+ "testing"
+
+ "cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/token"
+)
+
+// TestInvalidAST verifies behavior for various invalid AST inputs. In some
+// cases it is okay to be permissive, as long as the output is correct.
+func TestInvalidAST(t *testing.T) {
+ ident := func(s string) *ast.Ident {
+ return &ast.Ident{NamePos: token.NoSpace.Pos(), Name: s}
+ }
+ testCases := []struct {
+ desc string
+ node ast.Node
+ out string
+ }{{
+ desc: "label sequence for definition",
+ node: &ast.Field{Label: ident("foo"), Value: &ast.StructLit{
+ Elts: []ast.Decl{&ast.Field{
+ Label: ident("bar"),
+ Token: token.ISA,
+ Value: &ast.StructLit{},
+ }},
+ }},
+ // Force a new struct.
+ out: `foo: {
+ bar :: {}
+}`,
+ }}
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ b, err := Node(tc.node)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := string(b)
+ want := strings.TrimSpace(tc.out)
+ if got != want {
+ t.Errorf("\ngot %v;\nwant %v", got, want)
+ }
+ })
+ }
+}
diff --git a/cue/format/testdata/expressions.golden b/cue/format/testdata/expressions.golden
index 65aa717..275ccb2 100644
--- a/cue/format/testdata/expressions.golden
+++ b/cue/format/testdata/expressions.golden
@@ -28,6 +28,17 @@
aaa: 10
}
+ someDefinition :: {
+ embedding
+
+ field: 2
+ }
+
+ openDef :: {
+ a: int
+ ...
+ }
+
attrs: {
a: 8 @go(A) // comment
aa: 8 @go(A) // comment
diff --git a/cue/format/testdata/expressions.input b/cue/format/testdata/expressions.input
index 1d656ea..e44be2c 100644
--- a/cue/format/testdata/expressions.input
+++ b/cue/format/testdata/expressions.input
@@ -28,6 +28,17 @@
aaa: 10
}
+ someDefinition :: {
+ embedding
+
+ field: 2
+ }
+
+ openDef :: {
+ a: int
+ ...
+ }
+
attrs: {
a: 8 @go(A) // comment
aa: 8 @go(A) // comment
diff --git a/cue/parser/parser.go b/cue/parser/parser.go
index 32e104e..43ef86e 100644
--- a/cue/parser/parser.go
+++ b/cue/parser/parser.go
@@ -638,34 +638,25 @@
Rparen: rparen}
}
-func (p *parser) parseFieldList(allowEmit bool) (list []ast.Decl) {
+func (p *parser) parseFieldList() (list []ast.Decl) {
if p.trace {
defer un(trace(p, "FieldList"))
}
- origEmit := allowEmit
p.openList()
defer p.closeList()
- for p.tok != token.RBRACE && p.tok != token.EOF {
- d := p.parseField(allowEmit)
- if e, ok := d.(*ast.EmbedDecl); ok {
- if origEmit && !allowEmit {
- p.errf(p.pos, "only one emit allowed at top level")
- }
- if !origEmit || !allowEmit {
- d = &ast.BadDecl{From: e.Pos(), To: e.End()}
- for _, cg := range e.Comments() {
- d.AddComment(cg)
- }
- }
- // uncomment to only allow one emit per top-level
- // allowEmit = false
- }
- list = append(list, d)
+ for p.tok != token.RBRACE && p.tok != token.ELLIPSIS && p.tok != token.EOF {
+ list = append(list, p.parseField())
+ }
+
+ if p.tok == token.ELLIPSIS {
+ list = append(list, &ast.Ellipsis{Ellipsis: p.pos})
+ p.next()
}
return
}
-func (p *parser) parseField(allowEmit bool) (decl ast.Decl) {
+
+func (p *parser) parseField() (decl ast.Decl) {
if p.trace {
defer un(trace(p, "Field"))
}
@@ -678,6 +669,7 @@
this := &ast.Field{Label: nil}
m := this
+ multipleLabels := false
allowComprehension := true
for i := 0; ; i++ {
@@ -686,9 +678,6 @@
expr, ok := p.parseLabel(m)
if !ok {
- if !allowEmit {
- p.errf(pos, "expected label, found %s", tok)
- }
if expr == nil {
expr = p.parseExpr()
}
@@ -718,9 +707,14 @@
p.next()
}
- if p.tok == token.COLON {
+ _ = multipleLabels
+ if p.tok == token.COLON || p.tok == token.ISA {
+ if p.tok == token.ISA && multipleLabels {
+ p.errf(p.pos, "more than one label before '::' (only one allowed)")
+ }
break
}
+ multipleLabels = true
// TODO: consider disallowing comprehensions with more than one label.
// This can be a bit awkward in some cases, but it would naturally
@@ -730,7 +724,7 @@
switch p.tok {
default:
- if !allowEmit || p.tok != token.COMMA {
+ if p.tok != token.COMMA {
p.errorExpected(p.pos, "label or ':'")
}
switch tok {
@@ -747,12 +741,14 @@
m.Value = &ast.StructLit{Elts: []ast.Decl{field}}
m = field
}
-
- allowEmit = false
}
- this.Colon = p.pos
- p.expect(token.COLON)
+ m.TokenPos = p.pos
+ m.Token = p.tok
+ if p.tok != token.COLON && p.tok != token.ISA {
+ p.errorExpected(pos, "':' or '::'")
+ }
+ p.next() // : or ::
m.Value = p.parseRHS()
p.openList()
@@ -879,7 +875,7 @@
p.exprLev++
var elts []ast.Decl
if p.tok != token.RBRACE {
- elts = p.parseFieldList(false)
+ elts = p.parseFieldList()
}
p.exprLev--
@@ -1354,7 +1350,7 @@
if p.mode&importsOnlyMode == 0 {
// rest of package decls
// TODO: loop and allow multiple expressions.
- decls = append(decls, p.parseFieldList(true)...)
+ decls = append(decls, p.parseFieldList()...)
p.expect(token.EOF)
}
}
diff --git a/cue/parser/parser_test.go b/cue/parser/parser_test.go
index 557233e..c73ec5a 100644
--- a/cue/parser/parser_test.go
+++ b/cue/parser/parser_test.go
@@ -82,6 +82,24 @@
`,
`a: true, b?: "2", c?: 3, "g\("en")"?: 4`,
}, {
+ "definition",
+ `Def :: {
+ b: "2"
+ c: 3
+
+ embedding
+ }
+ `,
+ `Def :: {b: "2", c: 3, embedding}`,
+ }, {
+ "ellipsis in structs",
+ `Def :: {
+ b: "2"
+ ...
+ }
+ `,
+ `Def :: {b: "2", ...}`,
+ }, {
"emitted referencing non-emitted",
`a: 1
b: "2"
diff --git a/cue/parser/print.go b/cue/parser/print.go
index 9849e11..7fd2dda 100644
--- a/cue/parser/print.go
+++ b/cue/parser/print.go
@@ -137,7 +137,12 @@
out += "?"
}
if v.Value != nil {
- out += ": "
+ switch v.Token {
+ case token.ILLEGAL, token.COLON:
+ out += ": "
+ default:
+ out += fmt.Sprintf(" %s ", v.Token)
+ }
out += debugStr(v.Value)
for _, a := range v.Attrs {
out += " "
diff --git a/cue/parser/testdata/commas.src b/cue/parser/testdata/commas.src
index 3e93a54..3865c39 100644
--- a/cue/parser/testdata/commas.src
+++ b/cue/parser/testdata/commas.src
@@ -33,3 +33,4 @@
3
]
+not allowed :: /* ERROR "more than one label before '::'" */ {}
diff --git a/cue/parser/testdata/test.cue b/cue/parser/testdata/test.cue
index cc048d5..cf9ef15 100644
--- a/cue/parser/testdata/test.cue
+++ b/cue/parser/testdata/test.cue
@@ -1,4 +1,3 @@
-
import "math"
foo: 1
diff --git a/cue/scanner/scanner.go b/cue/scanner/scanner.go
index 19881fb..abf4c3e 100644
--- a/cue/scanner/scanner.go
+++ b/cue/scanner/scanner.go
@@ -864,7 +864,12 @@
insertEOL = true
tok, lit = s.scanAttribute()
case ':':
- tok = token.COLON
+ if s.ch == ':' {
+ s.next()
+ tok = token.ISA
+ } else {
+ tok = token.COLON
+ }
case ';':
tok = token.SEMICOLON
insertEOL = true
diff --git a/cue/scanner/scanner_test.go b/cue/scanner/scanner_test.go
index b182e21..3efea8c 100644
--- a/cue/scanner/scanner_test.go
+++ b/cue/scanner/scanner_test.go
@@ -159,6 +159,7 @@
{token.RBRACK, "]", operator},
{token.RBRACE, "}", operator},
{token.COLON, ":", operator},
+ {token.ISA, "::", operator},
// Keywords
{token.TRUE, "true", keyword},
@@ -399,6 +400,7 @@
"}^\n",
"}}^\n",
":\n",
+ "::\n",
";^\n",
"true^\n",
@@ -825,7 +827,7 @@
A: 1 // foo
}
- b: {
+ b :: {
B: 2
// foo
}
diff --git a/cue/token/token.go b/cue/token/token.go
index 3edfed1..5e15443 100644
--- a/cue/token/token.go
+++ b/cue/token/token.go
@@ -87,6 +87,7 @@
RBRACE // }
SEMICOLON // ;
COLON // :
+ ISA // ::
OPTION // ?
operatorEnd
@@ -160,6 +161,7 @@
RBRACE: "}",
SEMICOLON: ";",
COLON: ":",
+ ISA: "::",
OPTION: "?",
BOTTOM: "_|_",
