cue/literal: implement CUE-specific quoting
Using strconv.Quote is incorrect, as CUE strings are not
entirely the same, resulting in bugs
Code and test cases have been copied from strconv.Quote
and is not based on the deleted (similar) implementation
of internal/core/export/quote.go.
Tests have been augmented with CUE-specific cases.
Note that because of the more complex string syntax of CUE,
the API is somewhat tricky. This API is hopefully straigtforward
given the number of options that need to be covered (not all
implemented).
Also fixes:
- variable indentation
- using tabs instead of spaces for indentation
- fixes multiline interpolation export bug
Fixes #122
Fixes #514
Fixes #540
Fixes #541
Change-Id: If79954678acbd6c9ded2da564856ac28018ba8e1
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/7282
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
diff --git a/internal/core/adt/feature.go b/internal/core/adt/feature.go
index 74926a6..6ee3b6a 100644
--- a/internal/core/adt/feature.go
+++ b/internal/core/adt/feature.go
@@ -20,6 +20,7 @@
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/errors"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/cue/token"
"cuelang.org/go/internal"
)
@@ -65,7 +66,7 @@
if ast.IsValidIdent(s) && !internal.IsDefOrHidden(s) {
return s
}
- return strconv.Quote(s)
+ return literal.String.Quote(s)
default:
return index.IndexToString(int64(x))
}
diff --git a/internal/core/debug/compact.go b/internal/core/debug/compact.go
index cbd2dfb..16bb3bf 100644
--- a/internal/core/debug/compact.go
+++ b/internal/core/debug/compact.go
@@ -22,8 +22,8 @@
import (
"fmt"
- "strconv"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/internal/core/adt"
)
@@ -147,13 +147,10 @@
fmt.Fprint(w, &x.X)
case *adt.String:
- w.string(strconv.Quote(x.Str))
+ w.string(literal.String.Quote(x.Str))
case *adt.Bytes:
- b := []byte(strconv.Quote(string(x.B)))
- b[0] = '\''
- b[len(b)-1] = '\''
- w.string(string(b))
+ w.string(literal.Bytes.Quote(string(x.B)))
case *adt.Top:
w.string("_")
diff --git a/internal/core/debug/debug.go b/internal/core/debug/debug.go
index af3f413..a5d8e0b 100644
--- a/internal/core/debug/debug.go
+++ b/internal/core/debug/debug.go
@@ -27,6 +27,7 @@
"strings"
"cuelang.org/go/cue/errors"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/internal"
"cuelang.org/go/internal/core/adt"
"golang.org/x/xerrors"
@@ -312,13 +313,10 @@
fmt.Fprint(w, &x.X)
case *adt.String:
- w.string(strconv.Quote(x.Str))
+ w.string(literal.String.Quote(x.Str))
case *adt.Bytes:
- b := []byte(strconv.Quote(string(x.B)))
- b[0] = '\''
- b[len(b)-1] = '\''
- w.string(string(b))
+ w.string(literal.Bytes.Quote(string(x.B)))
case *adt.Top:
w.string("_")
diff --git a/internal/core/export/adt.go b/internal/core/export/adt.go
index d73b0eb..bff96e7 100644
--- a/internal/core/export/adt.go
+++ b/internal/core/export/adt.go
@@ -15,12 +15,14 @@
package export
import (
+ "bytes"
"fmt"
"strconv"
"strings"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/ast/astutil"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/cue/token"
"cuelang.org/go/internal/core/adt"
)
@@ -56,6 +58,7 @@
for _, d := range x.Decls {
s.Elts = append(s.Elts, e.decl(d))
}
+
return s
case *adt.FieldReference:
@@ -177,36 +180,43 @@
case *adt.Interpolation:
t := &ast.Interpolation{}
- multiline := false
+ f := literal.String.WithGraphicOnly() // TODO: also support bytes
+ openQuote := `"`
+ closeQuote := `"`
+ indent := ""
// TODO: mark formatting in interpolation itself.
for i := 0; i < len(x.Parts); i += 2 {
str := x.Parts[i].(*adt.String).Str
if strings.IndexByte(str, '\n') >= 0 {
- multiline = true
+ f = f.WithTabIndent(len(e.stack))
+ indent = strings.Repeat("\t", len(e.stack))
+ openQuote = `"""` + "\n" + indent
+ closeQuote = `"""`
break
}
}
- quote := `"`
- if multiline {
- quote = `"""`
- }
- prefix := quote
+ prefix := openQuote
suffix := `\(`
for i, elem := range x.Parts {
if i%2 == 1 {
t.Elts = append(t.Elts, e.expr(elem))
} else {
+ // b := strings.Builder{}
buf := []byte(prefix)
- if i == len(x.Parts)-1 {
- suffix = quote
- }
str := elem.(*adt.String).Str
- if multiline {
- buf = appendEscapeMulti(buf, str, '"')
+ buf = f.AppendEscaped(buf, str)
+ if i == len(x.Parts)-1 {
+ if len(closeQuote) > 1 {
+ buf = append(buf, '\n')
+ buf = append(buf, indent...)
+ }
+ buf = append(buf, closeQuote...)
} else {
- buf = appendEscaped(buf, str, '"', true)
+ if bytes.HasSuffix(buf, []byte("\n")) {
+ buf = append(buf, indent...)
+ }
+ buf = append(buf, suffix...)
}
- buf = append(buf, suffix...)
t.Elts = append(t.Elts, &ast.BasicLit{
Kind: token.STRING,
Value: string(buf),
diff --git a/internal/core/export/label.go b/internal/core/export/label.go
index be73089..b04cb93 100644
--- a/internal/core/export/label.go
+++ b/internal/core/export/label.go
@@ -18,6 +18,7 @@
"strconv"
"cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/cue/token"
"cuelang.org/go/internal/core/adt"
)
@@ -37,7 +38,7 @@
case adt.StringLabel:
s := e.ctx.IndexToString(int64(x))
if !ast.IsValidIdent(s) {
- return ast.NewLit(token.STRING, strconv.Quote(s))
+ return ast.NewLit(token.STRING, literal.Label.Quote(s))
}
fallthrough
diff --git a/internal/core/export/quote.go b/internal/core/export/quote.go
deleted file mode 100644
index 05e95c5..0000000
--- a/internal/core/export/quote.go
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2020 CUE Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package export
-
-import (
- "strconv"
- "strings"
- "unicode/utf8"
-)
-
-// quote quotes the given string.
-func quote(str string, quote byte) string {
- if strings.IndexByte(str, '\n') < 0 {
- buf := []byte{quote}
- buf = appendEscaped(buf, str, quote, true)
- buf = append(buf, quote)
- return string(buf)
- }
- buf := []byte{quote, quote, quote}
- buf = append(buf, multiSep...)
- buf = appendEscapeMulti(buf, str, quote)
- buf = append(buf, quote, quote, quote)
- return string(buf)
-}
-
-// TODO: consider the best indent strategy.
-const multiSep = "\n "
-
-func appendEscapeMulti(buf []byte, str string, quote byte) []byte {
- // TODO(perf)
- a := strings.Split(str, "\n")
- for _, s := range a {
- buf = appendEscaped(buf, s, quote, true)
- buf = append(buf, multiSep...)
- }
- return buf
-}
-
-const lowerhex = "0123456789abcdef"
-
-func appendEscaped(buf []byte, s string, quote byte, graphicOnly bool) []byte {
- for width := 0; len(s) > 0; s = s[width:] {
- r := rune(s[0])
- width = 1
- if r >= utf8.RuneSelf {
- r, width = utf8.DecodeRuneInString(s)
- }
- if width == 1 && r == utf8.RuneError {
- buf = append(buf, `\x`...)
- buf = append(buf, lowerhex[s[0]>>4])
- buf = append(buf, lowerhex[s[0]&0xF])
- continue
- }
- buf = appendEscapedRune(buf, r, quote, graphicOnly)
- }
- return buf
-}
-
-func appendEscapedRune(buf []byte, r rune, quote byte, graphicOnly bool) []byte {
- var runeTmp [utf8.UTFMax]byte
- if r == rune(quote) || r == '\\' { // always backslashed
- buf = append(buf, '\\')
- buf = append(buf, byte(r))
- return buf
- }
- // TODO(perf): IsGraphic calls IsPrint.
- if strconv.IsPrint(r) || graphicOnly && strconv.IsGraphic(r) {
- n := utf8.EncodeRune(runeTmp[:], r)
- buf = append(buf, runeTmp[:n]...)
- return buf
- }
- switch r {
- case '\a':
- buf = append(buf, `\a`...)
- case '\b':
- buf = append(buf, `\b`...)
- case '\f':
- buf = append(buf, `\f`...)
- case '\n':
- buf = append(buf, `\n`...)
- case '\r':
- buf = append(buf, `\r`...)
- case '\t':
- buf = append(buf, `\t`...)
- case '\v':
- buf = append(buf, `\v`...)
- default:
- switch {
- case r < ' ':
- // Invalid for strings, only bytes.
- buf = append(buf, `\x`...)
- buf = append(buf, lowerhex[byte(r)>>4])
- buf = append(buf, lowerhex[byte(r)&0xF])
- case r > utf8.MaxRune:
- r = 0xFFFD
- fallthrough
- case r < 0x10000:
- buf = append(buf, `\u`...)
- for s := 12; s >= 0; s -= 4 {
- buf = append(buf, lowerhex[r>>uint(s)&0xF])
- }
- default:
- buf = append(buf, `\U`...)
- for s := 28; s >= 0; s -= 4 {
- buf = append(buf, lowerhex[r>>uint(s)&0xF])
- }
- }
- }
- return buf
-}
diff --git a/internal/core/export/testdata/adt.txtar b/internal/core/export/testdata/adt.txtar
index 47a323f..c407f8e 100644
--- a/internal/core/export/testdata/adt.txtar
+++ b/internal/core/export/testdata/adt.txtar
@@ -96,11 +96,11 @@
}
}
c1: mystrings.Contains("aa", "a")
-s1: """multi
-
- \(bar)
- line
- """
+s1: """
+ multi
+ \(bar)
+ line
+ """
l1: [3, ...int]
l2: [...int]
l3: []
diff --git a/internal/core/export/testdata/strings.txtar b/internal/core/export/testdata/strings.txtar
new file mode 100644
index 0000000..75a806f
--- /dev/null
+++ b/internal/core/export/testdata/strings.txtar
@@ -0,0 +1,94 @@
+-- in.cue --
+a: """
+ multi
+ line
+ """
+
+b: "message: \(a)!"
+
+c: d: a
+
+-- out/definition --
+a: """
+ multi
+ line
+ """
+b: "message: \(a)!"
+c: {
+ d: a
+}
+-- out/doc --
+[]
+[a]
+[b]
+[c]
+[c d]
+-- out/value --
+== Simplified
+{
+ a: """
+ multi
+ line
+ """
+ b: """
+ message: multi
+ line!
+ """
+ c: {
+ d: """
+ multi
+ line
+ """
+ }
+}
+== Raw
+{
+ a: """
+ multi
+ line
+ """
+ b: """
+ message: multi
+ line!
+ """
+ c: {
+ d: """
+ multi
+ line
+ """
+ }
+}
+== Final
+{
+ a: """
+ multi
+ line
+ """
+ b: """
+ message: multi
+ line!
+ """
+ c: {
+ d: """
+ multi
+ line
+ """
+ }
+}
+== All
+{
+ a: """
+ multi
+ line
+ """
+ b: """
+ message: multi
+ line!
+ """
+ c: {
+ d: """
+ multi
+ line
+ """
+ }
+}
diff --git a/internal/core/export/value.go b/internal/core/export/value.go
index 375c11e..4217425 100644
--- a/internal/core/export/value.go
+++ b/internal/core/export/value.go
@@ -19,6 +19,7 @@
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/ast/astutil"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/cue/token"
"cuelang.org/go/internal/core/adt"
)
@@ -233,9 +234,10 @@
if b := extractBasic(orig); b != nil {
return b
}
+ s := literal.String.WithOptionalTabIndent(len(e.stack)).Quote(n.Str)
return &ast.BasicLit{
Kind: token.STRING,
- Value: quote(n.Str, '"'),
+ Value: s,
}
}
@@ -244,9 +246,10 @@
if b := extractBasic(orig); b != nil {
return b
}
+ s := literal.String.WithOptionalTabIndent(len(e.stack)).Quote(string(n.B))
return &ast.BasicLit{
Kind: token.STRING,
- Value: quote(string(n.B), '\''),
+ Value: s,
}
}
diff --git a/internal/third_party/yaml/decode.go b/internal/third_party/yaml/decode.go
index dbafa23..a509c49 100644
--- a/internal/third_party/yaml/decode.go
+++ b/internal/third_party/yaml/decode.go
@@ -15,6 +15,7 @@
"unicode"
"cuelang.org/go/cue/ast"
+ "cuelang.org/go/cue/literal"
"cuelang.org/go/cue/token"
)
@@ -458,24 +459,21 @@
return &ast.BasicLit{
ValuePos: d.start(n),
Kind: token.STRING,
- Value: strconv.Quote(n.value),
+ Value: literal.String.Quote(n.value),
}
case yaml_STR_TAG:
return &ast.BasicLit{
ValuePos: d.start(n),
Kind: token.STRING,
- Value: d.quoteString(n.value),
+ Value: quoteString(n.value),
}
case yaml_BINARY_TAG:
- buf := strconv.AppendQuote(nil, resolved.(string))
- buf[0] = '\''
- buf[len(buf)-1] = '\''
return &ast.BasicLit{
ValuePos: d.start(n),
Kind: token.STRING,
- Value: string(buf),
+ Value: literal.Bytes.Quote(resolved.(string)),
}
case yaml_BOOL_TAG:
@@ -562,7 +560,7 @@
return &ast.BasicLit{
ValuePos: d.start(n),
Kind: token.STRING,
- Value: strconv.Quote(n.value),
+ Value: literal.Label.Quote(n.value),
}
}
@@ -587,7 +585,7 @@
}
// quoteString converts a string to a CUE multiline string if needed.
-func (d *decoder) quoteString(s string) string {
+func quoteString(s string) string {
lines := []string{}
last := 0
for i, c := range s {
@@ -620,7 +618,7 @@
return string(buf)
}
quoted:
- return strconv.Quote(s)
+ return literal.String.Quote(s)
}
func (d *decoder) sequence(n *node) ast.Expr {
diff --git a/internal/third_party/yaml/decode_test.go b/internal/third_party/yaml/decode_test.go
index b6323e5..9156cda 100644
--- a/internal/third_party/yaml/decode_test.go
+++ b/internal/third_party/yaml/decode_test.go
@@ -384,6 +384,9 @@
C
""",
]`,
+ }, {
+ `"\0"`,
+ `"\u0000"`,
},
// Explicit tags.
