cue/token: record File pointer in Pos
This is the first step to eliminate FileSet.
Change-Id: Ibb8a17e429d9b2dc2fda0a739520ebef40c67761
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/2121
Reviewed-by: Marcel van Lohuizen <mpvl@google.com>
diff --git a/cmd/cue/cmd/import.go b/cmd/cue/cmd/import.go
index cf6dadb..4b8422b 100644
--- a/cmd/cue/cmd/import.go
+++ b/cmd/cue/cmd/import.go
@@ -462,8 +462,8 @@
}
if idx.field.Value == nil {
idx.field.Value = &ast.ListLit{
- Lbrack: token.Pos(token.NoSpace),
- Rbrack: token.Pos(token.NoSpace),
+ Lbrack: token.NoSpace.Pos(),
+ Rbrack: token.NoSpace.Pos(),
}
}
list := idx.field.Value.(*ast.ListLit)
diff --git a/cue/ast/ast.go b/cue/ast/ast.go
index bd71220..43bbde1 100644
--- a/cue/ast/ast.go
+++ b/cue/ast/ast.go
@@ -518,7 +518,8 @@
return x.NamePos.Add(len(x.Name))
}
func (x *TemplateLabel) End() token.Pos { return x.Rangle }
-func (x *BasicLit) End() token.Pos { return token.Pos(int(x.ValuePos) + len(x.Value)) }
+func (x *BasicLit) End() token.Pos { return x.ValuePos.Add(len(x.Value)) }
+
func (x *Interpolation) End() token.Pos { return x.Elts[len(x.Elts)-1].Pos() }
func (x *StructLit) End() token.Pos {
if x.Rbrace == token.NoPos && len(x.Elts) > 0 {
@@ -580,7 +581,7 @@
// func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() }
func (s *ImportSpec) End() token.Pos {
- if s.EndPos != 0 {
+ if s.EndPos != token.NoPos {
return s.EndPos
}
return s.Path.End()
diff --git a/cue/binop.go b/cue/binop.go
index 452aa64..11cb05a 100644
--- a/cue/binop.go
+++ b/cue/binop.go
@@ -152,7 +152,7 @@
}
return
}
- src := binSrc(0, opUnify, x.val, y.val)
+ src := binSrc(token.NoPos, opUnify, x.val, y.val)
d.add(ctx, binOp(ctx, src, opUnify, x.val, y.val), mark && x.mark && y.mark)
}
diff --git a/cue/errors.go b/cue/errors.go
index fe8ed52..018e267 100644
--- a/cue/errors.go
+++ b/cue/errors.go
@@ -91,7 +91,7 @@
func appendPositions(pos []token.Position, fset *token.FileSet, src source) []token.Position {
if src != nil {
if p := src.Pos(); p != token.NoPos {
- if p >= sharedOffset {
+ if p.Offset() >= sharedOffset {
fset = sharedIndex.fset
}
return append(pos, fset.Position(src.Pos()))
@@ -124,7 +124,7 @@
func appendLocations(locs []string, fset *token.FileSet, src source) []string {
if src != nil {
if p := src.Pos(); p != token.NoPos {
- if p >= sharedOffset {
+ if p.Offset() >= sharedOffset {
fset = sharedIndex.fset
}
return append(locs, fset.Position(src.Pos()).String())
diff --git a/cue/export.go b/cue/export.go
index a61131a..6adf44e 100644
--- a/cue/export.go
+++ b/cue/export.go
@@ -263,7 +263,7 @@
if p.mode.omitOptional || p.mode.concrete {
continue
}
- f.Optional = 1
+ f.Optional = token.NoSpace.Pos()
}
if a.feature&hidden != 0 && p.mode.concrete && p.mode.omitHidden {
continue
@@ -297,7 +297,7 @@
}
opt := token.NoPos
if yield.opt {
- opt = 1 // anything but token.NoPos
+ opt = token.NoSpace.Pos() // anything but token.NoPos
}
f := &ast.Field{
Label: label,
diff --git a/cue/format/format_test.go b/cue/format/format_test.go
index 801c975..ea4010f 100644
--- a/cue/format/format_test.go
+++ b/cue/format/format_test.go
@@ -243,7 +243,7 @@
Decls: []ast.Decl{
&ast.EmitDecl{
Expr: &ast.BasicLit{
- ValuePos: token.Pos(token.NoSpace),
+ ValuePos: token.NoSpace.Pos(),
Value: "1",
},
},
diff --git a/cue/go.go b/cue/go.go
index b2610b6..aad63bb 100644
--- a/cue/go.go
+++ b/cue/go.go
@@ -25,6 +25,7 @@
"sync"
"cuelang.org/go/cue/parser"
+ "cuelang.org/go/cue/token"
"cuelang.org/go/internal"
"github.com/cockroachdb/apd"
)
@@ -504,7 +505,7 @@
// Instead of unifying with the existing type, we substitute
// with the constraints from the tags. The type constraints
// will be implied when unified with a concrete value.
- obj.arcs[i].v = mkBin(ctx, 0, opUnify, a.v, v)
+ obj.arcs[i].v = mkBin(ctx, token.NoPos, opUnify, a.v, v)
}
}
}
diff --git a/cue/parser/error_test.go b/cue/parser/error_test.go
index 5bfff87..c6afecf 100644
--- a/cue/parser/error_test.go
+++ b/cue/parser/error_test.go
@@ -112,7 +112,7 @@
} else {
l = len(tok.String())
}
- here = prev + token.Pos(l)
+ here = prev.Add(l)
}
}
}
diff --git a/cue/parser/import.go b/cue/parser/import.go
index ce95246..d5885ab 100644
--- a/cue/parser/import.go
+++ b/cue/parser/import.go
@@ -164,6 +164,8 @@
type byCommentPos []*ast.CommentGroup
-func (x byCommentPos) Len() int { return len(x) }
-func (x byCommentPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
-func (x byCommentPos) Less(i, j int) bool { return x[i].Pos() < x[j].Pos() }
+func (x byCommentPos) Len() int { return len(x) }
+func (x byCommentPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x byCommentPos) Less(i, j int) bool {
+ return x[i].Pos().Before(x[j].Pos())
+}
diff --git a/cue/parser/parser.go b/cue/parser/parser.go
index e8ce7a5..1baf33e 100644
--- a/cue/parser/parser.go
+++ b/cue/parser/parser.go
@@ -446,7 +446,7 @@
p.syncCnt++
return
}
- if p.pos > p.syncPos {
+ if p.syncPos.Before(p.pos) {
p.syncPos = p.pos
p.syncCnt = 0
return
@@ -475,7 +475,7 @@
func (p *parser) safePos(pos token.Pos) (res token.Pos) {
defer func() {
if recover() != nil {
- res = token.Pos(p.file.Base() + p.file.Size()) // EOF position
+ res = p.file.Pos(p.file.Base()+p.file.Size(), pos.RelPos()) // EOF position
}
}()
_ = p.file.Offset(pos) // trigger a panic if position is out-of-range
@@ -969,7 +969,7 @@
if clauses := p.parseComprehensionClauses(); clauses != nil {
var expr ast.Expr
if len(elts) != 1 {
- p.error(lbrack+1, "list comprehension must have exactly one element")
+ p.error(lbrack.Add(1), "list comprehension must have exactly one element")
}
if len(elts) > 0 {
expr = elts[0]
diff --git a/cue/token/position.go b/cue/token/position.go
index f5ec34f..7f58a12 100644
--- a/cue/token/position.go
+++ b/cue/token/position.go
@@ -77,13 +77,16 @@
// to comparing the respective source file offsets. If p and q are in different
// files, p < q is true if the file implied by p was added to the respective
// file set before the file implied by cue.
-type Pos int
+type Pos struct {
+ file *File
+ offset int
+}
// NoPos is the zero value for Pos; there is no file and line information
// associated with it, and NoPos().IsValid() is false. NoPos is always
// smaller than any other Pos value. The corresponding Position value
// for NoPos is the zero value for Position.
-const NoPos Pos = 0
+var NoPos = Pos{}
// RelPos indicates the relative position of token to the previous token.
type RelPos int
@@ -108,8 +111,8 @@
// NewSection means there are two or more newlines after this token.
NewSection
- relMask Pos = 0xf
- relShift = 4
+ relMask = 0xf
+ relShift = 4
)
var relNames = []string{
@@ -118,14 +121,28 @@
func (p RelPos) String() string { return relNames[p] }
+func (p RelPos) Pos() Pos {
+ return Pos{nil, int(p)}
+}
+
// HasRelPos repors whether p has a relative position.
func (p Pos) HasRelPos() bool {
- return p&relMask != 0
+ return p.offset&relMask != 0
}
+func (p Pos) Before(q Pos) bool {
+ return p.file == q.file && p.Offset() < q.Offset()
+}
+
+// Offset reports the byte offset relative to the file.
+func (p Pos) Offset() int {
+ return p.offset >> relShift
+}
+
+// Add creates a new position relative to the p offset by n.
func (p Pos) Add(n int) Pos {
- return p + toPos(index(n))
+ return Pos{p.file, p.offset + toPos(index(n))}
}
// IsValid reports whether the position is valid.
@@ -140,19 +157,19 @@
}
func (p Pos) WithRel(rel RelPos) Pos {
- return p&^relMask | Pos(rel)
+ return Pos{p.file, p.offset&^relMask | int(rel)}
}
func (p Pos) RelPos() RelPos {
- return RelPos(p & relMask)
+ return RelPos(p.offset & relMask)
}
func (p Pos) index() index {
- return index(p) >> relShift
+ return index(p.offset) >> relShift
}
-func toPos(x index) Pos {
- return (Pos(x) << relShift)
+func toPos(x index) int {
+ return (int(x) << relShift)
}
// -----------------------------------------------------------------------------
@@ -316,7 +333,7 @@
if index(offset) > f.size {
panic("illegal file offset")
}
- return toPos(f.base+index(offset)) + Pos(rel)
+ return Pos{f, toPos(f.base+index(offset)) + int(rel)}
}
// Offset returns the offset for the given file position p;
diff --git a/cue/token/position_test.go b/cue/token/position_test.go
index 906cac0..39ca970 100644
--- a/cue/token/position_test.go
+++ b/cue/token/position_test.go
@@ -81,7 +81,7 @@
t.Errorf("%s, Offset: got offset %d; want %d", f.Name(), offs2, offs)
}
line, col := linecol(lines, offs)
- msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+ msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p.offset)
checkPos(t, msg, f.Position(f.Pos(offs, 0)), Position{f.Name(), offs, line, col})
checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
}
@@ -168,7 +168,7 @@
for offs := 0; offs <= f.Size(); offs++ {
p := f.Pos(offs, 0)
_, col := linecol(lines, offs)
- msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+ msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p.offset)
checkPos(t, msg, f.Position(f.Pos(offs, 0)), Position{"bar", offs, 42, col})
checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
}
@@ -204,7 +204,7 @@
for _, test := range tests {
fset.AddFile(test.filename, fset.Base(), test.size)
}
- if f := fset.File(toPos(index(fset.Base()))); f != nil {
+ if f := fset.File(Pos{nil, toPos(index(fset.Base()))}); f != nil {
t.Errorf("got %v, want nil", f)
}
}
@@ -217,7 +217,7 @@
fset.AddFile(test.filename, fset.Base(), test.size)
}
for file, pos := range offsets {
- f := fset.File(toPos(pos))
+ f := fset.File(Pos{nil, toPos(pos)})
if f.Name() != file {
t.Errorf("got %q at position %d, want %q", f.Name(), pos, file)
}
@@ -239,7 +239,7 @@
stop.Add(1)
go func() {
for i := 0; i < 1000; i++ {
- fset.Position(Pos(r.Int31n(max)))
+ fset.Position(Pos{nil, int(r.Int31n(max))})
}
stop.Done()
}()
diff --git a/internal/protobuf/parse.go b/internal/protobuf/parse.go
index 19eb4ec..b81bd5c 100644
--- a/internal/protobuf/parse.go
+++ b/internal/protobuf/parse.go
@@ -386,7 +386,7 @@
if x.Repeated {
f.Value = &ast.ListLit{
- Ellipsis: token.Pos(token.NoSpace),
+ Ellipsis: token.NoSpace.Pos(),
Type: f.Value,
}
}
@@ -571,7 +571,7 @@
p.addTag(f, o.tags)
if !o.required {
- f.Optional = token.Pos(token.NoSpace)
+ f.Optional = token.NoSpace.Pos()
}
return f
}
@@ -606,7 +606,7 @@
addComments(constraint, 1, o.Comment, o.InlineComment)
p.message.Elts = append(p.message.Elts, constraint)
if !p.required {
- constraint.Optional = token.Pos(token.NoSpace)
+ constraint.Optional = token.NoSpace.Pos()
}
default:
diff --git a/internal/protobuf/util.go b/internal/protobuf/util.go
index e001fa8..980a7d3 100644
--- a/internal/protobuf/util.go
+++ b/internal/protobuf/util.go
@@ -38,8 +38,8 @@
}
var (
- newline = token.Pos(token.Newline)
- newSection = token.Pos(token.NewSection)
+ newline = token.Newline.Pos()
+ newSection = token.NewSection.Pos()
)
func addComments(f ast.Node, i int, doc, inline *proto.Comment) bool {
diff --git a/internal/third_party/yaml/decode.go b/internal/third_party/yaml/decode.go
index bc42da9..fc91a8e 100644
--- a/internal/third_party/yaml/decode.go
+++ b/internal/third_party/yaml/decode.go
@@ -322,7 +322,7 @@
}
// fp := d.p.info.Pos(c.mark.index, 0)
comments = append(comments, &ast.Comment{
- token.Pos(c.pos),
+ c.pos.Pos(),
"//" + c.text[1:],
})
d.p.parser.comments = d.p.parser.comments[1:]
@@ -343,7 +343,7 @@
c := d.p.parser.comments[0]
if c.mark.index == m.index {
comment := &ast.Comment{
- token.Pos(c.pos),
+ c.pos.Pos(),
// d.p.info.Pos(m.index+1, 0),
"//" + c.text[1:],
}
@@ -374,7 +374,7 @@
func (d *decoder) ident(n *node, name string) *ast.Ident {
return &ast.Ident{
// NamePos: d.pos(n.startPos),
- NamePos: token.Pos(d.p.parser.relPos()),
+ NamePos: d.p.parser.relPos().Pos(),
Name: name,
}
}
@@ -424,7 +424,7 @@
case yaml_TIMESTAMP_TAG:
return &ast.BasicLit{
// ValuePos: d.start(n),
- ValuePos: token.Pos(d.p.parser.relPos()),
+ ValuePos: d.p.parser.relPos().Pos(),
Kind: token.STRING,
Value: strconv.Quote(n.value),
}
@@ -432,7 +432,7 @@
case yaml_STR_TAG:
return &ast.BasicLit{
// ValuePos: d.start(n),
- ValuePos: token.Pos(d.p.parser.relPos()),
+ ValuePos: d.p.parser.relPos().Pos(),
Kind: token.STRING,
Value: d.quoteString(n.value),
}
@@ -443,7 +443,7 @@
buf[len(buf)-1] = '\''
return &ast.BasicLit{
// ValuePos: d.start(n),
- ValuePos: token.Pos(d.p.parser.relPos()),
+ ValuePos: d.p.parser.relPos().Pos(),
Kind: token.STRING,
Value: string(buf),
}
@@ -487,11 +487,11 @@
}
err := &ast.BottomLit{
// Bottom: d.pos(n.startPos)
- Bottom: token.Pos(d.p.parser.relPos()),
+ Bottom: d.p.parser.relPos().Pos(),
}
comment := &ast.Comment{
// Slash: d.start(n),
- Slash: token.Pos(token.Blank),
+ Slash: token.Blank.Pos(),
Text: "// " + d.terror(n, tag),
}
err.AddComment(&ast.CommentGroup{
@@ -522,7 +522,7 @@
}
stringLabel:
return &ast.BasicLit{
- ValuePos: token.Pos(d.p.parser.relPos()),
+ ValuePos: d.p.parser.relPos().Pos(),
// ValuePos: d.start(n),
Kind: token.STRING,
Value: strconv.Quote(n.value),
@@ -535,15 +535,15 @@
minuses++
}
expr = &ast.BasicLit{
- // ValuePos: d.start(n) + token.Pos(minuses),
- ValuePos: token.Pos(d.p.parser.relPos()),
+ // ValuePos: d.start(n) + minuses.Pos(),
+ ValuePos: d.p.parser.relPos().Pos(),
Kind: kind,
Value: val,
}
if minuses > 0 {
expr = &ast.UnaryExpr{
// OpPos: d.start(n),
- OpPos: token.Pos(d.p.parser.relPos()),
+ OpPos: d.p.parser.relPos().Pos(),
Op: token.SUB,
X: expr,
}
