blob: 5ce0505588ec7766e632e451f26f38afc94ab9bc [file] [log] [blame]
// Copyright 2019 CUE Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
// Package protobuf defines functionality for parsing protocol buffer
// definitions and instances.
// Proto definition mapping follows the guidelines of mapping Proto to JSON as
// discussed in, and
// carries some of the mapping further when possible with CUE.
// Package Paths
// If a .proto file contains a go_package directive, it will be used as the
// destination package fo the generated .cue files. A common use case is to
// generate the CUE in the same directory as the .proto definition. If a
// destination package is not within the current CUE module, it will be written
// relative to the pkg directory.
// If a .proto file does not specify go_package, it will convert a proto package
// "google.parent.sub" to the import path "".
// It is safe to mix package with and without a go_package within the same
// project.
// Type Mappings
// The following type mappings of definitions apply:
// Proto type CUE type/def Comments
// message struct Message fields become CUE fields, whereby
// names are mapped to lowerCamelCase.
// enum e1 | e2 | ... Where ex are strings. A separate mapping is
// generated to obtain the numeric values.
// map<K, V> { <>: V } All keys are converted to strings.
// repeated V [...V] null is accepted as the empty list [].
// bool bool
// string string
// bytes bytes A base64-encoded string when converted to JSON.
// int32, fixed32 int32 An integer with bounds as defined by int32.
// uint32 uint32 An integer with bounds as defined by uint32.
// int64, fixed64 int64 An integer with bounds as defined by int64.
// uint64 uint64 An integer with bounds as defined by uint64.
// float float32 A number with bounds as defined by float32.
// double float64 A number with bounds as defined by float64.
// Struct struct See struct.proto.
// Value _ See struct.proto.
// ListValue [...] See struct.proto.
// NullValue null See struct.proto.
// BoolValue bool See struct.proto.
// StringValue string See struct.proto.
// NumberValue number See struct.proto.
// StringValue string See struct.proto.
// Empty close({})
// Timestamp time.Time See struct.proto.
// Duration time.Duration See struct.proto.
// Protobuf definitions can be annotated with CUE constraints that are included
// in the generated CUE:
// (cue.val) string CUE expression defining a constraint for this
// field. The string may refer to other fields
// in a message definition using their JSON name.
// (cue.opt) FieldOptions
// required bool Defines the field is required. Use with
// caution.
package protobuf
// TODO mappings:
// Wrapper types various types 2, "2", "foo", true, "true", null, 0, … Wrappers use the same representation in JSON as the wrapped primitive type, except that null is allowed and preserved during data conversion and transfer.
// FieldMask string "f.fooBar,h" See field_mask.proto.
// Any {"@type":"url", See struct.proto.
// f1: value,
// ...}
import (
// Config specifies the environment into which to parse a proto definition file.
type Config struct {
// Root specifies the root of the CUE project, which typically coincides
// with, for example, a version control repository root or the Go module.
// Any imports of proto files within the directory tree of this of this root
// are considered to be "project files" and are generated at the
// corresponding location with this hierarchy. Any other imports are
// considered to be external. Files for such imports are rooted under the
// $Root/pkg/, using the Go package path specified in the .proto file.
Root string
// Module is the Go package import path of the module root. It is the value
// as after "module" in a cue.mod/modules.cue file, if a module file is
// present.
Module string // TODO: determine automatically if unspecified.
// Paths defines the include directory in which to search for imports.
Paths []string
// PkgName specifies the package name for a generated CUE file. A value
// will be derived from the Go package name if undefined.
PkgName string
// An Extractor converts a collection of proto files, typically belonging to one
// repo or module, to CUE. It thereby observes the CUE package layout.
// CUE observes the same package layout as Go and requires .proto files to have
// the go_package directive. Generated CUE files are put in the same directory
// as their corresponding .proto files if the .proto files are located in the
// specified Root (or current working directory if none is specified).
// All other imported files are assigned to the CUE pkg dir ($Root/pkg)
// according to their Go package import path.
type Extractor struct {
root string
cwd string
module string
paths []string
pkgName string
fileCache map[string]result
imports map[string]*build.Instance
errs errors.Error
done bool
type result struct {
p *protoConverter
err error
// NewExtractor creates an Extractor. If the configuration contained any errors
// it will be observable by the Err method fo the Extractor. It is safe,
// however, to only check errors after building the output.
func NewExtractor(c *Config) *Extractor {
cwd, _ := os.Getwd()
b := &Extractor{
root: c.Root,
cwd: cwd,
paths: c.Paths,
pkgName: c.PkgName,
module: c.Module,
fileCache: map[string]result{},
imports: map[string]*build.Instance{},
if b.root == "" {
b.root = b.cwd
return b
// Err returns the errors accumulated during testing. The returned error may be
// of type
func (b *Extractor) Err() error {
return b.errs
func (b *Extractor) addErr(err error) {
b.errs = errors.Append(b.errs, errors.Promote(err, "unknown error"))
// AddFile adds a proto definition file to be converted into CUE by the builder.
// Relatives paths are always taken relative to the Root with which the b is
// configured.
// AddFile assumes that the proto file compiles with protoc and may not report
// an error if it does not. Imports are resolved using the paths defined in
// Config.
func (b *Extractor) AddFile(filename string, src interface{}) error {
if b.done {
err := errors.Newf(token.NoPos,
"protobuf: cannot call AddFile: Instances was already called")
b.errs = errors.Append(b.errs, err)
return err
if b.root != b.cwd && !filepath.IsAbs(filename) {
filename = filepath.Join(b.root, filename)
_, err := b.parse(filename, src)
return err
// TODO: some way of (recursively) adding multiple proto files with filter.
// Files returns a File for each proto file that was added or imported,
// recursively.
func (b *Extractor) Files() (files []*ast.File, err error) {
defer func() { err = b.Err() }()
b.done = true
instances, err := b.Instances()
if err != nil {
return nil, err
for _, p := range instances {
for _, f := range p.Files {
files = append(files, f)
return files, nil
// Instances creates a build.Instances for every package for which a proto file
// was added to the builder. This includes transitive dependencies. It does not
// write the generated files to disk.
// The returned instances can be passed to cue.Build to generated the
// corresponding CUE instances.
// All import paths are located within the specified Root, where external
// packages are located under $Root/pkg. Instances for builtin (like time)
// packages may be omitted, and if not will have no associated files.
func (b *Extractor) Instances() (instances []*build.Instance, err error) {
defer func() { err = b.Err() }()
b.done = true
for _, r := range b.fileCache {
if r.err != nil {
inst := b.getInst(r.p)
if inst == nil {
// Set canonical CUE path for generated file.
f := r.p.file
base := filepath.Base(f.Filename)
base = base[:len(base)-len(".proto")] + "_proto_gen.cue"
f.Filename = filepath.Join(inst.Dir, base)
buf, err := format.Node(f)
if err != nil {
// return nil, err
f, err = parser.ParseFile(f.Filename, buf, parser.ParseComments)
if err != nil {
inst.Files = append(inst.Files, f)
for pkg := range r.p.imported {
inst.ImportPaths = append(inst.ImportPaths, pkg)
for _, p := range b.imports {
instances = append(instances, p)
for _, i := range p.ImportPaths {
if imp := b.imports[i]; imp != nil {
p.Imports = append(p.Imports, imp)
sort.Slice(p.Files, func(i, j int) bool {
return p.Files[i].Filename < p.Files[j].Filename
sort.Slice(instances, func(i, j int) bool {
return instances[i].ImportPath < instances[j].ImportPath
if err != nil {
return instances, err
return instances, nil
func (b *Extractor) getInst(p *protoConverter) *build.Instance {
if b.errs != nil {
return nil
importPath := p.qualifiedImportPath()
if importPath == "" {
err := errors.Newf(token.NoPos,
"no package clause for proto package %q in file %s",, p.file.Filename)
b.errs = errors.Append(b.errs, err)
// TODO: find an alternative. Is proto package good enough?
return nil
dir := b.root
path := p.importPath()
file := p.file.Filename
if !filepath.IsAbs(file) {
file = filepath.Join(b.root, p.file.Filename)
// Determine whether the generated file should be included in place, or
// within cue.mod.
inPlace := strings.HasPrefix(file, b.root)
if !strings.HasPrefix(path, b.module) {
// b.module is either "", in which case we assume the setting for
// inPlace, or not, in which case the module in the protobuf must
// correspond with that of the proto package.
inPlace = false
if !inPlace {
dir = filepath.Join(internal.GenPath(dir), path)
} else {
dir = filepath.Dir(p.file.Filename)
// TODO: verify module name from go_package option against that of actual
// CUE module. Maybe keep this old code for some strict mode?
// want := filepath.Dir(p.file.Filename)
// dir = filepath.Join(dir, path[len(b.module)+1:])
// if !filepath.IsAbs(want) {
// want = filepath.Join(b.root, want)
// }
// if dir != want {
// err := errors.Newf(token.NoPos,
// "file %s mapped to inconsistent path %s; module name %q may be inconsistent with root dir %s",
// want, dir, b.module, b.root,
// )
// b.errs = errors.Append(b.errs, err)
// }
inst := b.imports[importPath]
if inst == nil {
inst = &build.Instance{
Root: b.root,
Dir: dir,
ImportPath: importPath,
PkgName: p.shortPkgName,
DisplayPath: p.protoPkg,
b.imports[importPath] = inst
return inst
// Extract parses a single proto file and returns its contents translated to a CUE
// file. If src is not nil, it will use this as the contents of the file. It may
// be a string, []byte or io.Reader. Otherwise Extract will open the given file
// name at the fully qualified path.
// Extract assumes the proto file compiles with protoc and may not report an error
// if it does not. Imports are resolved using the paths defined in Config.
func Extract(filename string, src interface{}, c *Config) (f *ast.File, err error) {
if c == nil {
c = &Config{}
b := NewExtractor(c)
p, err := b.parse(filename, src)
if err != nil {
return nil, err
p.file.Filename = filename[:len(filename)-len(".proto")] + "_gen.cue"
return p.file, b.Err()
// func GenDefinition
// func MarshalText(cue.Value) (string, error) {
// return "", nil
// }
// func MarshalBytes(cue.Value) ([]byte, error) {
// return nil, nil
// }
// func UnmarshalText(descriptor cue.Value, b string) (ast.Expr, error) {
// return nil, nil
// }
// func UnmarshalBytes(descriptor cue.Value, b []byte) (ast.Expr, error) {
// return nil, nil
// }