internal/ci/workflows.cue

// Copyright 2021 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ci

import (
	"github.com/SchemaStore/schemastore/src/schemas/json"
	encjson "encoding/json"
)

workflowsDir: *"./" | string @tag(workflowsDir)

_#masterBranch:      "master"
_#releaseTagPattern: "v*"

workflows: [...{file: string, schema: (json.#Workflow & {})}]
workflows: [
	{
		file:   "test.yml"
		schema: test
	},
	{
		file:   "repository_dispatch.yml"
		schema: repository_dispatch
	},
	{
		file:   "release.yml"
		schema: release
	},
	{
		file:   "tip_triggers.yml"
		schema: tip_triggers
	},
	{
		file:   "new_version_triggers.yml"
		schema: new_version_triggers
	},
	{
		file:   "mirror.yml"
		schema: mirror
	},
]

test: _#bashWorkflow & {

	name: "Test"
	on: {
		push: {
			branches: ["**"] // any branch (including '/' namespaced branches)
			"tags-ignore": [_#releaseTagPattern]
		}
	}

	jobs: {
		start: {
			"runs-on": _#linuxMachine
			steps: [...(_ & {if: "${{ \(_#isCLCITestBranch) }}"})]
			steps: [
				_#writeCookiesFile,
				_#startCLBuild,
			]
		}
		test: {
			needs:     "start"
			strategy:  _#testStrategy
			"runs-on": "${{ matrix.os }}"
			steps: [
				_#writeCookiesFile,
				_#installGo,
				_#checkoutCode,
				_#cacheGoModules,
				_#setGoBuildTags & {
					_#tags: "long"
					if:     "${{ \(_#isMaster) }}"
				},
				_#goGenerate,
				_#goTest,
				_#goTestRace & {
					if: "${{ \(_#isMaster) || \(_#isCLCITestBranch) && matrix.go-version == '\(_#latestStableGo)' && matrix.os == '\(_#linuxMachine)' }}"
				},
				_#goReleaseCheck,
				_#checkGitClean,
				_#pullThroughProxy,
				_#failCLBuild,
			]
		}
		mark_ci_success: {
			"runs-on": _#linuxMachine
			if:        "${{ \(_#isCLCITestBranch) }}"
			needs:     "test"
			steps: [
				_#writeCookiesFile,
				_#passCLBuild,
			]
		}
		delete_build_branch: {
			"runs-on": _#linuxMachine
			if:        "${{ \(_#isCLCITestBranch) && always() }}"
			needs:     "test"
			steps: [
				_#step & {
					run: """
						\(_#tempCueckooGitDir)
						git push https://github.com/cuelang/cue :${GITHUB_REF#\(_#branchRefPrefix)}
						"""
				},
			]
		}
	}

	// _#isCLCITestBranch is an expression that evaluates to true
	// if the job is running as a result of a CL triggered CI build
	_#isCLCITestBranch: "startsWith(github.ref, '\(_#branchRefPrefix)ci/')"

	// _#isMaster is an expression that evaluates to true if the
	// job is running as a result of a master commit push
	_#isMaster: "github.ref == '\(_#branchRefPrefix+_#masterBranch)'"

	_#pullThroughProxy: _#step & {
		name: "Pull this commit through the proxy on \(_#masterBranch)"
		run: """
			v=$(git rev-parse HEAD)
			cd $(mktemp -d)
			go mod init mod.com
			GOPROXY=https://proxy.golang.org go get -d cuelang.org/go/cmd/cue@$v
			"""
		if: "${{ \(_#isMaster) }}"
	}

	_#startCLBuild: _#step & {
		name: "Update Gerrit CL message with starting message"
		run:  (_#gerrit._#setCodeReview & {
			#args: {
				message: "Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"
			}
		}).res
	}

	_#failCLBuild: _#step & {
		if:   "${{ \(_#isCLCITestBranch) && failure() }}"
		name: "Post any failures for this matrix entry"
		run:  (_#gerrit._#setCodeReview & {
			#args: {
				message: "Build failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} for more details"
				labels: {
					"Code-Review": -1
				}
			}
		}).res
	}

	_#passCLBuild: _#step & {
		name: "Update Gerrit CL message with success message"
		run:  (_#gerrit._#setCodeReview & {
			#args: {
				message: "Build succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"
				labels: {
					"Code-Review": 1
				}
			}
		}).res
	}

	_#gerrit: {
		// _#setCodeReview assumes that it is invoked from a job where
		// _#isCLCITestBranch is true
		_#setCodeReview: {
			#args: {
				tag:     "trybot"
				message: string
				labels?: {
					"Code-Review": int
				}
			}
			res: #"""
			\#(_#curl) -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$(basename $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review
			"""#
		}
	}
}

repository_dispatch: _#bashWorkflow & {
	// These constants are defined by github.com/cue-sh/tools/cmd/cueckoo
	_#runtrybot: "runtrybot"
	_#mirror:    "mirror"
	_#importpr:  "importpr"
	_#unity:     "unity"

	_#dispatchJob: _#job & {
		_#type:    string
		"runs-on": _#linuxMachine
		if:        "${{ github.event.client_payload.type == '\(_#type)' }}"
	}

	name: "Repository Dispatch"
	on: ["repository_dispatch"]
	jobs: {
		"\(_#runtrybot)": _#dispatchJob & {
			_#type: _#runtrybot
			steps: [
				_#step & {
					name: "Trigger trybot"
					run:  """
						\(_#tempCueckooGitDir)
						git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }}
						git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD
						git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }}
						"""
				},
			]
		}
		"\(_#mirror)": _#dispatchJob & {
			_#type: _#mirror
			steps:  _#copybaraSteps & {_
				_#name: "Mirror Gerrit to GitHub"
				_#cmd:  "github"
			}
		}
		"\(_#importpr)": _#dispatchJob & {
			_#type: _#importpr
			steps:  _#copybaraSteps & {_
				_#name: "Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit"
				_#cmd:  "github-pr ${{ github.event.client_payload.payload.pr }}"
			}
		}
	}
}

mirror: _#bashWorkflow & {
	name: "Scheduled repo mirror"
	on:
		schedule: [{
			cron: "*/30 * * * *" // every 30 mins
		}]

	jobs: {
		"mirror": {
			"runs-on": _#linuxMachine
			steps:     _#copybaraSteps & {_
				_#name: "Mirror Gerrit to GitHub"
				_#cmd:  "github"
			}
		}
	}
}

release: _#bashWorkflow & {

	name: "Release"
	on: push: tags: [_#releaseTagPattern]
	jobs: {
		goreleaser: {
			"runs-on": _#linuxMachine
			steps: [
				_#checkoutCode & {
					with: "fetch-depth": 0
				},
				_#installGo & {
					with: version: _#latestStableGo
				},
				_#step & {
					name: "Run GoReleaser"
					env: GITHUB_TOKEN: "${{ secrets.ACTIONS_GITHUB_TOKEN }}"
					uses: "goreleaser/goreleaser-action@v2"
					with: {
						args:    "release --rm-dist"
						version: "v0.155.1"
					}
				},
			]
		}
		docker: {
			name:      "docker"
			"runs-on": _#linuxMachine
			steps: [
				_#checkoutCode,
				_#step & {
					name: "Set version environment"
					run: """
						CUE_VERSION=$(echo ${GITHUB_REF##refs/tags/v})
						echo \"CUE_VERSION=$CUE_VERSION\"
						echo \"CUE_VERSION=$(echo $CUE_VERSION)\" >> $GITHUB_ENV
						"""
				},
				_#step & {
					name: "Push to Docker Hub"
					env: {
						DOCKER_BUILDKIT: 1
						GOLANG_VERSION:  1.14
						CUE_VERSION:     "${{ env.CUE_VERSION }}"
					}
					uses: "docker/build-push-action@v1"
					with: {
						tags:           "${{ env.CUE_VERSION }},latest"
						repository:     "cuelang/cue"
						username:       "${{ secrets.DOCKER_USERNAME }}"
						password:       "${{ secrets.DOCKER_PASSWORD }}"
						tag_with_ref:   false
						tag_with_sha:   false
						target:         "cue"
						always_pull:    true
						build_args:     "GOLANG_VERSION=${{ env.GOLANG_VERSION }},CUE_VERSION=v${{ env.CUE_VERSION }}"
						add_git_labels: true
					}
				},
			]
		}
	}
}

tip_triggers: _#bashWorkflow & {

	name: "Push to tip triggers"
	on: push: branches: [_#masterBranch]
	jobs: push: {
		"runs-on": _#linuxMachine
		steps: [
			{
				name: "Rebuild tip.cuelang.org"
				run:  "\(_#curl) -X POST -d {} https://api.netlify.com/build_hooks/${{ secrets.CuelangOrgTipRebuildHook }}"
			},
			{
				name: "Trigger unity build"
				run:  #"""
					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary "{\"event_type\": \"Check against ${GITHUB_SHA}\", \"client_payload\": {\"type\": \"unity\", \"payload\": {\"versions\": \"\\\"commit:${GITHUB_SHA}\\\"\"}}}" https://api.github.com/repos/cue-sh/unity/dispatches
					"""#
			},
		]
	}
}

new_version_triggers: _#bashWorkflow & {

	name: "New release triggers"
	on: push: tags: [_#releaseTagPattern]
	jobs: push: {
		"runs-on": _#linuxMachine
		steps: [
			{
				name: "Rebuild tip.cuelang.org"
				run:  #"""
					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary "{\"event_type\": \"Re-test post release of ${GITHUB_REF##refs/tags/}\"}" https://api.github.com/repos/cuelang/cuelang.org/dispatches
					"""#
			},
			{
				name: "Trigger unity build"
				run:  #"""
					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary "{\"event_type\": \"Check against CUE ${GITHUB_REF##refs/tags/}\", \"client_payload\": {\"type\": \"unity\", \"payload\": {\"versions\": \"\\\"${GITHUB_REF##refs/tags/}\\\"}}}" https://api.github.com/repos/cue-sh/unity/dispatches
					"""#
			},
		]
	}
}

_#bashWorkflow: json.#Workflow & {
	jobs: [string]: defaults: run: shell: "bash"
}

// TODO: drop when cuelang.org/issue/390 is fixed.
// Declare definitions for sub-schemas
_#job:  ((json.#Workflow & {}).jobs & {x: _}).x
_#step: ((_#job & {steps:                 _}).steps & [_])[0]

// We need at least go1.14 for code generation
_#codeGenGo: "1.14.14"

// Use a specific latest version for release builds
_#latestStableGo: "1.15.8"

_#linuxMachine:   "ubuntu-18.04"
_#macosMachine:   "macos-10.15"
_#windowsMachine: "windows-2019"

_#testStrategy: {
	"fail-fast": false
	matrix: {
		// Use a stable version of 1.14.x for go generate
		"go-version": [_#codeGenGo, _#latestStableGo, "1.16"]
		os: [_#linuxMachine, _#macosMachine, _#windowsMachine]
	}
}

_#setGoBuildTags: _#step & {
	_#tags: string
	name:   "Set go build tags"
	run:    """
		go env -w GOFLAGS=-tags=\(_#tags)
		"""
}

_#installGo: _#step & {
	name: "Install Go"
	uses: "actions/setup-go@v2"
	with: {
		"go-version": *"${{ matrix.go-version }}" | string
		stable:       false
	}
}

_#checkoutCode: _#step & {
	name: "Checkout code"
	uses: "actions/checkout@v2"
}

_#cacheGoModules: _#step & {
	name: "Cache Go modules"
	uses: "actions/cache@v1"
	with: {
		path: "~/go/pkg/mod"
		key:  "${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}"
		"restore-keys": """
			${{ runner.os }}-${{ matrix.go-version }}-go-
			"""
	}
}

_#goGenerate: _#step & {
	name: "Generate"
	run:  "go generate ./..."
	// The Go version corresponds to the precise version specified in
	// the matrix. Skip windows for now until we work out why re-gen is flaky
	if: "matrix.go-version == '\(_#codeGenGo)' && matrix.os != '\(_#windowsMachine)'"
}

_#goTest: _#step & {
	name: "Test"
	run:  "go test ./..."
}

_#goTestRace: _#step & {
	name: "Test with -race"
	run:  "go test -race ./..."
}

_#goReleaseCheck: _#step & {
	name: "gorelease check"
	run:  "go run golang.org/x/exp/cmd/gorelease"
}

_#checkGitClean: _#step & {
	name: "Check that git is clean post generate and tests"
	run:  "test -z \"$(git status --porcelain)\" || (git status; git diff; false)"
}

_#writeCookiesFile: _#step & {
	name: "Write the gitcookies file"
	run:  "echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies"
}

_#branchRefPrefix: "refs/heads/"

_#tempCueckooGitDir: """
	mkdir tmpgit
	cd tmpgit
	git init
	git config user.name cueckoo
	git config user.email cueckoo@gmail.com
	git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
	"""

// The cueckoo/copybara Docker image to use
_#cueckooCopybaraImage: "cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c"

// Define the base command for copybara
_#copybaraCmd: {
	_#cmd: string
	#"""
		cd _scripts
		docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" \#(_#cueckooCopybaraImage) bash -c " \
			set -eu; \
			echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \
			chmod 600 ~/.gitcookies; \
			git config --global user.name cueckoo; \
			git config --global user.email cueckoo@gmail.com; \
			git config --global http.cookiefile \$HOME/.gitcookies; \
		  	echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \
			chmod 600 ~/.git-credentials; \
			java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky \#(_#cmd); \
			"
		"""#
}

_#copybaraSteps: {
	_#name: string
	_#cmd:  string
	let cmdCmd = _#cmd
	[
		_#checkoutCode, // needed for copy.bara.sky file
		_#step & {
			name: _#name
			run:  _#copybaraCmd & {_, _#cmd: cmdCmd}
		},
	]
}

_#curl: "curl -f -s"