ci: fix up the CI workflow
The current CI workflow is somewhat flawed because it uses the master
branch workflow file for the test run. This means that any CL which
changes the workflow itself cannot be tested until it hits master, which
is too late if it's broken/incorrect.
Therefore we switch to a model whereby we create temporary build
branches on GitHub. The repository_dispatch workflow creates the
temporary build branch (with the change ID, commit and ref encoded in
the branch name), the regular test workflow writes status updates back
to the CL, then a CRON-based workflow (to follow in a later CL) removes
"old" (currently defined as > 3 hours) temporary build branches.
We also simplify the code generation steps. Running go generate ./...
from the module root will now ensure all generated files are up to date
(including those which are a function of cmd/cue)
Change-Id: Id7c889bd574c6ac316910de98dd60d9b18ad8655
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/6322
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Paul Jolly <paul@myitcv.org.uk>
diff --git a/.github/workflows/gen.go b/.github/workflows/gen.go
deleted file mode 100644
index d4d12be..0000000
--- a/.github/workflows/gen.go
+++ /dev/null
@@ -1,3 +0,0 @@
-package workflows
-
-//go:generate go run cuelang.org/go/cmd/cue cmd genworkflows cuelang.org/go/internal/ci
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4a281b5..e4fc9f9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,6 +1,8 @@
# Generated by internal/ci/ci_tool.cue; do not edit
name: Test
+env:
+ GERRIT_COOKIE: ${{ secrets.gerritCookie }}
defaults:
run:
shell: bash
@@ -12,8 +14,17 @@
- v*
jobs:
test:
+ needs: start
runs-on: ${{ matrix.os }}
steps:
+ - name: Set build branch env vars
+ run: |-
+ echo "::set-env name=CUE_IS_BUILD_BRANCH::$(echo $GITHUB_REF | grep -q '^refs\/heads\/[[:digit:]]\{14\}-I[0-9a-f]\+-[0-9a-f]\+-refs_changes_[[:digit:]]\{2\}_[[:digit:]]\+_[[:digit:]]\+$' && echo true || echo false)"
+ echo "::set-env name=CUE_CHANGE_ID::$(echo $GITHUB_REF | cut -d '-' -f 2)"
+ echo "::set-env name=CUE_COMMIT::$(echo $GITHUB_REF | cut -d '-' -f 3)"
+ echo "::set-env name=CUE_REF::$(echo $GITHUB_REF | cut -d '-' -f 4 | sed 's/_/\//g')"
+ - name: Write the gitcookies file
+ run: echo "$GERRIT_COOKIE" > ~/.gitcookies
- name: Install Go
uses: actions/setup-go@v2
with:
@@ -46,6 +57,12 @@
cd $(mktemp -d)
go mod init mod.com
GOPROXY=https://proxy.golang.org go get -d cuelang.org/go@$v
+ - name: Post any failures for this matrix entry
+ if: ${{ env.CUE_IS_BUILD_BRANCH == 'true' && failure() }}
+ run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build
+ failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url
+ }}/actions/runs/${{ github.run_id }} for more details","labels":{"Code-Review":-1}}''
+ -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$CUE_CHANGE_ID/revisions/$CUE_COMMIT/review'
strategy:
matrix:
go-version:
@@ -57,3 +74,36 @@
- macos-latest
- windows-latest
fail-fast: false
+ start:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Set build branch env vars
+ run: |-
+ echo "::set-env name=CUE_IS_BUILD_BRANCH::$(echo $GITHUB_REF | grep -q '^refs\/heads\/[[:digit:]]\{14\}-I[0-9a-f]\+-[0-9a-f]\+-refs_changes_[[:digit:]]\{2\}_[[:digit:]]\+_[[:digit:]]\+$' && echo true || echo false)"
+ echo "::set-env name=CUE_CHANGE_ID::$(echo $GITHUB_REF | cut -d '-' -f 2)"
+ echo "::set-env name=CUE_COMMIT::$(echo $GITHUB_REF | cut -d '-' -f 3)"
+ echo "::set-env name=CUE_REF::$(echo $GITHUB_REF | cut -d '-' -f 4 | sed 's/_/\//g')"
+ - name: Write the gitcookies file
+ run: echo "$GERRIT_COOKIE" > ~/.gitcookies
+ - name: Update Gerrit CL message with starting message
+ if: ${{ env.CUE_IS_BUILD_BRANCH == 'true' }}
+ run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Started
+ the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{
+ github.run_id }}"}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$CUE_CHANGE_ID/revisions/$CUE_COMMIT/review'
+ end:
+ needs: test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Set build branch env vars
+ run: |-
+ echo "::set-env name=CUE_IS_BUILD_BRANCH::$(echo $GITHUB_REF | grep -q '^refs\/heads\/[[:digit:]]\{14\}-I[0-9a-f]\+-[0-9a-f]\+-refs_changes_[[:digit:]]\{2\}_[[:digit:]]\+_[[:digit:]]\+$' && echo true || echo false)"
+ echo "::set-env name=CUE_CHANGE_ID::$(echo $GITHUB_REF | cut -d '-' -f 2)"
+ echo "::set-env name=CUE_COMMIT::$(echo $GITHUB_REF | cut -d '-' -f 3)"
+ echo "::set-env name=CUE_REF::$(echo $GITHUB_REF | cut -d '-' -f 4 | sed 's/_/\//g')"
+ - name: Write the gitcookies file
+ run: echo "$GERRIT_COOKIE" > ~/.gitcookies
+ - name: Update Gerrit CL message with success message
+ if: ${{ env.CUE_IS_BUILD_BRANCH == 'true' }}
+ run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build
+ succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id
+ }}","labels":{"Code-Review":1}}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$CUE_CHANGE_ID/revisions/$CUE_COMMIT/review'
diff --git a/.github/workflows/test_dispatch.yml b/.github/workflows/test_dispatch.yml
index de3525e..2ee6bf1 100644
--- a/.github/workflows/test_dispatch.yml
+++ b/.github/workflows/test_dispatch.yml
@@ -1,87 +1,24 @@
# Generated by internal/ci/ci_tool.cue; do not edit
-name: Test
-env:
- GERRIT_COOKIE: ${{ secrets.gerritCookie }}
+name: Dispatch build branch
defaults:
run:
shell: bash
on:
- repository_dispatch
jobs:
- test:
- needs: start
- runs-on: ${{ matrix.os }}
+ create_build_branch:
+ runs-on: ubuntu-latest
steps:
- - name: Write the gitcookies file
- run: echo "$GERRIT_COOKIE" > ~/.gitcookies
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Checkout ref
run: |-
git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }}
git checkout FETCH_HEAD
- - name: Cache Go modules
- uses: actions/cache@v1
- with:
- path: ~/go/pkg/mod
- key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum')
- }}
- restore-keys: ${{ runner.os }}-${{ matrix.go-version }}-go-
- - name: Generate
- if: matrix.go-version == '1.14.3' && matrix.os != 'windows-latest'
- run: go generate ./...
- - name: Test
- run: go test ./...
- - name: Test with -race
- run: go test -race ./...
- - name: gorelease check
- if: matrix.go-version == '1.14.3' || matrix.go-version == '1.13.x'
- run: go run golang.org/x/exp/cmd/gorelease
- - name: Check that git is clean post generate and tests
- run: test -z "$(git status --porcelain)" || (git status; git diff; false)
- - name: Post any failures for this matrix entry
- if: ${{ failure() }}
- run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build
- failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url
- }}/actions/runs/${{ github.run_id }} for more details","labels":{"Code-Review":-1}}''
- -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID
- }}/revisions/${{ github.event.client_payload.commit }}/review'
- strategy:
- matrix:
- go-version:
- - 1.12.x
- - 1.13.x
- - 1.14.3
- os:
- - ubuntu-latest
- - macos-latest
- - windows-latest
- fail-fast: false
- start:
- runs-on: ubuntu-latest
- steps:
- - name: Write the gitcookies file
- run: echo "$GERRIT_COOKIE" > ~/.gitcookies
- - name: Update Gerrit CL message with starting message
- run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Started
- the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{
- github.run_id }}"}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{
- github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit
- }}/review'
- end:
- needs: test
- runs-on: ubuntu-latest
- steps:
- - name: Write the gitcookies file
- run: echo "$GERRIT_COOKIE" > ~/.gitcookies
- - name: Update Gerrit CL message with success message
- run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build
- succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id
- }}","labels":{"Code-Review":1}}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{
- github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit
- }}/review'
+ - name: Create build branch
+ run: |-
+ ref=$(echo ${{ github.event.client_payload.ref }} | sed -e 's/\//_/g')
+ branch=$( $(date -u +%Y%m%d%H%M%S)-${{ github.event.client_payload.changeID }}-${{ github.event.client_payload.commit }}-$ref)
+ git checkout -b $branch
+ git push origin $branch
diff --git a/cue.mod/gen.go b/cue.mod/gen.go
deleted file mode 100644
index 1d10683..0000000
--- a/cue.mod/gen.go
+++ /dev/null
@@ -1,3 +0,0 @@
-package cuemod
-
-//go:generate go run cuelang.org/go/cmd/cue cmd vendorgithubschema cuelang.org/go/internal/ci
diff --git a/gen.go b/gen.go
deleted file mode 100644
index cf14afe..0000000
--- a/gen.go
+++ /dev/null
@@ -1,3 +0,0 @@
-package cue
-
-//go:generate go generate ./.github/workflows
diff --git a/internal/ci/ci_tool.cue b/internal/ci/ci_tool.cue
index b150255..1f32fcf 100644
--- a/internal/ci/ci_tool.cue
+++ b/internal/ci/ci_tool.cue
@@ -2,15 +2,13 @@
import (
"tool/file"
- "tool/http"
- "tool/exec"
"encoding/yaml"
)
-command: genworkflows: task: {
+command: gengithub: task: {
for w in workflows {
"\(w.file)": file.Create & {
- filename: w.file
+ filename: "../../.github/workflows/\(w.file)"
contents: """
# Generated by internal/ci/ci_tool.cue; do not edit
@@ -19,16 +17,3 @@
}
}
}
-
-// vendorgithubschema is expected to be run within the cuelang.org/go
-// cue.mod directory
-command: vendorgithubschema: {
- get: http.Get & {
- request: body: ""
- url: "https://raw.githubusercontent.com/SchemaStore/schemastore/f7a0789ccb3bd74a720ddbd6691d60fd9e2d8b7a/src/schemas/json/github-workflow.json"
- }
- convert: exec.Run & {
- stdin: get.response.body
- cmd: "go run cuelang.org/go/cmd/cue import -f -p json -l #Workflow: jsonschema: - --outfile pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue"
- }
-}
diff --git a/internal/ci/gen.go b/internal/ci/gen.go
new file mode 100644
index 0000000..e5644ae
--- /dev/null
+++ b/internal/ci/gen.go
@@ -0,0 +1,3 @@
+package ci
+
+//go:generate go run cuelang.org/go/cmd/cue cmd gengithub .
diff --git a/internal/ci/workflows.cue b/internal/ci/workflows.cue
index 6d2d564..cb8f7aa 100644
--- a/internal/ci/workflows.cue
+++ b/internal/ci/workflows.cue
@@ -110,38 +110,14 @@
}
test: json.#Workflow & {
- name: "Test"
- on: {
- push: {
- branches: ["*"]
- "tags-ignore": ["v*"]
- }
- }
- defaults: run: shell: "bash"
- jobs: test: {
- strategy: #testStrategy
- "runs-on": "${{ matrix.os }}"
- steps: [
- #installGo,
- #checkoutCode,
- #cacheGoModules,
- #goGenerate,
- #goTest,
- #goTestRace,
- #goReleaseCheck,
- #checkGitClean,
- #pullThroughProxy,
- ]
- }
-}
-
-test_dispatch: json.#Workflow & {
- #checkoutRef: #step & {
- name: "Checkout ref"
- run: """
- git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }}
- git checkout FETCH_HEAD
- """
+ #setCUEEnv: #step & {
+ name: "Set build branch env vars"
+ run: #"""
+ echo "::set-env name=CUE_IS_BUILD_BRANCH::$(echo $GITHUB_REF | grep -q '^refs\/heads\/[[:digit:]]\{14\}-I[0-9a-f]\+-[0-9a-f]\+-refs_changes_[[:digit:]]\{2\}_[[:digit:]]\+_[[:digit:]]\+$' && echo true || echo false)"
+ echo "::set-env name=CUE_CHANGE_ID::$(echo $GITHUB_REF | cut -d '-' -f 2)"
+ echo "::set-env name=CUE_COMMIT::$(echo $GITHUB_REF | cut -d '-' -f 3)"
+ echo "::set-env name=CUE_REF::$(echo $GITHUB_REF | cut -d '-' -f 4 | sed 's/_/\//g')"
+ """#
}
#writeCookiesFile: #step & {
name: "Write the gitcookies file"
@@ -156,21 +132,28 @@
}
}
res: #"""
- curl -f -s -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit }}/review
+ curl -f -s -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$CUE_CHANGE_ID/revisions/$CUE_COMMIT/review
"""#
}
}
- name: "Test"
env: GERRIT_COOKIE: "${{ secrets.gerritCookie }}"
- on: ["repository_dispatch"]
+ name: "Test"
+ on: {
+ push: {
+ branches: ["*"]
+ "tags-ignore": ["v*"]
+ }
+ }
defaults: run: shell: "bash"
jobs: {
start: {
"runs-on": "ubuntu-latest"
steps: [
+ #setCUEEnv,
#writeCookiesFile,
#step & {
+ if: "${{ env.CUE_IS_BUILD_BRANCH == 'true' }}"
name: "Update Gerrit CL message with starting message"
run: (#gerrit.#setCodeReview & {
#args: message: "Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"
@@ -179,20 +162,23 @@
]
}
test: {
+ needs: "start"
+ strategy: #testStrategy
"runs-on": "${{ matrix.os }}"
steps: [
+ #setCUEEnv,
#writeCookiesFile,
#installGo,
#checkoutCode,
- #checkoutRef,
#cacheGoModules,
#goGenerate,
#goTest,
#goTestRace,
#goReleaseCheck,
#checkGitClean,
+ #pullThroughProxy,
#step & {
- if: "${{ failure() }}"
+ if: "${{ env.CUE_IS_BUILD_BRANCH == 'true' && failure() }}"
name: "Post any failures for this matrix entry"
run: (#gerrit.#setCodeReview & {
#args: {
@@ -204,14 +190,14 @@
}).res
},
]
- needs: "start"
- strategy: #testStrategy
}
end: {
"runs-on": "ubuntu-latest"
steps: [
+ #setCUEEnv,
#writeCookiesFile,
#step & {
+ if: "${{ env.CUE_IS_BUILD_BRANCH == 'true' }}"
name: "Update Gerrit CL message with success message"
run: (#gerrit.#setCodeReview & {
#args: {
@@ -227,6 +213,37 @@
}
}
}
+
+test_dispatch: json.#Workflow & {
+ name: "Dispatch build branch"
+ on: ["repository_dispatch"]
+ defaults: run: shell: "bash"
+ jobs: {
+ create_build_branch: {
+ "runs-on": "ubuntu-latest"
+ steps: [
+ #checkoutCode,
+ #step & {
+ name: "Checkout ref"
+ run: """
+ git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }}
+ git checkout FETCH_HEAD
+ """
+ },
+ #step & {
+ name: "Create build branch"
+ run: #"""
+ ref=$(echo ${{ github.event.client_payload.ref }} | sed -e 's/\//_/g')
+ branch=$( $(date -u +%Y%m%d%H%M%S)-${{ github.event.client_payload.changeID }}-${{ github.event.client_payload.commit }}-$ref)
+ git checkout -b $branch
+ git push origin $branch
+ """#
+ },
+ ]
+ }
+ }
+}
+
release: {
name: "Release"
on: push: tags: ["v*"]
diff --git a/internal/deps/deps_tool.cue b/internal/deps/deps_tool.cue
new file mode 100644
index 0000000..c6d5ebd
--- /dev/null
+++ b/internal/deps/deps_tool.cue
@@ -0,0 +1,22 @@
+package deps
+
+import (
+ "tool/http"
+ "tool/exec"
+ "strings"
+)
+
+command: vendorgithubschema: {
+ get: http.Get & {
+ request: body: ""
+ url: "https://raw.githubusercontent.com/SchemaStore/schemastore/f7a0789ccb3bd74a720ddbd6691d60fd9e2d8b7a/src/schemas/json/github-workflow.json"
+ }
+ root: exec.Run & {
+ cmd: "go list -m -f {{.Dir}}"
+ stdout: string
+ }
+ convert: exec.Run & {
+ stdin: get.response.body
+ cmd: "go run cuelang.org/go/cmd/cue import -f -p json -l #Workflow: jsonschema: - --outfile \(strings.TrimSpace(root.stdout))/cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue"
+ }
+}
diff --git a/internal/deps/gen.go b/internal/deps/gen.go
new file mode 100644
index 0000000..dff3ef9
--- /dev/null
+++ b/internal/deps/gen.go
@@ -0,0 +1,3 @@
+package deps
+
+//go:generate go run cuelang.org/go/cmd/cue cmd vendorgithubschema .
