From 744471e64c25dc03c7615a48aa4299aa266c7cd8 Mon Sep 17 00:00:00 2001 From: Daniel Chao Date: Thu, 7 Mar 2024 08:47:12 -0800 Subject: [PATCH] [com.circleci.v2] Improve Config module (#37) * Add tests and examples * Add OrbStep method for easy interaction with step definitions coming from an orb * Export class `AbstracStep` so it can be subclassed to define typed orb definitions --- packages/com.circleci.v2/Config.pkl | 14 +- packages/com.circleci.v2/PklProject | 2 +- .../examples/concurrent_workflow.pkl | 49 +++++ .../examples/fan_in_fan_out_workflow.pkl | 200 ++++++++++++++++++ .../examples/typed_orb_steps.pkl | 63 ++++++ packages/com.circleci.v2/tests/Config.pkl | 28 +++ .../tests/Config.pkl-expected.pcf | 172 +++++++++++++++ 7 files changed, 525 insertions(+), 3 deletions(-) create mode 100644 packages/com.circleci.v2/examples/concurrent_workflow.pkl create mode 100644 packages/com.circleci.v2/examples/fan_in_fan_out_workflow.pkl create mode 100644 packages/com.circleci.v2/examples/typed_orb_steps.pkl create mode 100644 packages/com.circleci.v2/tests/Config.pkl create mode 100644 packages/com.circleci.v2/tests/Config.pkl-expected.pcf diff --git a/packages/com.circleci.v2/Config.pkl b/packages/com.circleci.v2/Config.pkl index d4260b5..598218e 100644 --- a/packages/com.circleci.v2/Config.pkl +++ b/packages/com.circleci.v2/Config.pkl @@ -390,14 +390,20 @@ class FilterSpec { ignore: (String|Listing)? } -typealias Step = AbstractStep|SimpleStepName +typealias Step = AbstractStep|SimpleStepName|OrbStep typealias SimpleStepName = "checkout"|"setup_remote_docker"|"add_ssh_keyes"|String -local abstract class AbstractStep { +abstract class AbstractStep { fixed hidden __name__: String } +typealias OrbStep = Dynamic(hasProperty("__name__")) + +function OrbStep(name: String): OrbStep = new { + __name__ = name +} + function run(_command: String): RunStep = new { command = _command } @@ -761,6 +767,10 @@ output { renderer = new YamlRenderer { converters { [AbstractStep] = (it) -> Map(it.__name__, it.toMap()) + [Dynamic] = (it) -> + if (it.hasProperty("__name__")) + Map(it.__name__, it.toMap().remove("__name__")) + else it } } } diff --git a/packages/com.circleci.v2/PklProject b/packages/com.circleci.v2/PklProject index 3965bc2..0ea5f7c 100644 --- a/packages/com.circleci.v2/PklProject +++ b/packages/com.circleci.v2/PklProject @@ -17,5 +17,5 @@ amends "../basePklProject.pkl" package { - version = "1.1.0" + version = "1.1.1" } diff --git a/packages/com.circleci.v2/examples/concurrent_workflow.pkl b/packages/com.circleci.v2/examples/concurrent_workflow.pkl new file mode 100644 index 0000000..4ed8369 --- /dev/null +++ b/packages/com.circleci.v2/examples/concurrent_workflow.pkl @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// +module com.circleci.v2.examples.concurrent_workflow + +// Adapdation of https://circleci.com/docs/sample-config/#concurrent-workflow +amends "../Config.pkl" + +jobs { + ["build"] { + docker { + new { image = "cimg/base:2023.03" } + } + steps { + "checkout" + module.run(#"echo "this is the build job""#) + } + } + ["test"] { + docker { + new { image = "cimg/base:2023.03"} + } + steps { + "checkout" + module.run(#"echo "this is the build job""#) + } + } +} + +workflows { + ["build_and_test"] { + jobs { + "build" + "test" + } + } +} diff --git a/packages/com.circleci.v2/examples/fan_in_fan_out_workflow.pkl b/packages/com.circleci.v2/examples/fan_in_fan_out_workflow.pkl new file mode 100644 index 0000000..e3ba67f --- /dev/null +++ b/packages/com.circleci.v2/examples/fan_in_fan_out_workflow.pkl @@ -0,0 +1,200 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// +/// Adapdation of +/// +/// +/// This example uses [OrbStep] for ad-hoc usage of orbs. +/// Alternatively, orb steps can be defined as their own classes that extend [AbstractStep]. +/// For an example of that, see file `typed_orb_steps.pkl`. +module com.circleci.v2.examples.fan_in_fan_out_workflow + +amends "../Config.pkl" + +orbs { + ["docker"] = "circleci/docker@1.0.1" +} + +jobs { + ["prepare-dependencies"] { + docker { + new { image = "node:current-alpine" } + } + steps { + "checkout" + new RunStep { + name = "Compute version number" + command = #"echo "0.0.${CIRCLE_BUILD_NUM}-${CIRCLE_SHA1:0:7}" | tee version.txt"# + } + new RestoreCacheStep { + keys { + #"yarn-deps-{{ checksum "yarn.lock" }}"# + "yarn-deps" + } + } + new RunStep { + name = "yarn install" + command = "yarn install" + } + new SaveCacheStep { + paths { + "node_modules" + } + key = #"yarn-deps-{{ checksum "yarn.lock" }}-{{ epoch }}"# + } + new StoreArtifacts { + path = "yarn.lock" + } + new PersistToWorkspaceStep { + root = "." + paths { "." } + } + } + } + ["build-production"] { + docker { + new { image = "node:current-alpine" } + } + steps { + new AttachWorkspaceStep { at = "." } + new RunStep { + name = "Production build" + command = """ + export __BUILD_VERSION="$(cat version.txt)" + yarn build + """ + } + new StoreArtifacts { path = "dist/server.js" } + new PersistToWorkspaceStep { + root = "." + paths { "." } + } + } + } + ["build-docker-image"] { + machine { + image = "ubuntu-2004:current" + } + steps { + new AttachWorkspaceStep { at = "." } + new RunStep { + name = "Setup __BUILD_VERSION envvar" + command = """ + echo 'export __BUILD_VERSION="$(cat version.txt)"' >> "$BASH_ENV" + """ + } + (module.OrbStep("docker/check")) { + registry = "$DOCKER_REGISTRY" + } + (module.OrbStep("docker/build")) { + image = "$DOCKER_IMAGE_NAME" + tag = "$__BUILD_VERSION" + registry = "$DOCKER_REGISTRY" + } + (module.OrbStep("docker/push")) { + image = "$DOCKER_IMAGE_NAME" + tag = "$__BUILD_VERSION" + registry = "$DOCKER_REGISTRY" + } + } + } + ["test"] { + docker { + new { image = "node:current-alpine" } + } + parallelism = 2 + steps { + new AttachWorkspaceStep { at = "." } + new RunStep { + name = "Run tests" + command = #""" + circleci tests glob '**/*.test.ts' | circleci tests split --split-by timings | xargs yarn test:ci + """# + } + new StoreArtifacts { path = "test-results" } + new StoreTestResults { path = "test-results" } + } + } + ["deploy-docker-image"] { + machine { + image = "ubuntu-2004:current" + } + steps { + new AttachWorkspaceStep { at = "." } + new RunStep { + name = "Setup __BUILD_VERSION envvar" + command = """ + echo 'export __BUILD_VERSION="$(cat version.txt)"' >> "$BASH_ENV" + """ + } + (module.OrbStep("docker/check")) { + registry = "$DOCKER_REGISTRY" + } + (module.OrbStep("docker/pull")) { + images = "$DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:$__BUILD_VERSION" + } + new RunStep { + name = "Tag the image as latest" + command = """ + docker tag $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:$__BUILD_VERSION $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:latest + """ + } + (module.OrbStep("docker/push")) { + image = "$DOCKER_IMAGE_NAME" + tag = "latest" + registry = "$DOCKER_REGISTRY" + } + } + } +} + +workflows { + ["build-test-deploy"] { + jobs { + "prepare-dependencies" + new { + ["build-production"] { + requires { + "prepare-dependencies" + } + } + } + new { + ["build-docker-image"] { + context = "docker-hub" + requires { + "build-production" + } + } + } + new { + ["test"] { + requires { + "prepare-dependencies" + } + } + } + new { + ["deploy-docker-image"] { + context = "docker-hub" + requires { + "build-docker-image" + "test" + } + } + } + } + } +} diff --git a/packages/com.circleci.v2/examples/typed_orb_steps.pkl b/packages/com.circleci.v2/examples/typed_orb_steps.pkl new file mode 100644 index 0000000..eaf0b24 --- /dev/null +++ b/packages/com.circleci.v2/examples/typed_orb_steps.pkl @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// +/// This example uses steps from the [circleci/node](https://circleci.com/developer/orbs/orb/circleci/node) orb. +/// +/// It defins local classes that extend [AbstractStep], so that step definitions are type safe. +amends "../Config.pkl" + +/// Install custom versions of Node.js, and optionally NPM/Yarn, in any +/// execution environment (Docker/Linux, macOS, machine) that does not have +/// it preinstalled. +local class NodeInstall extends AbstractStep { + fixed hidden __name__ = "node/install" + + /// Install Yarn? + /// + /// Default: `false` + `install-yarn`: Boolean? + + /// Where should Node.js be installed? + /// + /// Default: `/usr/local` + `node-install-dir`: String? + + /// Specify the full version tag to install. + /// + /// To install the latest version, set the version to `latest`. + /// If unspecified, the version listed in .nvmrc will be installed. + /// If no .nvmrc file exists the active LTS version of Node.js will be installed by default. + /// For a full list of releases, see the following: + `node-version`: String? + + /// Pick a version of Yarn to install. + /// + /// If no version is specified, the latest stable version will be installed: + `yarn-version`: String? +} + +jobs { + ["test"] { + machine { + image = "ubuntu-2004:current" + } + steps { + "checkout" + new NodeInstall { `node-version` = "v20.11.1" } + new RunStep { command = "npm install" } + new RunStep { command = "npm test" } + } + } +} diff --git a/packages/com.circleci.v2/tests/Config.pkl b/packages/com.circleci.v2/tests/Config.pkl new file mode 100644 index 0000000..ae5a460 --- /dev/null +++ b/packages/com.circleci.v2/tests/Config.pkl @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// +module com.circleci.v2.tests.Config + +amends "pkl:test" + +local allExamples = import*("../examples/*.pkl") + +examples { + for (example in allExamples.keys) { + [example.drop("../examples/".length).replaceLast("pkl", "yaml")] { + allExamples[example].output.text + } + } +} diff --git a/packages/com.circleci.v2/tests/Config.pkl-expected.pcf b/packages/com.circleci.v2/tests/Config.pkl-expected.pcf new file mode 100644 index 0000000..28e3aeb --- /dev/null +++ b/packages/com.circleci.v2/tests/Config.pkl-expected.pcf @@ -0,0 +1,172 @@ +examples { + ["concurrent_workflow.yaml"] { + """ + # Generated from CircleCI.pkl. DO NOT EDIT. + version: '2.1' + jobs: + build: + steps: + - checkout + - run: + command: echo "this is the build job" + docker: + - image: cimg/base:2023.03 + test: + steps: + - checkout + - run: + command: echo "this is the build job" + docker: + - image: cimg/base:2023.03 + workflows: + build_and_test: + jobs: + - build + - test + + """ + } + ["fan_in_fan_out_workflow.yaml"] { + """ + # Generated from CircleCI.pkl. DO NOT EDIT. + version: '2.1' + orbs: + docker: circleci/docker@1.0.1 + jobs: + prepare-dependencies: + steps: + - checkout + - run: + command: echo "0.0.${CIRCLE_BUILD_NUM}-${CIRCLE_SHA1:0:7}" | tee version.txt + name: Compute version number + - restore_cache: + keys: + - yarn-deps-{{ checksum "yarn.lock" }} + - yarn-deps + - run: + command: yarn install + name: yarn install + - save_cache: + paths: + - node_modules + key: yarn-deps-{{ checksum "yarn.lock" }}-{{ epoch }} + - store_artifacts: + path: yarn.lock + - persist_to_workspace: + root: '.' + paths: + - '.' + docker: + - image: node:current-alpine + build-production: + steps: + - attach_workspace: + at: '.' + - run: + command: |- + export __BUILD_VERSION="$(cat version.txt)" + yarn build + name: Production build + - store_artifacts: + path: dist/server.js + - persist_to_workspace: + root: '.' + paths: + - '.' + docker: + - image: node:current-alpine + build-docker-image: + steps: + - attach_workspace: + at: '.' + - run: + command: echo 'export __BUILD_VERSION="$(cat version.txt)"' >> "$BASH_ENV" + name: Setup __BUILD_VERSION envvar + - docker/check: + registry: $DOCKER_REGISTRY + - docker/build: + image: $DOCKER_IMAGE_NAME + tag: $__BUILD_VERSION + registry: $DOCKER_REGISTRY + - docker/push: + image: $DOCKER_IMAGE_NAME + tag: $__BUILD_VERSION + registry: $DOCKER_REGISTRY + machine: + image: ubuntu-2004:current + test: + steps: + - attach_workspace: + at: '.' + - run: + command: circleci tests glob '**/*.test.ts' | circleci tests split --split-by timings | xargs yarn test:ci + name: Run tests + - store_artifacts: + path: test-results + - store_test_results: + path: test-results + parallelism: 2 + docker: + - image: node:current-alpine + deploy-docker-image: + steps: + - attach_workspace: + at: '.' + - run: + command: echo 'export __BUILD_VERSION="$(cat version.txt)"' >> "$BASH_ENV" + name: Setup __BUILD_VERSION envvar + - docker/check: + registry: $DOCKER_REGISTRY + - docker/pull: + images: $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:$__BUILD_VERSION + - run: + command: docker tag $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:$__BUILD_VERSION $DOCKER_REGISTRY/$DOCKER_IMAGE_NAME:latest + name: Tag the image as latest + - docker/push: + image: $DOCKER_IMAGE_NAME + tag: latest + registry: $DOCKER_REGISTRY + machine: + image: ubuntu-2004:current + workflows: + build-test-deploy: + jobs: + - prepare-dependencies + - build-production: + requires: + - prepare-dependencies + - build-docker-image: + requires: + - build-production + context: docker-hub + - test: + requires: + - prepare-dependencies + - deploy-docker-image: + requires: + - build-docker-image + - test + context: docker-hub + + """ + } + ["typed_orb_steps.yaml"] { + """ + # Generated from CircleCI.pkl. DO NOT EDIT. + version: '2.1' + jobs: + test: + steps: + - checkout + - node/install: + node-version: v20.11.1 + - run: + command: npm install + - run: + command: npm test + machine: + image: ubuntu-2004:current + + """ + } +}