cli-tests #156
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cli-tests | |
| # Runs the Codex CLI tests | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| merge_group: | |
| branches: | |
| - main | |
| workflow_call: | |
| inputs: | |
| run-mac-tests: | |
| type: boolean | |
| workflow_dispatch: | |
| inputs: | |
| run-mac-tests: | |
| type: boolean | |
| description: Run tests on macOS | |
| example-debug: | |
| type: boolean | |
| description: Run example tests with CODEX_DEBUG=1 to increase verbosity | |
| schedule: | |
| - cron: '30 8 * * *' # Run nightly at 8:30 UTC | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| defaults: | |
| run: | |
| # Explicitly setting the shell to bash runs commands with | |
| # `bash --noprofile --norc -eo pipefail` instead of `bash -e`. | |
| shell: bash | |
| env: | |
| HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CODEX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HOMEBREW_NO_ANALYTICS: 1 | |
| HOMEBREW_NO_AUTO_UPDATE: 1 | |
| HOMEBREW_NO_EMOJI: 1 | |
| HOMEBREW_NO_ENV_HINTS: 1 | |
| HOMEBREW_NO_INSTALL_CLEANUP: 1 | |
| CODEX_DEBUG: 1 | |
| jobs: | |
| build-codex: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-13] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: Build codex | |
| run: go build -o dist/codex ./cmd/codex | |
| - name: Upload codex artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: codex-${{ runner.os }}-${{ runner.arch }} | |
| path: ./dist/codex | |
| retention-days: 7 | |
| typos: | |
| name: Spell Check with Typos | |
| if: github.ref != 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: crate-ci/typos@v1.16.26 | |
| golangci-lint: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-13] | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install codex | |
| uses: khulnasoft/codex-install-action@main | |
| with: | |
| enable-cache: true | |
| - name: Mount golang cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/golangci-lint | |
| ~/.cache/go-build | |
| ~/go/pkg | |
| key: go-${{ runner.os }}-${{ hashFiles('go.sum') }} | |
| - run: codex run lint | |
| test: | |
| needs: build-codex | |
| strategy: | |
| matrix: | |
| is-main: | |
| - ${{ github.ref == 'refs/heads/main' && 'is-main' || 'not-main' }} | |
| os: [ubuntu-latest, macos-13] | |
| # This is an optimization that runs tests twice, with and without | |
| # the codex.json tests. We can require the other tests to complete before | |
| # merging, while keeping the others as an additional non-required signal | |
| run-project-tests: ["project-tests-only", "project-tests-off"] | |
| # Run tests on: | |
| # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0) | |
| # 2. nix 2.19.2: version before nix profile changes | |
| # 2. latest nix version (note, 2.20.1 introduced a new profile change) | |
| nix-version: ["2.12.0", "2.19.2", "2.20.1"] | |
| exclude: | |
| # Only runs tests on macos if explicitly requested, or on a schedule | |
| - os: "${{ (inputs.run-mac-tests || github.event.schedule != '') && 'dummy' || 'macos-13' }}" | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 60 | |
| env: | |
| # For codex.json tests, we default to non-debug mode since the debug output is less useful than for unit testscripts. | |
| # But we allow overriding via inputs.example-debug | |
| CODEX_DEBUG: ${{ (matrix.run-project-tests == 'project-tests-off' || inputs.example-debug) && '1' || '0' }} | |
| CODEX_GOLANG_TEST_TIMEOUT: "${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && '1h' || '30m' }}" | |
| steps: | |
| - name: Maximize build disk space | |
| uses: easimon/maximize-build-space@v10 | |
| if: matrix.os == 'ubuntu-latest' | |
| with: | |
| root-reserve-mb: 32768 | |
| temp-reserve-mb: 10000 | |
| remove-dotnet: true | |
| remove-android: true | |
| remove-haskell: true | |
| remove-codeql: true | |
| - uses: actions/checkout@v4 | |
| - name: Mount golang cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg | |
| key: go-codex-tests-${{ runner.os }}-${{ hashFiles('go.sum') }} | |
| - name: Install additional shells (dash, zsh) | |
| run: | | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| sudo apt-get update | |
| sudo apt-get install dash zsh | |
| elif [ "$RUNNER_OS" == "macOS" ]; then | |
| brew update | |
| brew install dash zsh | |
| fi | |
| - name: Install codex | |
| uses: khulnasoft/codex-install-action@v0.11.0 | |
| with: | |
| enable-cache: true | |
| - name: Run fast tests | |
| if: matrix.run-project-tests == 'project-tests-off' | |
| run: | | |
| echo "::group::Nix version" | |
| nix --version | |
| echo "::endgroup::" | |
| echo "::group::Contents of /etc/nix/nix.conf" | |
| cat /etc/nix/nix.conf || true | |
| echo "::endgroup::" | |
| echo "::group::Resolved Nix config" | |
| nix show-config --extra-experimental-features nix-command | |
| echo "::endgroup::" | |
| codex run go test -v -timeout $CODEX_GOLANG_TEST_TIMEOUT ./... | |
| - name: Run project (slow) tests | |
| if: matrix.run-project-tests == 'project-tests-only' | |
| run: codex run test-projects-only | |
| auto-nix-install: # ensure Codex installs nix and works properly after installation. | |
| needs: build-codex | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-13] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download codex | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: codex-${{ runner.os }}-${{ runner.arch }} | |
| - name: Add codex to path | |
| run: | | |
| chmod +x ./codex | |
| sudo mv ./codex /usr/local/bin/ | |
| - name: Install nix and codex packages | |
| run: | | |
| export NIX_INSTALLER_NO_CHANNEL_ADD=1 | |
| # Setup github authentication to ensure Github's rate limits are not hit. | |
| # If this works, we can consider refactoring this into a reusable github action helper. | |
| mkdir -p ~/.config/nix | |
| echo "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" > ~/.config/nix/nix.conf | |
| codex run echo "Installing packages..." | |
| - name: Test removing package | |
| run: codex rm go | |
| # Run a sanity test to make sure Codex can install and remove packages with | |
| # the last 3 Nix releases. | |
| test-nix-versions: | |
| needs: build-codex | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-13] | |
| nix-version: [2.15.1, 2.16.1, 2.17.0, 2.18.0, 2.19.2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download codex | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: codex-${{ runner.os }}-${{ runner.arch }} | |
| - name: Add codex to path | |
| run: | | |
| chmod +x ./codex | |
| sudo mv ./codex /usr/local/bin/ | |
| - name: Install Nix | |
| uses: DeterminateSystems/nix-installer-action@v4 | |
| with: | |
| logger: pretty | |
| extra-conf: experimental-features = ca-derivations fetch-closure | |
| nix-package-url: https://releases.nixos.org/nix/nix-${{ matrix.nix-version }}/nix-${{ matrix.nix-version }}-${{ runner.arch == 'X64' && 'x86_64' || 'aarch64' }}-${{ runner.os == 'macOS' && 'darwin' || 'linux' }}.tar.xz | |
| - name: Run codex install, codex run, codex rm | |
| run: | | |
| echo "::group::Nix version" | |
| nix --version | |
| echo "::endgroup::" | |
| echo "::group::Contents of /etc/nix/nix.conf" | |
| cat /etc/nix/nix.conf || true | |
| echo "::endgroup::" | |
| echo "::group::Resolved Nix config" | |
| nix show-config --extra-experimental-features nix-command | |
| echo "::endgroup::" | |
| codex install | |
| codex run -- echo "Hello from codex!" | |
| codex rm go |