From 26aad33c44007f6ca26d2fbf086deadb4b70dd3f Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 27 Jan 2025 20:21:35 +0000 Subject: [PATCH 1/4] build-tools: add diesel_cli --- build-tools.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-tools.Dockerfile b/build-tools.Dockerfile index 7a2ec9c43e75..a862bff47feb 100644 --- a/build-tools.Dockerfile +++ b/build-tools.Dockerfile @@ -251,6 +251,7 @@ ARG CARGO_HAKARI_VERSION=0.9.33 ARG CARGO_DENY_VERSION=0.16.2 ARG CARGO_HACK_VERSION=0.6.33 ARG CARGO_NEXTEST_VERSION=0.9.85 +ARG CARGO_DIESEL_CLI_VERSION=2.2.6 RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && whoami && \ chmod +x rustup-init && \ ./rustup-init -y --default-toolchain ${RUSTC_VERSION} && \ @@ -264,6 +265,8 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux cargo install cargo-deny --locked --version ${CARGO_DENY_VERSION} && \ cargo install cargo-hack --version ${CARGO_HACK_VERSION} && \ cargo install cargo-nextest --version ${CARGO_NEXTEST_VERSION} && \ + cargo install diesel_cli --version ${CARGO_DIESEL_CLI_VERSION} \ + --features postgres-bundled --no-default-features && \ rm -rf /home/nonroot/.cargo/registry && \ rm -rf /home/nonroot/.cargo/git From c29998c2a0ce01fd0cb8a56c898634cc259f2e8b Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 27 Jan 2025 21:17:59 +0000 Subject: [PATCH 2/4] check-codestyle-rust: check diesel schema --- .github/workflows/_check-codestyle-rust.yml | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index cbc47c640640..df177d32d5f0 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -30,6 +30,9 @@ jobs: password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init + permissions: + id-token: write # .github/actions/download + steps: - name: Checkout uses: actions/checkout@v4 @@ -89,3 +92,33 @@ jobs: - name: Check rust licenses/bans/advisories/sources if: ${{ !cancelled() }} run: cargo deny check --hide-inclusion-graph + + # To check diesel schema, we need any version of Postgres, so get it from build artifacts + - name: Download Latest Neon artifact + if: ${{ !cancelled() }} + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact + path: /tmp/neon/ + prefix: latest + aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + + - name: Check diesel schema + if: ${{ !cancelled() }} + env: + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + DATABASE_URL: postgresql://localhost:1235/storage_controller + run: | + export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v16/lib + + cargo neon init + cargo neon storage_controller start + + diesel print-schema > storage_controller/src/schema.rs + + if [ -n "$(git diff storage_controller/src/schema.rs)" ]; then + echo >&2 "Uncommitted changes in diesel schema" + + git diff . + exit 1 + fi From 361222ebbd8f8e6963a631b0e7a244d64a4e435d Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 28 Jan 2025 10:56:14 +0000 Subject: [PATCH 3/4] One more try --- .github/workflows/_check-codestyle-rust.yml | 33 ++++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index df177d32d5f0..f72ab7c9c653 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -16,6 +16,9 @@ defaults: run: shell: bash -euxo pipefail {0} +# No permission for GITHUB_TOKEN by default; the **minimal required** set of permissions should be granted in each job. +permissions: {} + jobs: check-codestyle-rust: strategy: @@ -30,9 +33,6 @@ jobs: password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init - permissions: - id-token: write # .github/actions/download - steps: - name: Checkout uses: actions/checkout@v4 @@ -93,23 +93,28 @@ jobs: if: ${{ !cancelled() }} run: cargo deny check --hide-inclusion-graph - # To check diesel schema, we need any version of Postgres, so get it from build artifacts - - name: Download Latest Neon artifact - if: ${{ !cancelled() }} - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact - path: /tmp/neon/ - prefix: latest - aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + # # To check diesel schema, we need any version of Postgres, so get it from build artifacts + # - name: Download Latest Neon artifact + # if: ${{ !cancelled() }} + # uses: ./.github/actions/download + # with: + # name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact + # path: /tmp/neon/ + # prefix: latest + # aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + + - name: Compile required thing + run: | + make postgres-v16 -j$(nproc) + make neon -j$(nproc) - name: Check diesel schema if: ${{ !cancelled() }} env: - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + # POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install DATABASE_URL: postgresql://localhost:1235/storage_controller run: | - export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v16/lib + # export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v16/lib cargo neon init cargo neon storage_controller start From fce6b0e70e0daf0d8a7582d621c3f0698eeed5a0 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 31 Jan 2025 01:07:15 +0000 Subject: [PATCH 4/4] build-and-test-locally: check diesel schema --- .github/workflows/_build-and-test-locally.yml | 20 +++++++++++ .github/workflows/_check-codestyle-rust.yml | 35 ------------------- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index f97402a90b92..1dec8106b484 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -267,6 +267,26 @@ jobs: path: /tmp/neon aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + - name: Check diesel schema + if: inputs.build-type == 'release' && inputs.arch == 'x64' + env: + DATABASE_URL: postgresql://localhost:1235/storage_controller + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + run: | + /tmp/neon/bin/neon_local init + /tmp/neon/bin/neon_local storage_controller start + + diesel print-schema > storage_controller/src/schema.rs + + if [ -n "$(git diff storage_controller/src/schema.rs)" ]; then + echo >&2 "Uncommitted changes in diesel schema" + + git diff . + exit 1 + fi + + /tmp/neon/bin/neon_local storage_controller stop + # XXX: keep this after the binaries.list is formed, so the coverage can properly work later - name: Merge and upload coverage data if: inputs.build-type == 'debug' diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index f72ab7c9c653..f7518d650027 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -92,38 +92,3 @@ jobs: - name: Check rust licenses/bans/advisories/sources if: ${{ !cancelled() }} run: cargo deny check --hide-inclusion-graph - - # # To check diesel schema, we need any version of Postgres, so get it from build artifacts - # - name: Download Latest Neon artifact - # if: ${{ !cancelled() }} - # uses: ./.github/actions/download - # with: - # name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact - # path: /tmp/neon/ - # prefix: latest - # aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - - - name: Compile required thing - run: | - make postgres-v16 -j$(nproc) - make neon -j$(nproc) - - - name: Check diesel schema - if: ${{ !cancelled() }} - env: - # POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DATABASE_URL: postgresql://localhost:1235/storage_controller - run: | - # export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v16/lib - - cargo neon init - cargo neon storage_controller start - - diesel print-schema > storage_controller/src/schema.rs - - if [ -n "$(git diff storage_controller/src/schema.rs)" ]; then - echo >&2 "Uncommitted changes in diesel schema" - - git diff . - exit 1 - fi