Nightly Release Tests #408
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Release Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run every night at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| jobs: | |
| not-cluster-not-server-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| - name: Run not cluster tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "not cluster and not servertest" --ci | |
| timeout-minutes: 60 | |
| cluster-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run cluster and not on-demand tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "clustertest and not ondemand and not multinode and not cluster_list" --ci | |
| timeout-minutes: 60 | |
| ondemand-aws-tests-local-launcher: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run on-demand aws tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "ondemand and local_launched_ondemand_aws_docker_cluster and not multinode and not servertest and not secrettest" --ci | |
| timeout-minutes: 60 | |
| ondemand-aws-tests-den-launcher: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run on-demand aws tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "den_launched_ondemand_aws_docker_cluster" --ci | |
| timeout-minutes: 60 | |
| ondemand-aws-multinode-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run on-demand aws tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "multinode" --ci | |
| timeout-minutes: 60 | |
| ondemand-gcp-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run on-demand gcp tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "ondemand_gcp_cluster" --ci | |
| timeout-minutes: 60 | |
| kubernetes-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run kubernetes tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "ondemand_k8s_cluster and not cluster_list" --ci | |
| timeout-minutes: 60 | |
| cluster-list-tests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Run cluster list tests | |
| env: | |
| KITCHEN_TESTER_TOKEN: ${{ secrets.KITCHEN_TESTER_PROD_TOKEN }} | |
| KITCHEN_TESTER_USERNAME: ${{ secrets.KITCHEN_TESTER_USERNAME }} | |
| ORG_MEMBER_TOKEN: ${{ secrets.ORG_MEMBER_PROD_TOKEN }} | |
| ORG_MEMBER_USERNAME: ${{ secrets.ORG_MEMBER_USERNAME }} | |
| run: pytest --level release tests -k "cluster_list and not aws and not gcp" --ci | |
| timeout-minutes: 60 | |
| # making sure that the clusters were terminated after the test runs. | |
| check-cluster-status: | |
| if: always() | |
| needs: | |
| - not-cluster-not-server-tests | |
| - cluster-tests | |
| - ondemand-aws-tests-local-launcher | |
| - ondemand-aws-tests-den-launcher | |
| - ondemand-gcp-tests | |
| - kubernetes-tests | |
| - ondemand-aws-multinode-tests | |
| - cluster-list-tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Check out repository code | |
| uses: actions/checkout@v3 | |
| - name: Setup Release Testing | |
| uses: ./.github/workflows/setup_release_testing | |
| with: | |
| KUBECONFIG: ${{ secrets.KUBECONFIG }} | |
| AWS_OSS_ROLE_ARN: ${{ secrets.AWS_OSS_ROLE_ARN }} | |
| DEV_AWS_ACCESS_KEY: ${{ secrets.DEV_AWS_ACCESS_KEY }} | |
| DEV_AWS_SECRET_KEY: ${{ secrets.DEV_AWS_SECRET_KEY }} | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| DEN_TESTER_TOKEN: ${{ secrets.DEN_TESTER_PROD_TOKEN }} | |
| DEN_TESTER_USERNAME: ${{ secrets.DEN_TESTER_USERNAME }} | |
| API_SERVER_URL: ${{ env.API_SERVER_URL }} | |
| EKS_ARN: ${{ secrets.EKS_ARN }} | |
| - name: Wait to check cluster status | |
| run: sleep 300 # 5 minutes | |
| - name: Check cluster status | |
| run: runhouse cluster list |