load-testing-main.yml

name: Load testing

on:
#  push:
#    branches:
#      - pipeline_performance
  pull_request:
    types: [ opened, reopened, synchronize ]
    branches:
      - pipeline_performance
  workflow_dispatch:
    inputs:
      test_config_json:
        description: test configs
        default: "default"
        type: String


env:
  PROJECT_ID: "dlp-dataflow-load-test"
  REGION: "us-central1"
  INSPECT_TEMPLATE: "projects/dlp-dataflow-load-test/inspectTemplates/dlp-demo-inspect-latest-1706594483019"
  DEID_TEMPLATE: "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
  PUB_SUB_TOPIC: "projects/dlp-dataflow-load-test/topics/load_test_pub_sub_topic"


jobs:
  generate-uuid:
    runs-on: ubuntu-latest

    timeout-minutes: 5

    outputs:
      uuid: ${{ steps.gen-uuid.outputs.uuid }}

    steps:
      - name: Generate UUID for workflow
        id: gen-uuid
        run: |
          new_uuid=$(uuidgen)
          modified_uuid=$(echo "$new_uuid" | cut -c1-8 )
          echo "uuid=$modified_uuid" >> "$GITHUB_OUTPUT"

  pre-processing:
    needs: generate-uuid
    runs-on: [self-hosted, load-testing]
    timeout-minutes: 5
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v2

      - name: Read test details
        id: set-matrix
        run: |
          matrix=$(jq -c . < .github/workflows/configs/load_tests_details.json)
          echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT

      - name: Test if
        id: test-step
        if: always() && env.REGION == 'us-west2'
        run : |
            echo "If condition true"
          

  run-load-test:
    needs:
      - generate-uuid
      - pre-processing

    runs-on: [self-hosted, load-testing]
    continue-on-error: true
    strategy:
      max-parallel: 1
      matrix: ${{ fromJSON(needs.pre-processing.outputs.matrix) }}

    steps:
      - name: Set job parameters
        id: set-job-params
        run: |
          uuid=${{needs.generate-uuid.outputs.uuid}}
          test_name=$(echo "${{matrix.name}}" | tr '_' '-')
          echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name" >> $GITHUB_OUTPUT
          echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}" >> $GITHUB_OUTPUT
          echo "Test details: ${{matrix.name}}"
          echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name"
          echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}"

      - name: Submit dataflow job
        id: submit-dataflow-job
        uses: ./.github/workflows/submit-dataflow-job
        with:
          project_id: ${{env.PROJECT_ID}}
          input_gcs_bucket: "input_dlp_load_test_2"
          gcs_file_path: ${{ matrix.gcs_file_path }}
          dataset: ${{ steps.set-job-params.outputs.dataset }}
          inspect_template: ${{env.INSPECT_TEMPLATE}}
          deid_template: ${{ matrix.deid_template }}
          job_name: ${{steps.set-job-params.outputs.job_name}}
          job_type: ${{ matrix.type }}

      - name: execute copy files workflow for streaming jobs
        id: copy-files
        if: always() &&  matrix.type == 'streaming'
        uses: ./.github/workflows/execute-copy-workflow
        with:
          raw_bucket: ${{ matrix.source_file_bucket }}
          source_file: ${{ matrix.source_file_pattern }}
          input_gcs_bucket: "input_load_test_streaming_job"
          job_id: ${{steps.submit-dataflow-job.outputs.job_id}}
          workflow_name: "generate_files_workflow"
          region: ${{env.REGION}}

      - name: Poll till job finishes
        uses: ./.github/workflows/poll-job
        with:
          job_id: ${{steps.submit-dataflow-job.outputs.job_id}}
          region: ${{env.REGION}}

      - name: Fetch metrics
        uses: ./.github/workflows/fetch-metrics
        with:
          job_id: ${{steps.submit-dataflow-job.outputs.job_id}}
          project_id: ${{env.PROJECT_ID}}
          test_uuid: ${{needs.generate-uuid.outputs.uuid}}
          test_name: ${{ matrix.name }}
          test_details: ${{ toJSON(matrix) }}

      - name: Cleanup
        if: always()
        uses: ./.github/workflows/cleanup
        with:
          project_id: ${{env.PROJECT_ID}}
          job_id: ${{steps.submit-dataflow-job.outputs.job_id}}
          dataset: ${{steps.set-job-params.outputs.dataset}}
          input_gcs_bucket: "input_load_test_streaming_job"
          job_type: ${{ matrix.type }}

  publish-test-results:
    needs:
      - generate-uuid
      - pre-processing
      - run-load-test
    runs-on: [self-hosted, load-testing]
    steps:
      - uses: actions/checkout@v2

      - name: Execute publishMetrics Script
        run: |
          python3 .github/workflows/scripts/publishTestReport.py ${{env.PROJECT_ID}} ${{ needs.generate-uuid.outputs.uuid }}