Add on-device tests

.github/actions/on_device_tests/action.yaml

@@ -1,9 +1,94 @@
-name: On Host Tests
-description: Runs on-host tests.
+name: On Device Test
+description: Runs on-device tests.
+inputs:
+  gcs_results_path:
+    description: "GCS path for the test results"
+    required: true
+  test_results_key:
+    description: "Artifact key used to store test results."
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Run On-Device Tests
+    - name: Install Requirements
+      run: |
+        pip3 install --require-hashes --no-deps -r ${GITHUB_WORKSPACE}/cobalt/tools/requirements.txt
+      shell: bash
+    - name: Generate gRPC files
+      run: |
+        python -m grpc_tools.protoc -I${GITHUB_WORKSPACE}/cobalt/tools/ \
+            --python_out=${GITHUB_WORKSPACE}/cobalt/tools/ \
+            --grpc_python_out=${GITHUB_WORKSPACE}/cobalt/tools/ \
+            ${GITHUB_WORKSPACE}/cobalt/tools/on_device_tests_gateway.proto
+      shell: bash
+    - name: Set Up Cloud SDK
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Set GCS Project Name
+      run: |
+        echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
+      shell: bash
+    - name: Run Tests on ${{ matrix.platform }} Platform
+      env:
+        GCS_ARTIFACTS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}
+        GCS_RESULTS_PATH: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_TOKEN: ${{ github.token }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        GITHUB_ACTOR_ID: ${{ github.actor_id }}
+        GITHUB_REPO: ${{ github.repository }}
+        GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }}
+        GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }}
+        GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
+        GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
+        GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }}
+        # TODO(b/382508397): Replace hardcoded list with dynamically generated one.
+        TEST_TARGETS_JSON_FILE: cobalt/build/testing/targets/${{ matrix.platform }}/test_targets.json
+      run: |
+        set -uxe
+        python3 -u cobalt/tools/on_device_tests_gateway_client.py \
+          --token ${GITHUB_TOKEN} \
+          trigger \
+          --targets $(cat "${{ env.TEST_TARGETS_JSON_FILE }}" | jq -cr '.test_targets | join(",")') \
+          --filter_json_dir "${GITHUB_WORKSPACE}/cobalt/testing/filters/${{ matrix.platform}}" \
+          --label github_${GITHUB_PR_NUMBER:-postsubmit} \
+          --label builder-${{ matrix.platform }} \
+          --label builder_url-${GITHUB_RUN_URL} \
+          --label github \
+          --label ${GITHUB_EVENT_NAME} \
+          --label ${GITHUB_WORKFLOW} \
+          --label actor-${GITHUB_ACTOR} \
+          --label actor_id-${GITHUB_ACTOR_ID} \
+          --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
+          --label sha-${GITHUB_SHA} \
+          --label repository-${GITHUB_REPO} \
+          --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
+          --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} \
+          ${DIMENSION:+"--dimension" "$DIMENSION"} \
+          ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
+          --gcs_archive_path "${GCS_ARTIFACTS_PATH}" \
+          --gcs_result_path "${GCS_RESULTS_PATH}"
       shell: bash
+    - name: Download ${{ matrix.platform }} Test Results from GCS
+      if: always()
+      env:
+        GCS_RESULTS_PATH: ${{ inputs.gcs_results_path }}
       run: |
-        echo "Nothing yet"
+        set -uxe
+        test_output="${GITHUB_WORKSPACE}/results"
+        echo "test_output=${test_output}" >> $GITHUB_ENV
+
+        mkdir -p "${test_output}"
+        gsutil cp -r "${GCS_RESULTS_PATH}/" "${test_output}"
+      shell: bash
+    - name: Archive Test Results
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ inputs.test_results_key }}
+        path: ${{ env.test_output }}/*

.github/actions/on_host_tests/action.yaml

@@ -82,5 +82,5 @@ runs:
       if: success() || failure()
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ inputs.test_results_key }}-${{ matrix.shard }}
+        name: ${{ inputs.test_results_key }}
         path: ${{ env.results_dir }}/shard_${{ matrix.shard }}/*.xml

.github/actions/process_test_results/action.yaml

@@ -13,7 +13,7 @@ runs:
     - name: Download Test Results
       uses: actions/download-artifact@v4
       with:
-        pattern: ${{ inputs.test_results_key }}-*
+        pattern: ${{ inputs.test_results_key }}*
         path: results/**/*
 
     - name: Test Summary action

.github/actions/upload_test_artifacts/action.yaml

@@ -16,27 +16,16 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Set up Cloud SDK
-      if: inputs.upload_on_device_test_artifacts == 'true'
-      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
-    - name: Upload Android Test Artifacts to GCS
-      if: inputs.upload_on_device_test_artifacts == 'true'
-      env:
-        WORKFLOW: ${{ github.workflow }}
-      run: |
-        set -eux
-        project_name=$(gcloud config get-value project)
-        gsutil cp "${GITHUB_WORKSPACE}/src/out/${{ matrix.platform }}_${{ matrix.config }}/**/*.apk" \
-          "gs://${project_name}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}/"
-      shell: bash
-
-    - name: Create On-Host Test Artifacts Archive
-      if: inputs.upload_on_host_test_artifacts == 'true'
+    - name: Archive Test Artifacts
       run: |
         set -x
         mkdir ${GITHUB_WORKSPACE}/artifacts
         cd src/
-        ./cobalt/build/archive_test_artifacts.py \
+
+        # Put test targets json file in the out folder for the archiving script to pick up.
+        cp "${{ inputs.test_targets_json_file }}" out/${{ matrix.platform }}_${{ matrix.config }}/
+
+        time ./cobalt/build/archive_test_artifacts.py \
           --source out/${{ matrix.platform }}_${{ matrix.config }}/ \
           --destination ${GITHUB_WORKSPACE}/artifacts \
           --platform ${{ matrix.platform }} \
@@ -49,3 +38,17 @@ runs:
         name: ${{ inputs.test_artifacts_key }}
         path: artifacts/*
         retention-days: 3
+    - name: Set up Cloud SDK
+      if: inputs.upload_on_device_test_artifacts == 'true'
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Upload Android Test Artifacts to GCS
+      if: inputs.upload_on_device_test_artifacts == 'true'
+      env:
+        WORKFLOW: ${{ github.workflow }}
+      run: |
+        set -eux
+        project_name=$(gcloud config get-value project)
+
+        gsutil cp "${GITHUB_WORKSPACE}/artifacts/*" \
+          "gs://${project_name}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}/"
+      shell: bash

.github/config/android-arm.json

@@ -3,6 +3,11 @@
   "platforms": [
     "android-arm"
   ],
+  "test_on_device": true,
+  "test_dimensions": {
+    "gtest_device": "sabrina",
+    "gtest_lab": "maneki"
+  },
   "targets": [
     "content_shell",
     "cobalt:gn_all"

.github/config/chromium_android-arm.json

@@ -4,16 +4,6 @@
     "chromium_android-arm"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/config/chromium_android-arm64.json

@@ -4,16 +4,6 @@
     "chromium_android-arm64"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/config/chromium_android-x86.json

@@ -4,16 +4,6 @@
     "chromium_android-x86"
   ],
   "targets": [
-    "base_unittests",
-    "sql_unittests",
-    "net_unittests",
-    "url_unittests",
-    "ipc_tests",
-    "mojo_unittests",
-    "gpu_unittests",
-    "gin_unittests",
-    "blink_unittests",
-    "media_unittests",
     "content_shell",
     "system_webview_apk",
     "system_webview_shell_apk"

.github/workflows/main.yaml

@@ -22,7 +22,6 @@
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.platform }} @ ${{ github.event.label.name || github.event.pull_request.number || github.sha }} @ ${{ github.event.label.name && github.event.pull_request.number || github.event.action }}
   cancel-in-progress: true
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # Retrieves configuration from json file.
   initialize:
@@ -92,7 +91,7 @@
           num_gtest_shards=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -rc '.num_gtest_shards' || echo 1)
           echo "num_gtest_shards=${num_gtest_shards}" >> $GITHUB_ENV
 
-          # create a zero-indexed list of shards for use by matrix, e.g. [0,1,2,3,4,5]
+          # Create a zero-indexed list of shards for use by matrix, e.g. [0,1,2,3,4,5].
           gtest_shards="[$(seq -s, 0 1 $((${num_gtest_shards} - 1)))]"
           echo "gtest_shards=${gtest_shards}" >> $GITHUB_ENV
     outputs:
@@ -134,7 +133,6 @@
     outputs:
       docker_tag: ${{ env.docker_tag }}
 
-  # Runs builds.
   build:
     needs: [initialize, docker-build-image]
     permissions: {}
@@ -183,9 +181,44 @@
           upload_on_host_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_host }}
           upload_on_device_test_artifacts: ${{ matrix.config == 'devel' && needs.initialize.outputs.test_on_device }}
 
-  test:
+  on-device-test:
+    needs: [initialize, build]
+    # Run ODT when on_device label is applied on PR.
+    # Also, run ODT on push and schedule if not explicitly disabled via repo vars.
+    if: needs.initialize.outputs.test_on_device == 'true' &&
+        (
+          (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'on_device')) ||
+          ((inputs.nightly == 'true' || github.event_name == 'schedule') && vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') ||
+          (github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False')
+        )
+    runs-on: [self-hosted, odt-runner]
+    name: ${{ matrix.name }}_on_device_tests
+    permissions: {}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
+        include: ${{ fromJson(needs.initialize.outputs.includes) }}
+        config: [devel]
+    env:
+      TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        timeout-minutes: 30
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+      - name: Run On-Device Tests
+        id: on-device-tests
+        uses: ./.github/actions/on_device_tests
+        with:
+          test_results_key: ${{ env.TEST_RESULTS_KEY }}
+          gcs_results_path: gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}
+
+  on-host-test:
     needs: [initialize, docker-build-image, build]
-    if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true'
+    if: always() && needs.initialize.outputs.test_on_host == 'true'
     permissions: {}
     # TODO(b/372303096): Should have dedicated runner?
     runs-on: [self-hosted, chrobalt-linux-runner]
@@ -201,38 +234,36 @@
     env:
       TMPDIR: /__w/_temp
       TEST_ARTIFACTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_artifacts
-      TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results
+      TEST_RESULTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_results-${{ matrix.shard }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           path: src
       - name: Run On-Host Tests
         id: on-host-tests
-        if: always() && needs.initialize.outputs.test_on_host == 'true'
         uses: ./src/.github/actions/on_host_tests
         with:
           test_artifacts_key: ${{ env.TEST_ARTIFACTS_KEY }}
           test_results_key: ${{ env.TEST_RESULTS_KEY }}
           num_gtest_shards: ${{ needs.initialize.outputs.num_gtest_shards }}
 
   test-upload:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, on-host-test, on-device-test]
     if: always() &&
-        (
-          needs.initialize.outputs.test_on_host == 'true' ||
-          needs.initialize.outputs.test_on_device == 'true'
-        )
+      (
+        needs.initialize.outputs.test_on_host == 'true' ||
+        needs.initialize.outputs.test_on_device == 'true'
+      )
     permissions: {}
-    runs-on: [self-hosted, chrobalt-linux-runner]
+    runs-on: ubuntu-latest
     name: ${{ matrix.name }}_tests_upload
     strategy:
       fail-fast: false
       matrix:
         platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
         include: ${{ fromJson(needs.initialize.outputs.includes) }}
         config: [devel]
-    container: ${{ needs.docker-build-image.outputs.docker_tag }}
     env:
       TMPDIR: /__w/_temp
       TEST_ARTIFACTS_KEY: ${{ matrix.platform }}_${{ matrix.name }}_test_artifacts
@@ -249,8 +280,9 @@
           datadog_api_key: ${{ secrets.datadog_api_key }}
         continue-on-error: true
 
+
   validate-test-result:
-    needs: [initialize, docker-build-image, build, test]
+    needs: [initialize, on-device-test, on-host-test]
     if: always() &&
         (
           needs.initialize.outputs.test_on_host == 'true' ||
@@ -265,8 +297,15 @@
         include: ${{ fromJson(needs.initialize.outputs.includes) }}
         config: [devel]
     steps:
-      - name: Fail if any test shards have failed
-        if:  ${{ needs.test.result != 'success' }}
+      - name: Check Status
+        if: ${{ ! (needs.on-device-test.result == 'success' || needs.on-host-test.result == 'success') }}
         run: |
-          echo "Failing because at least one test shard had errors."
-          exit 1
+          if [ "${{ needs.on-device-test.result }}" != "success" ]; then
+            echo "On device tests failed. See separate job log for details."
+            exit 1
+          fi
+
+          if [ "${{ needs.on-host-test.result }}" != "success" ]; then
+            echo "On host tests failed. See separate job log for details."
+            exit 1
+          fi