Workflow to dispatch jobs to the lab (infra) (#1466)

* Quote variable

* Fix quotes

* remove buggy constraint

* Workflow dispatch workflow

* Generic template to run job from source

* All resources and update with second envsubst

* Try to actually submit the job

* Update launcher name everywhere

* Use the testflinger action to launch the job

* Rename all artifacts to make pathing simpler

* Don't submit the template but the actual launcher

* Better documentation and location for files

* Remove outdated file

* Explicit source instead of .

* Use pipx instead of pip

Minor: omit .service from service name

* Better naming

* Rename in a sensible way the checkbox conf during the workflow

.github/workflows/dispatch_lab_job.yaml

@@ -1,14 +1,62 @@
-name: Run Workflow on PR Comment
+name: Dispatch Checkbox jobs in the lab
 on:
-  issue_comment:
-    types: [created]
+  workflow_dispatch:
+    inputs:
+      # matrix to create is an array where each item is a job configuration
+      # to be dispatched in the lab.
+      # A job configuration is a dict with
+      # - data_source: distribution to provision (ex. distro: desktop-22-04-2-uefi)
+      # - queue: machine that will run the job (ex. 202012-28526)
+      # - test_plan: Checkbox test plan to run (ex. com.canonical.certification::sru)
+      # - match: subset of jobs to run (ex. .*wireless.*)
+      #
+      # One possible matrix_to_create would therefore look like this:
+      # matrix_to_create=[{ data_source: "distro: desktop-22-04-2-uefi", queue: "202012-28526", match: ".*wireless.*", test_plan: "com.canonical.certification::sru" }]'
+      #
+      # To run this workflow manually you can use the `gh` cli utility as follows:
+      # gh workflow run dispatch_lab_job.yaml -f 'matrix_to_create=[...]'
+      matrix_to_create:
+        description: 'Json formatted description of the jobs to dispatch'
+        required: true
+        type: string
 
 jobs:
-  trigger:
-    runs-on: ubuntu-latest
+  run-matrix:
+    runs-on: [self-hosted, testflinger]
+    strategy:
+      matrix:
+        spec: ${{ fromJson(inputs.matrix_to_create) }}
+    defaults:
+      run:
+        working-directory: tools/lab_dispatch
     steps:
-      - name: Dispatch test in the lab and monitor it
-        if: ${{ contains(github.event.comment.body, '/lab') && github.event.issue.pull_request && github.event.issue.author_association == "MEMBER" }}
-        run:
-          COMMENT_BODY="${{ github.event.comment.body }}"
-          echo $COMMENT_BODY
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        run: |
+          sudo apt install gettext
+      - name: Build test resource
+        env:
+          INPUT_DATA_SOURCE: ${{ matrix.spec.data_source }}
+          INPUT_QUEUE: ${{ matrix.spec.queue }}
+          INPUT_MATCH: ${{ matrix.spec.match }}
+          INPUT_TEST_PLAN: ${{ matrix.spec.test_plan }}
+          INPUT_PASSWORD_SECRET: ${{ secrets.INPUT_PASSWORD_SECRET }}
+        run: |
+          echo "::group::Building the testflinger job"
+          export INPUT_CHECKBOX_REVISION="$(git rev-parse HEAD)"
+          envsubst '$INPUT_CHECKBOX_REVISION $INPUT_DATA_SOURCE $INPUT_QUEUE' < generic_source.yaml | tee job.yaml
+          echo "::endgroup::"
+
+          echo "::group::Building the Checkbox launcher"
+
+          # this goes from .template. (missing secret, testplan, match etc. to .partial.)
+          # this is partial as some values are filled in on the agent (like wireless access points names)
+          envsubst '$INPUT_TEST_PLAN $INPUT_MATCH $INPUT_PASSWORD_SECRET' < resources/checkbox.no-manifest.template.conf | tee resources/checkbox.no-manifest.partial.conf
+          echo "::endgroup::"
+      - name: Submit and monitor job
+        uses: canonical/testflinger/.github/actions/submit@main
+        with:
+          poll: true
+          job-path: tools/lab_dispatch/job.yaml

tools/lab_dispatch/build_install_deb.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import shutil
+import argparse
+import subprocess
+
+from pathlib import Path
+from contextlib import suppress
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("paths", type=Path, nargs="+")
+    parser.add_argument("--clean", action="store_true")
+
+    return parser.parse_args()
+
+
+def prepare_repo(repo_root, package_path):
+    shutil.move(package_path / "debian", repo_root)
+
+
+def install_build_depends(repo_root):
+    subprocess.check_call(
+        ["sudo", "apt-get", "-y", "build-dep", "."], cwd=repo_root
+    )
+
+
+def build_package(repo_root):
+    # -Pnocheck: skip tests as we have a pipeline that builds/tests debian
+    #            packages and doing them on slow machines is a big waste of
+    #            time/resources
+    subprocess.check_call(["dpkg-buildpackage", "-Pnocheck"], cwd=repo_root)
+
+
+def install_local_package(repo_root, deb_name_glob):
+    # we build in path.parent, dpkg will put the result on ..
+    package_list = list(repo_root.parent.glob(deb_name_glob))
+    print(f"==== Installing {package_list} ====")
+    package_list = [str(x.resolve()) for x in package_list]
+    subprocess.check_call(
+        [
+            "sudo",
+            "apt-get",
+            "--fix-broken",
+            "-y",
+            "install",
+        ]
+        + package_list,
+        cwd=repo_root.parent,
+    )
+
+
+def install_package(repo_root, package_path):
+    deb_name_glob = f"*{package_path.name}*.deb"
+    install_local_package(repo_root, deb_name_glob)
+
+
+def clean_repo(repo_root):
+    subprocess.check_call(["git", "clean", "-xfd", "."], cwd=repo_root)
+
+
+def build_install_deb(path, clean):
+    package_path = path.resolve()
+
+    repo_root = package_path.parent
+    if "providers" in str(package_path):
+        repo_root = repo_root.parent
+
+    prepare_repo(repo_root, package_path)
+    install_build_depends(repo_root)
+    build_package(repo_root)
+    install_package(repo_root, package_path)
+    if clean:
+        clean_repo(repo_root)
+
+
+def main():
+    args = parse_args()
+    for path in args.paths:
+        build_install_deb(path, args.clean)
+
+
+if __name__ == "__main__":
+    main()

tools/lab_dispatch/generic_source.yaml

@@ -0,0 +1,74 @@
+job_queue: $INPUT_QUEUE
+global_timeout: 3600
+output_timeout: 1800
+provision_data:
+  $INPUT_DATA_SOURCE
+test_data:
+  attachments:
+    - local: "tools/lab_dispatch/resources/manifest.conf"
+      agent: "manifest.conf"
+    - local: "tools/lab_dispatch/resources/checkbox.no-manifest.partial.conf"
+      agent: "checkbox.no-manifest.partial.conf"
+    - local: "tools/lab_dispatch/build_install_deb.py"
+      agent: "build_install_deb.py"
+  test_cmds: |
+    #!/usr/bin/env bash
+
+    set -x
+
+    # input arguments
+    CHECKBOX_REVISION=$INPUT_CHECKBOX_REVISION
+    RESOURCES_PATH="attachments/test"
+
+    TOOLS_PATH=tools
+
+    # retrieve all scripts/tools necessary from a repo
+    curl -Ls -o install_tools.sh https://raw.githubusercontent.com/canonical/hwcert-jenkins-tools/main/install_tools.sh
+    source install_tools.sh $TOOLS_PATH
+
+    # ensure device is available before continuing
+    wait_for_ssh
+
+    _run sudo add-apt-repository ppa:checkbox-dev/edge
+    _run install_packages git python3 python3-pip dpkg-dev
+
+    wait_for_ssh
+
+    _put $RESOURCES_PATH/build_install_deb.py :
+    _run git clone https://github.com/canonical/checkbox.git
+    _run git -C checkbox checkout $CHECKBOX_REVISION
+    _run python3 build_install_deb.py --clean checkbox/checkbox-ng \
+        checkbox/checkbox-support checkbox/providers/resource \
+        checkbox/providers/base checkbox/providers/sru
+    _run sudo systemctl restart checkbox-ng
+
+    git clone https://github.com/canonical/checkbox.git
+    git -C checkbox checkout $CHECKBOX_REVISION
+    pipx install --spec checkbox/checkbox-ng checkbox-ng
+
+    # retrieve manifest
+    MANIFEST_FILE=manifest.conf
+    fetch_manifest --manifest_file manifest.conf $CID $HEXR_DEVICE_SECURE_ID
+    if [ $? -ne 0 ]; then
+        echo "Using default manifest"
+        MANIFEST_FILE=$RESOURCES_PATH/manifest.conf
+    fi
+
+    ### create checkbox launcher
+    # first dump the location specific infos in the launcher
+    which envsubst || install_packages gettext
+    envsubst < $RESOURCES_PATH/checkbox.no-manifest.partial.conf > checkbox.no-manifest.conf
+    # then insert the manifest entries via the stacker
+    stacker --output checkbox.conf checkbox.no-manifest.conf $MANIFEST_FILE
+
+    wait_for_ssh
+
+    check_for_checkbox_service
+
+    # run the canary test plan
+    PYTHONUNBUFFERED=1 checkbox-cli control $DEVICE_IP checkbox.conf
+    EXITCODE=$?
+
+    # placeholder for gathering possible artifacts
+
+    exit $EXITCODE

tools/lab_dispatch/resources/checkbox.no-manifest.template.conf

@@ -0,0 +1,40 @@
+[launcher]
+launcher_version = 1
+stock_reports = certification, text
+
+[test plan]
+unit = $INPUT_TEST_PLAN
+forced = yes
+
+[test selection]
+forced = true
+match=$INPUT_MATCH
+
+[ui]
+type = silent
+
+[environment]
+ROUTERS = multiple
+WPA_BG_SSID = $WPA_BG_SSID
+WPA_BG_PSK = $INPUT_PASSWORD_SECRET
+WPA_N_SSID = $WPA_N_SSID
+WPA_N_PSK = $INPUT_PASSWORD_SECRET
+WPA_AC_SSID = $WPA_AC_SSID
+WPA_AC_PSK = $INPUT_PASSWORD_SECRET
+WPA_AX_SSID = $WPA_AX_SSID
+WPA_AX_PSK = $INPUT_PASSWORD_SECRET
+WPA3_AX_SSID = $WPA3_AX_SSID
+WPA3_AX_PSK = $INPUT_PASSWORD_SECRET
+OPEN_BG_SSID = $OPEN_BG_SSID
+OPEN_N_SSID = $OPEN_N_SSID
+OPEN_AC_SSID = $OPEN_AC_SSID
+OPEN_AX_SSID = $OPEN_AX_SSID
+BTDEVADDR = 54:35:30:15:BC:DA
+TRANSFER_SERVER = cdimage.ubuntu.com
+DISPLAY= :0
+SUDO_USER = ubuntu
+STRESS_NG_DISK_TIME = 15
+STRESS_NG_CPU_TIME = 180
+PTS_CACHE_URL = http://10.102.196.9/sru/phoronix_cache/
+SNAPD_TASK_TIMEOUT = 120
+ZAPPER_HOST = $ZAPPER_IP

tools/lab_dispatch/resources/manifest.conf

@@ -0,0 +1,12 @@
+[manifest]
+com.canonical.certification::has_bt_smart = false
+com.canonical.certification::has_camera = true
+com.canonical.certification::has_card_reader = true
+com.canonical.certification::has_ethernet_adapter = true
+com.canonical.certification::has_thunderbolt = false
+com.canonical.certification::has_thunderbolt3 = false
+com.canonical.certification::has_touchscreen = false
+com.canonical.certification::has_tpm2_chip = false
+com.canonical.certification::has_usb_storage = true
+com.canonical.certification::has_usb_type_c = false
+com.canonical.certification::has_wlan_adapter = true