Update CI files

[noissue]

.ci/ansible/start_container.yaml

@@ -19,6 +19,8 @@
       community.crypto.openssh_keypair:
         path: ssh/id_ed25519
         type: ed25519
+        owner: 700  # pulp in the container
+      become: true
       when: stream_test | default(false)
 
     - name: "Generate Pulp Settings"

.ci/scripts/update_ci_branches.py

@@ -1,34 +1,25 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_cookbook' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
 import os
-import re
 import sys
 import requests
-from packaging.version import Version
-from git import Repo
-
-try:
-    initial_branch = sys.argv[1]
-except IndexError:
-    initial_branch = None
 
-repo = Repo(os.getcwd())
-heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_cookbook.git").split("\n")
-branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])]
-branches.sort(key=lambda ver: Version(ver))
+branches = sys.argv[1:]
 
 headers = {
     "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}",
     "Accept": "application/vnd.github.v3+json",
 }
 
-if not initial_branch or initial_branch not in branches:
-    exit("Initial branch not found")
-else:
-    starting = branches.index(initial_branch)
-
 github_api = "https://api.github.com"
 workflow_path = "/actions/workflows/update_ci.yml/dispatches"
 url = f"{github_api}/repos/pulp/pulp_cookbook{workflow_path}"
 
-for branch in branches[starting:]:
+for branch in branches:
     print(f"Updating {branch}")
     requests.post(url, headers=headers, json={"ref": branch})

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-170-g40698bb
+2021.08.26-179-g8af2847

.github/workflows/scripts/before_script.sh

@@ -29,6 +29,10 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
+# Needed for some functional tests
+cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
+cmd_prefix bash -c "usermod -a -G wheel pulp"
+
 SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings")
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   # Many functional tests require these

.github/workflows/scripts/install.sh

@@ -70,8 +70,10 @@ services:
     volumes:
       - ./settings:/etc/pulp
       - ./ssh:/keys/
-      - ~/.config:/root/.config
+      - ~/.config:/var/lib/pulp/.config
       - ../../../pulp-openapi-generator:/root/pulp-openapi-generator
+    env:
+      PULP_WORKERS: "4"
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
@@ -126,6 +128,12 @@ fi
 
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
+
+# .config needs to be accessible by the pulp user in the container, but some
+# files will likely be modified on the host by post/pre scripts.
+chmod 777 ~/.config/pulp_smash/
+chmod 666 ~/.config/pulp_smash/settings.json
+sudo chown -R 700:700 ~runner/.config
 echo ::group::SSL
 # Copy pulp CA
 sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt

.github/workflows/scripts/script.sh

@@ -64,8 +64,8 @@ fi
 echo "machine pulp
 login admin
 password password
-" | cmd_stdin_prefix bash -c "cat > /root/.netrc"
-cmd_stdin_prefix bash -c "chmod og-rw /root/.netrc"
+" | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc"
+cmd_user_stdin_prefix bash -c "chmod og-rw ~pulp/.netrc"
 
 cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt"
 cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt"
@@ -87,17 +87,17 @@ cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt  | tee -a "$CERTIFI"
 
 # check for any uncommitted migrations
 echo "Checking for uncommitted migrations..."
-cmd_prefix bash -c "django-admin makemigrations --check --dry-run"
+cmd_user_prefix bash -c "django-admin makemigrations --check --dry-run"
 
 # Run unit tests.
-cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_cookbook.tests.unit"
+cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_cookbook.tests.unit"
 
 # Run functional tests
 if [[ "$TEST" == "performance" ]]; then
   if [[ -z ${PERFORMANCE_TEST+x} ]]; then
-    cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_cookbook.tests.performance"
+    cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_cookbook.tests.performance"
   else
-    cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_cookbook.tests.performance.test_$PERFORMANCE_TEST"
+    cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_cookbook.tests.performance.test_$PERFORMANCE_TEST"
   fi
   exit
 fi
@@ -107,20 +107,20 @@ if [ -f $FUNC_TEST_SCRIPT ]; then
 else
 
     if [[ "$GITHUB_WORKFLOW" == "Cookbook Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_cookbook.tests.functional -m parallel -n 8 --nightly"
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_cookbook.tests.functional -m 'not parallel' --nightly"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_cookbook.tests.functional -m parallel -n 8 --nightly"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_cookbook.tests.functional -m 'not parallel' --nightly"
 
 
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8 --nightly"
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'  --nightly"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8 --nightly"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'  --nightly"
 
     else
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_cookbook.tests.functional -m parallel -n 8"
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_cookbook.tests.functional -m 'not parallel'"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_cookbook.tests.functional -m parallel -n 8"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_cookbook.tests.functional -m 'not parallel'"
 
 
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8"
-        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8"
+        cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'"
 
     fi
 

.github/workflows/scripts/utils.sh

@@ -14,7 +14,17 @@ cmd_prefix() {
   docker exec "$PULP_CI_CONTAINER" "$@"
 }
 
+# Run a command as the limited pulp user
+cmd_user_prefix() {
+  docker exec -u pulp "$PULP_CI_CONTAINER" "$@"
+}
+
 # Run a command, and pass STDIN
 cmd_stdin_prefix() {
   docker exec -i "$PULP_CI_CONTAINER" "$@"
 }
+
+# Run a command as the lmited pulp user, and pass STDIN
+cmd_user_stdin_prefix() {
+  docker exec -i -u pulp "$PULP_CI_CONTAINER" "$@"
+}

.github/workflows/update_ci.yml

@@ -54,8 +54,6 @@ jobs:
         id: vars
         run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
 
-
-
       - name: Run update
         run: |
           .github/workflows/scripts/update_ci.sh

template_config.yml

@@ -1,7 +1,7 @@
 # This config represents the latest values used when running the plugin-template. Any settings that
 # were not present before running plugin-template have been added with their default values.
 
-# generated with plugin_template@2021.08.26-170-g40698bb
+# generated with plugin_template@2021.08.26-179-g8af2847
 
 additional_repos: []
 aiohttp_fixtures_origin: 172.18.0.1
@@ -15,6 +15,7 @@ check_stray_pulpcore_imports: true
 cherry_pick_automation: false
 ci_env: {}
 ci_trigger: '{pull_request: {branches: [''*'']}}'
+ci_update_branches: []
 core_import_allowed: []
 coverage: false
 deploy_client_to_pypi: true
@@ -29,8 +30,8 @@ flake8: true
 flake8_ignore: []
 github_org: pulp
 issue_tracker: github
+kanban: true
 keep_ci_update_for_latest_branches: 5
-keep_ci_update_since_branch: null
 lint_requirements: true
 noissue_marker: '[noissue]'
 parallel_test_workers: 8
@@ -50,6 +51,7 @@ publish_docs_to_pulpprojectdotorg: true
 pulp_scheme: https
 pulp_settings: null
 pulp_settings_azure: null
+pulp_settings_s3: null
 pulp_settings_s6: null
 pulp_settings_stream: null
 pulpcore_branch: main