diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f0c5bb5e..f095c16c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -47,29 +47,37 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump2version --dry-run --list release + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65af1fd4..8057b54d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,7 +80,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible~=10.3.0 mkdocs jq jsonpatch + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_catdog/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: diff --git a/plugin-template b/plugin-template index 3d7343b3..30101770 100755 --- a/plugin-template +++ b/plugin-template @@ -424,17 +424,19 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): except Exception: gitref = "unknown" + setup_py = os.path.exists(os.path.join(plugin_root_dir, "setup.py")) ci_update_branches = [config["plugin_default_branch"]] latest_release_branch = config["latest_release_branch"] if latest_release_branch is not None: if latest_release_branch not in config["supported_release_branches"]: ci_update_branches.append(latest_release_branch) ci_update_branches.extend(config["supported_release_branches"]) - ci_update_hour = sum((ord(c) for c in config["plugin_app_label"])) % 24 + template_vars = { "section": name, "gitref": gitref, + "setup_py": setup_py, "ci_update_branches": ci_update_branches, "python_version": "3.11", "ci_update_hour": ci_update_hour, diff --git a/templates/bootstrap/.bumpversion.cfg.j2 b/templates/bootstrap/.bumpversion.cfg.j2 index a034cb4c..267debd0 100644 --- a/templates/bootstrap/.bumpversion.cfg.j2 +++ b/templates/bootstrap/.bumpversion.cfg.j2 @@ -14,7 +14,7 @@ values = dev prod -"[bumpversion:file:./{{ plugin_name | snake }}/app/__init__.py]" +[bumpversion:file:./{{ plugin_name | snake }}/app/__init__.py] [bumpversion:file:./setup.py] diff --git a/templates/github/.ci/scripts/calc_constraints.py.j2 b/templates/github/.ci/scripts/calc_constraints.py.j2 index 77253a95..a9aa42fb 100644 --- a/templates/github/.ci/scripts/calc_constraints.py.j2 +++ b/templates/github/.ci/scripts/calc_constraints.py.j2 @@ -8,6 +8,11 @@ from packaging.requirements import Requirement from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -54,11 +59,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -95,18 +100,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/templates/github/.ci/scripts/check_release.py.j2 b/templates/github/.ci/scripts/check_release.py.j2 index 8d6957b5..1d680923 100755 --- a/templates/github/.ci/scripts/check_release.py.j2 +++ b/templates/github/.ci/scripts/check_release.py.j2 @@ -81,6 +81,13 @@ def main(): ) if req_txt_diff: reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) if reasons: curr_version = Version(last_tag) diff --git a/templates/github/.ci/scripts/check_requirements.py.j2 b/templates/github/.ci/scripts/check_requirements.py.j2 index 165076d2..6eb9f36e 100644 --- a/templates/github/.ci/scripts/check_requirements.py.j2 +++ b/templates/github/.ci/scripts/check_requirements.py.j2 @@ -1,10 +1,12 @@ {% include "header.j2" %} +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -15,17 +17,33 @@ CHECK_MATRIX = [ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -44,18 +62,21 @@ for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: and req.name != "{{ plugin_name | dash }}-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass - -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + + +if __name__ == "__main__": + main() diff --git a/templates/github/.github/workflows/build.yml.j2 b/templates/github/.github/workflows/build.yml.j2 index 1dfff3e5..a49def96 100644 --- a/templates/github/.github/workflows/build.yml.j2 +++ b/templates/github/.github/workflows/build.yml.j2 @@ -22,7 +22,7 @@ jobs: {{ checkout(path=plugin_name) | indent(6) }} {{ checkout(repository="pulp/pulp-openapi-generator", path="pulp-openapi-generator") | indent(6) }} {{ setup_python() | indent(6) }} - {{ install_python_deps(["packaging", "twine", "wheel", "mkdocs", "jq"]) | indent(6) }} + {{ install_python_deps(["build", "packaging", "twine", "wheel", "mkdocs", "jq"]) | indent(6) }} {%- if os_required_packages %} - name: "Install OS packages" run: | @@ -30,7 +30,7 @@ jobs: {%- endif %} - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/templates/github/.github/workflows/lint.yml.j2 b/templates/github/.github/workflows/lint.yml.j2 index 6b0ab75e..dd88d444 100644 --- a/templates/github/.github/workflows/lint.yml.j2 +++ b/templates/github/.github/workflows/lint.yml.j2 @@ -15,7 +15,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: {{ checkout(path=plugin_name) | indent(6) }} @@ -24,13 +24,17 @@ jobs: {{ install_python_deps(["-r", "lint_requirements.txt"]) | indent(6) }} - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + + - name: "Verify bump version config" + run: | + bump2version --dry-run --list release {%- if black %} # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . @@ -38,25 +42,29 @@ jobs: {%- if flake8 %} # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 {%- endif %} - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh {%- if check_manifest %} - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest {%- endif %} {%- if check_stray_pulpcore_imports %} - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh {%- endif %} {%- if check_gettext %} - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh {%- endif %} diff --git a/templates/github/.github/workflows/scripts/before_install.sh.j2 b/templates/github/.github/workflows/scripts/before_install.sh.j2 index b7002d63..909a6915 100755 --- a/templates/github/.github/workflows/scripts/before_install.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_install.sh.j2 @@ -25,7 +25,11 @@ fi COMMIT_MSG=$(git log --format=%B --no-merges -1) export COMMIT_MSG +{% if setup_py -%} COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) +{%- else -%} +COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") +{%- endif %} mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml @@ -40,11 +44,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "{{ plugin_default_branch }}" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml @@ -61,10 +60,10 @@ then fi if [[ "$TEST" = "pulp" ]]; then - python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/templates/github/lint_requirements.txt.j2 b/templates/github/lint_requirements.txt.j2 index 53bbd687..251bcf8f 100644 --- a/templates/github/lint_requirements.txt.j2 +++ b/templates/github/lint_requirements.txt.j2 @@ -4,6 +4,7 @@ {% if black -%} black==24.3.0 {% endif -%} +bump2version check-manifest flake8 {% if black -%}