plugin-template

#!/usr/bin/env python3

import argparse
import os
import pprint
import shlex
import shutil
import subprocess
import sys
import textwrap
from pathlib import Path

import yaml
from jinja2 import Environment, FileSystemLoader

import utils

DEFAULT_SETTINGS = {
    "api_root": "/pulp/",
    "black": True,
    "check_commit_message": True,
    "check_gettext": True,
    "check_manifest": True,
    "check_stray_pulpcore_imports": True,
    "cli_package": "pulp-cli",
    "cli_repo": "https://github.com/pulp/pulp-cli.git",
    "ci_base_image": "ghcr.io/pulp/pulp-ci-centos9",
    "ci_env": {},
    "ci_trigger": "{pull_request: {branches: ['*']}}",
    "core_import_allowed": [],
    "deploy_client_to_pypi": True,
    "deploy_client_to_rubygems": True,
    "deploy_to_pypi": True,
    "disabled_redis_runners": [],
    "docker_fixtures": False,
    "flake8_ignore": [],
    "flake8": True,
    "github_org": "pulp",
    "latest_release_branch": None,
    "lint_requirements": True,
    "os_required_packages": [],
    "parallel_test_workers": 8,
    "plugin_app_label": None,
    "plugin_default_branch": "main",
    "plugin_name": None,
    "plugins": None,
    "post_job_template": None,
    "pre_job_template": None,
    "pulp_env": {},
    "pulp_env_azure": {},
    "pulp_env_gcp": {},
    "pulp_env_s3": {},
    "pulp_scheme": "https",
    "pulp_settings_azure": {"domain_enabled": True},
    "pulp_settings_gcp": None,
    "pulp_settings": None,
    "pulp_settings_s3": None,
    "pydocstyle": True,
    "release_email": "pulp-infra@redhat.com",
    "release_user": "pulpbot",
    "stalebot_days_until_close": 30,
    "stalebot_days_until_stale": 90,
    "stalebot_limit_to_pulls": True,
    "stalebot": True,
    "supported_release_branches": [],
    "sync_ci": True,
    "test_azure": False,
    "test_cli": False,
    "test_deprecations": True,
    "test_gcp": False,
    "test_lowerbounds": True,
    "test_performance": False,
    "test_reroute": True,
    "test_s3": False,
    "test_storages_compat_layer": False,
    "use_issue_template": True,
}


DEPRECATED_FILES = {
    "github": [
        ".bumpversion.cfg",
        ".ci/assets/bindings/.gitkeep",
        ".ci/scripts/changelog.py",
        ".ci/scripts/calc_deps_lowerbounds.py",
        ".ci/scripts/cherrypick.sh",
        ".ci/scripts/docs-builder.py",
        ".ci/scripts/publish_docs.sh",
        ".ci/scripts/publish_client_gem.sh",
        ".ci/scripts/publish_client_pypi.sh",
        ".ci/scripts/publish_plugin_pypi.sh",
        ".ci/scripts/redmine.py",
        ".ci/scripts/release_requirements.txt",
        ".ci/scripts/release.py",
        ".ci/scripts/tweet.py",
        ".ci/scripts/update_ci_branches.py",
        ".ci/scripts/update_github.sh",
        ".ci/scripts/update_redmine.sh",
        ".ci/scripts/upper_bound.py",
        ".github/workflows/kanban.yml",
        ".github/workflows/changelog.yml",
        ".github/workflows/fips.yml",
        ".github/workflows/scripts/create_release_from_tag.sh",
        ".github/workflows/scripts/install_python_client.sh",
        ".github/workflows/scripts/install_ruby_client.sh",
        ".github/workflows/scripts/release.py",
        ".github/workflows/scripts/stage-changelog-for-master.py",
        ".github/workflows/scripts/post_docs_test.sh",
        ".github/workflows/scripts/update_ci.sh",
        ".github/workflows/scripts/docs-publisher.py",
        ".github/workflows/scripts/publish_docs.sh",
        ".travis",
        ".travis.yml",
        "flake8.cfg",
        "CHANGES/.TEMPLATE.rst",
    ],
}


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description="Create or update a plugin using the current template.",
    )
    parser.add_argument(
        "plugin_name",
        type=str,
        help=textwrap.dedent(
            """\
                Create or update this plugin. The name should start with pulp- or pulp_.

            """
        ),
    )
    parser.add_argument(
        "--plugin-app-label",
        type=str,
        help=textwrap.dedent(
            """\
                the Django app label for the plugin - usually the part after pulp_ or pulp-.

            """
        ),
    )
    parser.add_argument(
        "--generate-config",
        action="store_true",
        help=textwrap.dedent(
            """\
                Create or update a plugin template config for a plugin and exit.

            """
        ),
    )
    parser.add_argument(
        "--bootstrap",
        action="store_true",
        help=textwrap.dedent(
            """\
                Create a new plugin and template boilerplate code.

            """
        ),
    )
    parser.add_argument(
        "--test",
        action="store_true",
        help=textwrap.dedent(
            """\
                Generate or update functional and unit tests.

            """
        ),
    )
    parser.add_argument(
        "--github",
        action="store_true",
        help=textwrap.dedent(
            """\
                Generate or update github CI/CD configuration files.

            """
        ),
    )
    parser.add_argument(
        "--all",
        action="store_true",
        help=textwrap.dedent(
            """\
                Create a new plugin and template all non-excluded files.

            """
        ),
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help=textwrap.dedent(
            """\
                Include more output.

            """
        ),
    )
    parser.add_argument(
        "--latest-release-branch",
        metavar="VERSION_BRANCH",
        action="store",
        help=textwrap.dedent(
            """\
                Mark specified version as the latest_release_branch before templating.

            """
        ),
    )
    args = parser.parse_args()

    here = os.path.dirname(os.path.abspath(__file__))
    plugin_root_dir = os.path.join(os.path.dirname(here), args.plugin_name)
    plugin_config_path = os.path.join(plugin_root_dir, "template_config.yml")
    write_new_config = False

    try:
        with open(plugin_config_path) as config_file:
            try:
                config_in_file = yaml.safe_load(config_file)
                if config_in_file:
                    config = config_in_file
                    # Add any missing value from the list of defaults
                    for key, value in DEFAULT_SETTINGS.items():
                        if key not in config:
                            config[key] = value
                            write_new_config = True
                    # remove deprecated options
                    for key in set(config.keys()) - set(DEFAULT_SETTINGS.keys()):
                        config.pop(key)
                        write_new_config = True
                    # TODO: validate config
                    # validate versions are strings
                    assert all(
                        (
                            isinstance(version, str)
                            for version in config["supported_release_branches"]
                        )
                    )
                    print(
                        "\nLoaded plugin template config from "
                        "{path}/template_config.yml.\n".format(path=plugin_root_dir)
                    )

                    if args.verbose:
                        print("\nUsing the following config:\n")
                        pp = pprint.PrettyPrinter(indent=4)
                        pp.pprint(config)
                        print("\n")
            except yaml.YAMLError as exc:
                print(exc)
                return 2
    except FileNotFoundError:
        print(
            "\nCould not find a plugin template config at {path}/template_config.yml.\n".format(
                path=plugin_root_dir
            )
        )
        if args.all or args.generate_config:
            if not args.plugin_app_label:
                print(
                    "\n--plugin-app-label needs to be specified when generating a template "
                    "config. See ./plugin-template -h for usage.\n"
                )
                return 2
            else:
                config = generate_config(args.plugin_name, args.plugin_app_label)
        else:
            return 2

    if args.latest_release_branch:
        config["latest_release_branch"] = str(args.latest_release_branch)
        write_new_config = True

    # Config key is used by the template_config.yml.j2 template to dump
    # the config. (note: uses .copy() to avoid a self reference)
    config["config"] = config.copy()

    sections = [
        section
        for section in ["generate_config", "bootstrap", "github", "test"]
        if getattr(args, section) or args.all
    ]
    for section in sections:
        write_template_section(config, section, plugin_root_dir, verbose=args.verbose)

    if write_new_config and not (args.generate_config or args.all):
        write_template_section(config, "generate_config", plugin_root_dir, verbose=args.verbose)
        file_path = os.path.join(plugin_root_dir, "template_config.yml")
        print("\nAn updated plugin template config written to {path}.\n".format(path=file_path))

    if "github" in sections:
        migrate_dummy(plugin_root_dir)

    if plugin_root_dir:
        print("\nDeprecation check:")
        check_for_deprecated_files(plugin_root_dir, sections)


def migrate_dummy(plugin_root_dir):
    pass


def write_template_section(config, name, plugin_root_dir, verbose=False):
    """
    Template or copy all files for the section.
    """
    plugin_root_path = Path(plugin_root_dir)
    section_template_dir = f"templates/{name}"
    env = Environment(
        loader=FileSystemLoader(
            [
                section_template_dir,  # The specified section folder
                "templates",  # The default templates folder
                "../",  # The parent dir to allow including pre/post templates from
            ]
        ),
        line_statement_prefix="%%",
        line_comment_prefix="%#",
    )
    env.filters["camel"] = utils.to_camel
    env.filters["caps"] = utils.to_caps
    env.filters["dash"] = utils.to_dash
    env.filters["snake"] = utils.to_snake
    env.filters["to_yaml"] = utils.to_nice_yaml
    env.filters["from_yaml"] = utils.from_yaml
    env.filters["shquote"] = shlex.quote

    files_templated = 0
    files_copied = 0

    try:
        gitref = subprocess.check_output(["git", "describe", "--dirty"]).decode().strip()
    except Exception:
        gitref = "unknown"

    template_vars = {
        "section": name,
        "gitref": gitref,
        "setup_py": (plugin_root_path / "setup.py").exists(),
        "ci_update_branches": utils.ci_update_branches(config),
        "python_version": "3.11",
        "ci_update_hour": sum((ord(c) for c in config["plugin_app_label"])) % 24,
        "current_version": utils.current_version(plugin_root_path),
        "is_pulpdocs_member": config["plugin_name"] in utils.get_pulpdocs_members(),
        **config,
    }

    for relative_path in generate_relative_path_set(section_template_dir):
        if not config["stalebot"] and "stale" in relative_path:
            continue
        if config["use_issue_template"] is False and "ISSUE_TEMPLATE" in relative_path:
            continue
        destination_relative_path = relative_path.replace(
            "plugin_name", utils.to_snake(config["plugin_name"])
        )
        necessary_dir_structure = os.path.dirname(
            os.path.join(plugin_root_dir, destination_relative_path)
        )

        if not os.path.exists(necessary_dir_structure):
            os.makedirs(necessary_dir_structure)

        if relative_path.endswith(".j2"):
            template = env.get_template(relative_path)
            destination = destination_relative_path[: -len(".j2")]
            if destination.startswith("pyproject.toml."):
                utils.merge_toml(
                    template,
                    plugin_root_path,
                    destination,
                    template_vars,
                )
            else:
                utils.template_to_file(
                    template,
                    plugin_root_path,
                    destination,
                    template_vars,
                )
            files_templated += 1
            if verbose:
                print(f"Templated file: {relative_path}")
        else:
            if destination_relative_path.endswith(".copy"):
                destination_relative_path = destination_relative_path[: -len(".copy")]
            shutil.copy(
                os.path.join(section_template_dir, relative_path),
                os.path.join(plugin_root_dir, destination_relative_path),
            )
            files_copied += 1
            if verbose:
                print(f"Copied file: {relative_path}")

    print(f"Section: {name} \n    templated: {files_templated}\n    copied: {files_copied}")
    return 0


def generate_relative_path_set(root_dir):
    """
    Create a set of relative paths within the specified directory.
    """
    applicable_paths = set()
    for root, dirs, files in os.walk(root_dir, topdown=False):
        for file_name in files:
            template_abs_path = os.path.join(root, file_name)
            template_relative_path = os.path.relpath(template_abs_path, root_dir)
            applicable_paths.add(template_relative_path)
    return applicable_paths


def generate_config(plugin_name, plugin_app_label):
    """
    Generates a default config for a plugin

    Args:
        plugin_name (str): the full name of the plugin repository name (e.g. pulp_file)
        plugin_app_label (str): the Django app label for the plugin - usually the part after
            pulp_ or pulp-

    Returns:
        config (dict): a dictionary containing a default template configuration for the plugin
    """
    config = DEFAULT_SETTINGS.copy()
    config["plugin_name"] = plugin_name
    config["plugin_app_label"] = plugin_app_label
    config["plugins"] = [{"name": plugin_name, "app_label": plugin_app_label}]
    return config


def check_for_deprecated_files(plugin_root_dir, sections):
    """Check for files that have been deprecated (ie moved or removed)."""
    files_found = False
    for section in sections:
        for fp in DEPRECATED_FILES.get(section, []):
            path = Path(plugin_root_dir) / fp
            if path.exists():
                if path.is_dir():
                    shutil.rmtree(path)
                    print(f"Removed deprecated directory: '{path}'.")
                else:
                    path.unlink()
                    print(f"Removed deprecated file: '{path}'.")
                files_found = True

    if not files_found:
        print("No deprecated files found.")


if __name__ == "__main__":
    sys.exit(main())