From 976289d0cfd85139701b26ddd133abdd025a7b5f Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 8 May 2024 03:33:47 -0500 Subject: [PATCH] Add template-files composite action (#127) * Adding initial template-files action * Include requirements.txt for cache key * Use github context instead of envvar * Undo caching * Improve argument validation * Less verbose pip install * Improve error message * Improve errors * Add rich colors * Preserve trailing newlines * Format * Gracefully terminate if no config file * Rework into functions for error handling * Format * Add option to remove files * Better error handling * Fix jsonschema type * Add requirements.txt * Convert GHA version tag to commit hash * Suggestions from code review Co-Authored-By: Jannis Leidel <1610+jezdez@users.noreply.github.com> * Correct src/dst variables * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Specify requirements file for setup-python cache * Manual caching * Correct regex --------- Co-authored-by: Jannis Leidel <1610+jezdez@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- template-files/README.md | 46 ++++++ template-files/action.py | 247 ++++++++++++++++++++++++++++++++ template-files/action.yml | 43 ++++++ template-files/requirements.txt | 5 + 5 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 template-files/README.md create mode 100644 template-files/action.py create mode 100644 template-files/action.yml create mode 100644 template-files/requirements.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8773ad4..4ef67d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: file-contents-sorter files: | (?x)^( - combine-durations/requirements\.txt + .*/requirements\.txt ) args: [--unique] - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks diff --git a/template-files/README.md b/template-files/README.md new file mode 100644 index 0000000..916b209 --- /dev/null +++ b/template-files/README.md @@ -0,0 +1,46 @@ +# Template Files + +A composite GitHub Action to template (or copy) files from other repositories and +commits them to the specified PR. + +## GitHub Action Usage + +In your GitHub repository include this action in your workflows: + +```yaml +- uses: conda/actions/template-files + with: + # [optional] + # the path to the configuration file + config: .github/template-files/config.yml + + # [optional] + # the path to the template stubs + stubs: .github/template-files/templates/ + + # [optional] + # the GitHub token with API access + token: ${{ github.token }} +``` + +Define what files to template in a configuration file, e.g., `.github/templates/config.yml`: + +```yaml +user/repo: + # copy to same path + - path/to/file + - src: path/to/file + + # copy to different path + - src: path/to/other + dst: path/to/another + + # templating + - src: path/to/template + with: + name: value + + # removing + - dst: path/to/remove + remove: true +``` diff --git a/template-files/action.py b/template-files/action.py new file mode 100644 index 0000000..0c24496 --- /dev/null +++ b/template-files/action.py @@ -0,0 +1,247 @@ +"""Copy files from external locations as defined in `sync.yml`.""" +from __future__ import annotations + +import os +import sys +from argparse import ArgumentParser, ArgumentTypeError, Namespace +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml +from github import Auth, Github, UnknownObjectException +from github.Repository import Repository +from jinja2 import Environment, FileSystemLoader +from jsonschema import validate +from rich.console import Console + +if TYPE_CHECKING: + from typing import Any, Literal + +print = Console(color_system="standard", soft_wrap=True).print +perror = Console( + color_system="standard", + soft_wrap=True, + stderr=True, + style="bold red", +).print + + +class ActionError(Exception): + pass + +def validate_file(value: str) -> Path | None: + try: + path = Path(value).expanduser().resolve() + path.read_text() + return path + except (IsADirectoryError, PermissionError) as err: + # IsADirectoryError: value is a directory, not a file + # PermissionError: value is not readable + raise ArgumentTypeError(f"{value} is not a valid file: {err}") + except FileNotFoundError: + # FileNotFoundError: value does not exist + return None + + +def validate_dir(value: str) -> Path: + try: + path = Path(value).expanduser().resolve() + path.mkdir(parents=True, exist_ok=True) + ignore = path / ".ignore" + ignore.touch() + ignore.unlink() + return path + except (FileExistsError, PermissionError) as err: + # FileExistsError: value is a file, not a directory + # PermissionError: value is not writable + raise ArgumentTypeError(f"{value} is not a valid directory: {err}") + + +def parse_args() -> Namespace: + # parse CLI for inputs + parser = ArgumentParser() + parser.add_argument("--config", type=validate_file, required=True) + parser.add_argument("--stubs", type=validate_dir, required=True) + return parser.parse_args() + + +def read_config(args: Namespace) -> dict: + # read and validate configuration file + config = yaml.load( + args.config.read_text(), + Loader=yaml.SafeLoader, + ) + validate( + config, + schema={ + "type": "object", + "patternProperties": { + r"\w+/\w+": { + "type": "array", + "items": { + "type": ["string", "object"], + "minLength": 1, + "properties": { + "src": {"type": "string"}, + "dst": {"type": "string"}, + "remove": {"type": "boolean"}, + "with": { + "type": "object", + "patternProperties": { + r"\w+": {"type": "string"}, + }, + }, + }, + }, + } + }, + }, + ) + return config + + +def parse_config(file: str | dict) -> tuple[str | None, Path, bool, dict[str, Any]]: + src: str | None + dst: Path + remove: bool + context: dict[str, Any] + + if isinstance(file, str): + src = file + dst = Path(file) + remove = False + context = {} + elif isinstance(file, dict): + src = file.get("src", None) + if (tmp := file.get("dst", src)) is None: + perror(f"❌ Invalid file definition ({file}), expected dst") + raise ActionError + dst = Path(tmp) + remove = file.get("remove", False) + context = file.get("with", {}) + else: + perror(f"❌ Invalid file definition ({file}), expected str or dict") + raise ActionError + + # to template a file we need a source file + if not remove and src is None: + perror(f"❌ Invalid file definition ({file}), expected src") + raise ActionError + + return src, dst, remove, context + + +def iterate_config( + config: dict, + gh: Github, + env: Environment, + current_repo: Repository, +) -> int: + # iterate over configuration and template files + errors = 0 + for upstream_name, files in config.items(): + try: + upstream_repo = gh.get_repo(upstream_name) + except UnknownObjectException as err: + perror(f"❌ Failed to fetch {upstream_name}: {err}") + errors += 1 + continue + + for file in files: + try: + # parse/standardize configuration + src, dst, remove, context = parse_config(file) + except ActionError: + errors += 1 + continue + + # remove dst file + if remove: + try: + dst.unlink() + except FileNotFoundError: + # FileNotFoundError: dst does not exist + print(f"⚠️ {dst} has already been removed") + except PermissionError as err: + # PermissionError: not possible to remove dst + perror(f"❌ Failed to remove {dst}: {err}") + errors += 1 + continue + else: + print(f"✅ Removed {dst}") + else: + # fetch src file + try: + content = upstream_repo.get_contents(src).decoded_content.decode() + except UnknownObjectException as err: + perror(f"❌ Failed to fetch {src} from {upstream_name}: {err}") + errors += 1 + continue + else: + # inject stuff about the source and destination + context.update({ + # the current repository from which this GHA is being run, + # where the new files will be written + "repo": current_repo, + "dst": current_repo, + "destination": current_repo, + "current": current_repo, + # source (should be rarely, if ever, used in templating) + "src": upstream_repo, + "source": upstream_repo, + }) + + template = env.from_string(content) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(template.render(**context)) + + print(f"✅ Templated {upstream_name}/{src} as {dst}") + + return errors + + +def main(): + errors = 0 + + args = parse_args() + if not args.config: + print("⚠️ No configuration file found, nothing to update") + sys.exit(0) + + config = read_config(args) + + # initialize Jinja environment and GitHub client + env = Environment( + loader=FileSystemLoader(args.stubs), + # {{ }} is used in MermaidJS + # ${{ }} is used in GitHub Actions + # { } is used in Python + # %( )s is used in Python + block_start_string="[%", + block_end_string="%]", + variable_start_string="[[", + variable_end_string="]]", + comment_start_string="[#", + comment_end_string="#]", + keep_trailing_newline=True, + ) + gh = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"])) + + # get current repository + current_name = os.environ["GITHUB_REPOSITORY"] + try: + current_repo = gh.get_repo(current_name) + except UnknownObjectException as err: + perror(f"❌ Failed to fetch {current_name}: {err}") + errors += 1 + + if not errors: + errors += iterate_config(config, gh, env, current_repo) + + if errors: + perror(f"Got {errors} error(s)") + sys.exit(errors) + + +if __name__ == "__main__": + main() diff --git a/template-files/action.yml b/template-files/action.yml new file mode 100644 index 0000000..b2cc91c --- /dev/null +++ b/template-files/action.yml @@ -0,0 +1,43 @@ +name: Template Files +description: Template (or copy) files from other repositories and commits them to the specified PR. +author: Anaconda Inc. +branding: + icon: book-open + color: green + +inputs: + config: + description: Configuration path defining what files to template/copy. + default: .github/template-files/config.yml + stubs: + description: >- + Path to where stub files are located in the current repository. + default: .github/template-files/templates/ + token: + description: >- + A token with ability to comment, label, and modify the commit status + (`pull_request: write` and `statuses: write` for fine-grained PAT; `repo` for classic PAT) + default: ${{ github.token }} + +runs: + using: composite + steps: + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ~/.cache/pip + # invalidate the cache anytime a workflow changes + key: ${{ hashFiles('.github/workflows/*') }} + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: '3.11' + + - name: install dependencies + shell: bash + run: pip install --quiet -r ${{ github.action_path }}/requirements.txt + + - name: sync & template files + shell: bash + run: python ${{ github.action_path }}/action.py --config ${{ inputs.config }} --stubs ${{ inputs.stubs }} + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/template-files/requirements.txt b/template-files/requirements.txt new file mode 100644 index 0000000..2ecf0cc --- /dev/null +++ b/template-files/requirements.txt @@ -0,0 +1,5 @@ +jinja2 +jsonschema +pygithub +pyyaml +rich