Define YML to modify GitHub workflows & validate loading it with pydantic

doc/changes/unreleased.md

@@ -1,3 +1,7 @@
 # Unreleased
 
 ## Summary
+
+## Feature
+
+* #691: Started customization of PTB workflows by defining the YML schema

exasol/toolbox/config.py

@@ -279,3 +279,20 @@ def github_template_dict(self) -> dict[str, Any]:
             "os_version": self.os_version,
             "python_versions": self.python_versions,
         }
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def github_workflow_patcher_yaml(self) -> Path | None:
+        """
+        For customizing the GitHub workflow templates provided by the PTB,
+        a project can define a `.workflow-patcher.yml` file containing instructions to
+        delete or modify jobs in the PTB template. Modification includes replacing and
+        inserting steps.
+
+        This feature is a work-in-progress that will be completed with:
+            https://github.com/exasol/python-toolbox/issues/690
+        """
+        workflow_patcher_yaml = self.root_path / ".workflow-patcher.yml"
+        if workflow_patcher_yaml.exists():
+            return workflow_patcher_yaml
+        return None

exasol/toolbox/util/workflows/patch_workflow.py

@@ -0,0 +1,59 @@
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+)
+from ruamel.yaml import CommentedMap
+
+from exasol.toolbox.util.workflows.render_yaml import YamlRenderer
+
+
+class ActionType(str, Enum):
+    INSERT_AFTER = "INSERT_AFTER"
+    REPLACE = "REPLACE"
+
+
+class StepContent(BaseModel):
+    model_config = ConfigDict(extra="allow")  # This allows extra fields
+
+    name: str
+    id: str
+    uses: str | None = None
+    run: str | None = None
+    with_: dict[str, Any] | None = Field(None, alias="with")
+    env: dict[str, str] | None = None
+
+
+class StepCustomization(BaseModel):
+    action: ActionType
+    job: str
+    step_id: str
+    content: StepContent
+
+
+class Workflow(BaseModel):
+    name: str
+    remove_jobs: list[str] = Field(default_factory=list)
+    step_customizations: list[StepCustomization] = Field(default_factory=list)
+
+
+class WorkflowPatcherConfig(BaseModel):
+    workflows: list[Workflow]
+
+
+class WorkflowPatcher(YamlRenderer):
+    """
+    The :class:`WorkflowPatcher` enables users to define a YAML file
+    to customize PTB-provided workflows by removing or modifying jobs in the file.
+    A job can be modified by replacing or inserting steps.
+    The provided YAML file must meet the conditions of :class:`WorkflowPatcherConfig`.
+    """
+
+    def get_yaml_dict(self, file_path: Path) -> CommentedMap:
+        loaded_yaml = super().get_yaml_dict(file_path)
+        WorkflowPatcherConfig.model_validate(loaded_yaml)
+        return loaded_yaml

exasol/toolbox/util/workflows/process_template.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+from exasol.toolbox.util.workflows.render_yaml import YamlRenderer
+
+
+class WorkflowRenderer(YamlRenderer):
+    """
+    The :class:`WorkflowRenderer` renders a workflow template provided by the PTB into
+    a final workflow. It renders the final workflow by:
+      - resolving Jinja variables.
+      - standardizing formatting via ruamel.yaml for a consistent output.
+    """
+
+    def render(self, file_path: Path) -> str:
+        """
+        Render the template to the contents of a valid GitHub workflow.
+        """
+        workflow_dict = self.get_yaml_dict(file_path)
+        return self.get_as_string(workflow_dict)

exasol/toolbox/util/workflows/render_yaml.py

@@ -0,0 +1,69 @@
+import io
+from dataclasses import dataclass
+from inspect import cleandoc
+from pathlib import Path
+from typing import Any
+
+from jinja2 import Environment
+from ruamel.yaml import (
+    YAML,
+    CommentedMap,
+)
+
+jinja_env = Environment(
+    variable_start_string="((", variable_end_string="))", autoescape=True
+)
+
+
+@dataclass(frozen=True)
+class YamlRenderer:
+    """
+    The :class:`YamlRenderer` provides a standardised interface for rendering YAML
+    files within the PTB. To simplify configuration and reduce manual coordination,
+    use Jinja variables as defined in :meth:`BaseConfig.github_template_dict` in your
+    YAML files.
+    """
+
+    github_template_dict: dict[str, Any]
+
+    @staticmethod
+    def _get_standard_yaml() -> YAML:
+        """
+        Prepare standard YAML class.
+        """
+        yaml = YAML()
+        yaml.width = 200
+        yaml.preserve_quotes = True
+        yaml.sort_base_mapping_type_on_output = False  # type: ignore
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        return yaml
+
+    def _render_with_jinja(self, input_str: str) -> str:
+        """
+        Render the template with Jinja.
+        """
+        jinja_template = jinja_env.from_string(input_str)
+        return jinja_template.render(self.github_template_dict)
+
+    def get_yaml_dict(self, file_path: Path) -> CommentedMap:
+        """
+        Load a file as a CommentedMap (dictionary form of a YAML), after
+        rendering it with Jinja.
+        """
+        with file_path.open("r", encoding="utf-8") as stream:
+            raw_content = stream.read()
+
+        workflow_string = self._render_with_jinja(raw_content)
+
+        yaml = self._get_standard_yaml()
+        return yaml.load(workflow_string)
+
+    def get_as_string(self, yaml_dict: CommentedMap) -> str:
+        """
+        Output a YAML string.
+        """
+        yaml = self._get_standard_yaml()
+        with io.StringIO() as stream:
+            yaml.dump(yaml_dict, stream)
+            workflow_string = stream.getvalue()
+        return cleandoc(workflow_string)

exasol/toolbox/util/workflows/workflow.py

@@ -6,7 +6,7 @@
     ConfigDict,
 )
 
-from exasol.toolbox.util.workflows.template_processing import TemplateRenderer
+from exasol.toolbox.util.workflows.process_template import WorkflowRenderer
 
 
 class Workflow(BaseModel):
@@ -20,11 +20,10 @@ def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any
             raise FileNotFoundError(file_path)
 
         try:
-            raw_content = file_path.read_text()
-            template_renderer = TemplateRenderer(
-                template_str=raw_content, github_template_dict=github_template_dict
+            workflow_renderer = WorkflowRenderer(
+                github_template_dict=github_template_dict
             )
-            workflow = template_renderer.render_to_workflow()
+            workflow = workflow_renderer.render(file_path=file_path)
             return cls(content=workflow)
         except Exception as e:
             raise ValueError(f"Error rendering file: {file_path}") from e

test/unit/config_test.py

@@ -34,6 +34,7 @@ def test_works_as_defined(test_project_config_factory):
                 "dist",
                 "venv",
             ),
+            "github_workflow_patcher_yaml": None,
             "github_template_dict": {
                 "dependency_manager_version": "2.3.0",
                 "minimum_python_version": "3.10",

test/unit/util/workflows/patch_workflow_test.py

@@ -0,0 +1,90 @@
+from dataclasses import dataclass
+from inspect import cleandoc
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from exasol.toolbox.util.workflows.patch_workflow import (
+    ActionType,
+    WorkflowPatcher,
+)
+from noxconfig import PROJECT_CONFIG
+
+
+@dataclass(frozen=True)
+class ExampleYaml:
+    remove_jobs = """
+        workflows:
+        - name: "checks.yml"
+          remove_jobs:
+            - documentation
+        """
+    step_customization = """
+        workflows:
+        - name: "checks.yml"
+          step_customizations:
+            - action: {action}
+              job: Tests
+              step_id: checkout-repo
+              content:
+                name: SCM Checkout
+                id: checkout-repo
+                uses: actions/checkout@v6
+                with:
+                  fetch-depth: 0
+        """
+
+
+@pytest.fixture
+def workflow_patcher() -> WorkflowPatcher:
+    return WorkflowPatcher(github_template_dict=PROJECT_CONFIG.github_template_dict)
+
+
+@pytest.fixture
+def workflow_patcher_yaml(tmp_path: Path) -> Path:
+    return tmp_path / ".workflow-patcher.yml"
+
+
+class TestWorkflowPatcher:
+    @staticmethod
+    def test_remove_jobs(workflow_patcher_yaml, workflow_patcher):
+        content = cleandoc(ExampleYaml.remove_jobs)
+        workflow_patcher_yaml.write_text(content)
+
+        yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml)
+
+        assert workflow_patcher.get_as_string(yaml_dict) == content
+
+    @staticmethod
+    @pytest.mark.parametrize("action", ActionType)
+    def test_step_customizations(workflow_patcher_yaml, action, workflow_patcher):
+        content = cleandoc(ExampleYaml.step_customization.format(action=action.value))
+        workflow_patcher_yaml.write_text(content)
+
+        yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml)
+
+        assert workflow_patcher.get_as_string(yaml_dict) == content
+
+
+class TestStepCustomization:
+    @staticmethod
+    def test_allows_extra_field(workflow_patcher_yaml, workflow_patcher):
+        content = f"""
+        {ExampleYaml.step_customization.format(action="REPLACE")}
+                  extra-field: "test"
+        """
+        content = cleandoc(content)
+        workflow_patcher_yaml.write_text(content)
+
+        yaml_dict = workflow_patcher.get_yaml_dict(workflow_patcher_yaml)
+
+        assert workflow_patcher.get_as_string(yaml_dict) == content
+
+    @staticmethod
+    def test_raises_error_for_unknown_action(workflow_patcher_yaml, workflow_patcher):
+        content = cleandoc(ExampleYaml.step_customization.format(action="UNKNOWN"))
+        workflow_patcher_yaml.write_text(content)
+
+        with pytest.raises(ValidationError, match="Input should be"):
+            workflow_patcher.get_yaml_dict(workflow_patcher_yaml)