From b4648026cf12b80a8d8575832fbd13d45fb9f934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Wed, 19 Apr 2023 04:12:29 +0300 Subject: [PATCH] CI: Add Renovate config and chart changelog + linting. --- .github/CODEOWNERS | 2 + .github/renovate.json5 | 104 +++++++++++++++ .github/scripts/check-releasenotes.sh | 49 ++++++++ .github/scripts/renovate-releasenotes.py | 153 +++++++++++++++++++++++ .github/scripts/requirements.txt | 5 + .github/workflows/charts-changelog.yaml | 87 +++++++++++++ .github/workflows/charts-lint.yaml | 75 +++++++++++ 7 files changed, 475 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/renovate.json5 create mode 100755 .github/scripts/check-releasenotes.sh create mode 100755 .github/scripts/renovate-releasenotes.py create mode 100644 .github/scripts/requirements.txt create mode 100644 .github/workflows/charts-changelog.yaml create mode 100644 .github/workflows/charts-lint.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..2ab61ad --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners +* @samip5 \ No newline at end of file diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 0000000..a587509 --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,104 @@ +{ + "enabled": true, + "dependencyDashboard": true, + "dependencyDashboardTitle": "Renovate Dashboard", + "assigneesFromCodeOwners": true, + "reviewersFromCodeOwners": true, + "suppressNotifications": ["prIgnoreNotification"], + "prConcurrentLimit": 5, + "helm-values": { + "enabled": false + }, + "helmv3": { + "fileMatch": ["charts/.+/Chart\\.yaml$"] + }, + "packageRules": [ + // Setup datasources + { + "matchDatasources": ["helm"], + "commitMessageTopic": "Helm chart {{depName}}", + "separateMinorPatch": true + }, + // Custom version schemes + { + "matchDatasources": ["github-tags"], + "matchPackageNames": ["potiuk/get-workflow-origin"], + "versioning": "regex:^v(?\\d+)_(?\\d+)(_(?\\d+))?$" + }, + /// + /// Automatically update minor/patch Github Actions + /// + { + "matchManagers": ["github-actions"], + "automerge": true, + "automergeType": "branch", + "matchUpdateTypes": ["minor", "patch"] + }, + // + // Common library dep + // + { + "matchDatasources": ["helm"], + "commitMessagePrefix": "[{{{parentDir}}}]", + "branchTopic": "{{{parentDir}}}-{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}", + "updateTypes": ["major"], + "bumpVersion": "major", + "labels": ["type/major"], + "packageNames": ["common"], + "groupName": ["common library major"] + }, + { + "matchDatasources": ["helm"], + "updateTypes": ["minor"], + "bumpVersion": "minor", + "labels": ["type/minor"], + "packageNames": ["common"], + "groupName": ["common library minor"] + }, + { + "matchDatasources": ["helm"], + "updateTypes": ["patch"], + "bumpVersion": "patch", + "labels": ["type/patch"], + "packageNames": ["common"], + "groupName": ["common library patch"] + }, + // + // Other external chart deps + // + { + "matchDatasources": ["helm"], + "commitMessagePrefix": "[{{{parentDir}}}]", + "branchTopic": "{{{parentDir}}}-{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}", + "updateTypes": ["major"], + "bumpVersion": "major", + "labels": ["type/major"], + "excludePackageNames": ["common"], + "schedule": [ + "every 3 months on the first day of the month" + ] + }, + { + "matchDatasources": ["helm"], + "updateTypes": ["minor"], + "bumpVersion": "minor", + "labels": ["type/minor"], + "excludePackageNames": ["common"], + "groupName": ["external dependency minor"], + "schedule": [ + "every 2 months on the first day of the month" + ] + }, + { + "matchDatasources": ["helm"], + "updateTypes": ["patch"], + "bumpVersion": "patch", + "labels": ["type/patch"], + "excludePackageNames": ["common"], + "groupName": ["external dependency patch"], + "schedule": [ + "every 1 months on the first day of the month" + ] + } + ] +} \ No newline at end of file diff --git a/.github/scripts/check-releasenotes.sh b/.github/scripts/check-releasenotes.sh new file mode 100755 index 0000000..222f60c --- /dev/null +++ b/.github/scripts/check-releasenotes.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +# Check if release notes have been changed +# Usage ./check-releasenotes.sh path + +# require yq +command -v yq >/dev/null 2>&1 || { + printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting." + exit 1 +} + +# Absolute path of repository +repository=$(git rev-parse --show-toplevel) + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ]; then + root="$1" + chart_file="${1}/Chart.yaml" + if [ ! -f "$chart_file" ]; then + printf >&2 "File %s\n does not exist.\n" "${chart_file}" + exit 1 + fi + + cd $root + + if [ -z "$DEFAULT_BRANCH" ]; then + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + fi + + CURRENT=$(cat Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "" ] || [ "$CURRENT" == "null" ]; then + printf >&2 "Changelog annotation has not been set in %s!\n" "$chart_file" + exit 1 + fi + + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + ORIGINAL=$(git show origin/$DEFAULT_BRANCH:./Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "$ORIGINAL" ]; then + printf >&2 "Changelog annotation has not been updated in %s!\n" "$chart_file" + exit 1 + fi +else + printf >&2 "%s\n" "No chart folder has been specified." + exit 1 +fi \ No newline at end of file diff --git a/.github/scripts/renovate-releasenotes.py b/.github/scripts/renovate-releasenotes.py new file mode 100755 index 0000000..3a854a5 --- /dev/null +++ b/.github/scripts/renovate-releasenotes.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +import os +import sys +import typer + +from git import Repo +from loguru import logger +from pathlib import Path + +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.scalarstring import LiteralScalarString +from typing import List + +app = typer.Typer(add_completion=False) + + +def _setup_logging(debug): + """ + Setup the log formatter for this script + """ + + log_level = "INFO" + if debug: + log_level = "DEBUG" + + logger.remove() + logger.add( + sys.stdout, + colorize=True, + format="{message}", + level=log_level, + ) + + +@app.command() +def main( + chart_folders: List[Path] = typer.Argument( + ..., help="Folders containing the chart to process"), + check_branch: str = typer.Option( + None, help="The branch to compare against."), + chart_base_folder: Path = typer.Option( + "charts", help="The base folder where the charts reside."), + debug: bool = False, +): + _setup_logging(debug) + + git_repository = Repo(search_parent_directories=True) + + if check_branch: + logger.info(f"Trying to find branch {check_branch}...") + branch = next( + (ref for ref in git_repository.remotes.origin.refs if ref.name == check_branch), + None + ) + else: + logger.info(f"Trying to determine default branch...") + branch = next( + (ref for ref in git_repository.remotes.origin.refs if ref.name == "origin/HEAD"), + None + ) + + if not branch: + logger.error( + f"Could not find branch {check_branch} to compare against.") + raise typer.Exit(1) + + logger.info(f"Comparing against branch {branch}") + + for chart_folder in chart_folders: + chart_folder = chart_base_folder.joinpath(chart_folder) + if not chart_folder.is_dir(): + logger.error(f"Could not find folder {str(chart_folder)}") + raise typer.Exit(1) + + chart_metadata_file = chart_folder.joinpath('Chart.yaml') + + if not chart_metadata_file.is_file(): + logger.error(f"Could not find file {str(chart_metadata_file)}") + raise typer.Exit(1) + + logger.info(f"Updating changelog annotation for chart {chart_folder}") + + yaml = YAML(typ=['rt', 'string']) + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.explicit_start = True + yaml.preserve_quotes = True + yaml.width = 4096 + + old_chart_metadata = yaml.load( + git_repository.git.show(f"{branch}:{chart_metadata_file}") + ) + new_chart_metadata = yaml.load(chart_metadata_file.read_text()) + + try: + old_chart_dependencies = old_chart_metadata["dependencies"] + except KeyError: + old_chart_dependencies = [] + + try: + new_chart_dependencies = new_chart_metadata["dependencies"] + except KeyError: + new_chart_dependencies = [] + + annotations = [] + for dependency in new_chart_dependencies: + old_dep = None + if "alias" in dependency.keys(): + old_dep = next( + (old_dep for old_dep in old_chart_dependencies if "alias" in old_dep.keys( + ) and old_dep["alias"] == dependency["alias"]), + None + ) + else: + old_dep = next( + (old_dep for old_dep in old_chart_dependencies if old_dep["name"] == dependency["name"]), + None + ) + + add_annotation = False + if old_dep: + if dependency["version"] != old_dep["version"]: + add_annotation = True + else: + add_annotation = True + + if add_annotation: + if "alias" in dependency.keys(): + annotations.append({ + "kind": "changed", + "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']} for alias '{dependency['alias']}'" + }) + else: + annotations.append({ + "kind": "changed", + "description": f"Upgraded `{dependency['name']}` chart dependency to version {dependency['version']}" + }) + + if annotations: + annotations = YAML(typ=['rt', 'string'] + ).dump_to_string(annotations) + + if not "annotations" in new_chart_metadata: + new_chart_metadata["annotations"] = CommentedMap() + + new_chart_metadata["annotations"]["artifacthub.io/changes"] = LiteralScalarString( + annotations) + yaml.dump(new_chart_metadata, chart_metadata_file) + + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt new file mode 100644 index 0000000..646d154 --- /dev/null +++ b/.github/scripts/requirements.txt @@ -0,0 +1,5 @@ +GitPython==3.1.31 +loguru==0.6.0 +ruamel.yaml==0.17.21 +ruamel.yaml.string==0.1.0 +typer==0.7.0 \ No newline at end of file diff --git a/.github/workflows/charts-changelog.yaml b/.github/workflows/charts-changelog.yaml new file mode 100644 index 0000000..612fe9b --- /dev/null +++ b/.github/workflows/charts-changelog.yaml @@ -0,0 +1,87 @@ +name: "Charts: Update README" + +on: + workflow_call: + inputs: + modifiedCharts: + required: true + type: string + isRenovatePR: + required: true + type: string + outputs: + commitHash: + description: "The most recent commit hash at the end of this workflow" + value: ${{ jobs.generate-changelog.outputs.commitHash }} + +jobs: + validate-changelog: + name: Validate changelog + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Check changelog annotations + if: inputs.isRenovatePR != 'true' + run: | + IN_CHARTS=(${{ inputs.modifiedCharts }}) + CHARTS=($(python -c 'import sys;a=sys.argv[1].translate(str.maketrans("","","[]")).split(",");print(" ".join(a))' $IN_CHARTS)) + for i in "${CHARTS[@]}" + do + IFS='/' read -r -a chart_parts <<< "$i" + ./.github/scripts/check-releasenotes.sh "${chart_parts[0]}/${chart_parts[1]}" + echo "" + done + generate-changelog: + name: Generate changelog annotations + runs-on: ubuntu-latest + needs: + - validate-changelog + outputs: + commitHash: ${{ steps.save-commit-hash.outputs.commit_hash }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Annotate Charts.yaml for Renovate PR's + if: inputs.isRenovatePR == 'true' + env: + CHECK_BRANCH: "origin/${{ github.event.repository.default_branch }}" + run: | + pip install -r ./.github/scripts/requirements.txt + IN_CHARTS=(${{ inputs.modifiedCharts }}) + CHARTS=($(python -c 'import sys;a=sys.argv[1].translate(str.maketrans("","","[]")).split(",");print(" ".join(a))' $IN_CHARTS)) + for i in "${CHARTS[@]}" + do + IFS='/' read -r -a chart_parts <<< "$i" + ./.github/scripts/renovate-releasenotes.py --debug --check-branch "$CHECK_BRANCH" "${chart_parts[0]}/${chart_parts[1]}" + echo "" + done + - name: Create commit + id: create-commit + if: inputs.isRenovatePR == 'true' + uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: charts/ + commit_message: "chore: Auto-update chart metadata" + commit_user_name: ${{ github.actor }} + commit_user_email: ${{ github.actor }}@users.noreply.github.com + + - name: Save commit hash + id: save-commit-hash + run: | + if [ "${{ steps.create-commit.outputs.changes_detected || 'unknown' }}" == "true" ]; then + echo '::set-output name=commit_hash::${{ steps.create-commit.outputs.commit_hash }}' + else + echo "::set-output name=commit_hash::${GITHUB_SHA}" + fi \ No newline at end of file diff --git a/.github/workflows/charts-lint.yaml b/.github/workflows/charts-lint.yaml new file mode 100644 index 0000000..3148134 --- /dev/null +++ b/.github/workflows/charts-lint.yaml @@ -0,0 +1,75 @@ +name: "Charts: Lint" + +on: + workflow_call: + inputs: + checkoutCommit: + required: true + type: string + isRenovatePR: + required: true + default: 'false' + type: string + chartsToLint: + description: > + A JSON encoded array of charts to lint + required: true + type: string + +env: + HELM_VERSION: 3.10.2 + +jobs: + lint-chart: + if: ${{ inputs.chartsToLint != '[]' }} + name: Lint chart + strategy: + matrix: + chart: ${{ fromJSON(inputs.chartsToLint) }} + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Verify chart version + uses: bjw-s/helm-charts-actions/verify-chart-version@main + id: verify-chart-version + with: + chart: "charts/${{ matrix.chart }}" + + - name: Verify chart changelog + uses: bjw-s/helm-charts-actions/verify-chart-changelog@main + if: inputs.isRenovatePR != 'true' + id: verify-chart-changelog + with: + chart: "charts/${{ matrix.chart }}" + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.9.3 + with: + setup-tools: | + helmv3 + helm: "${{ env.HELM_VERSION }}" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.4.0 + + - name: Run chart-testing (install) + run: ct lint --config .ci/ct/ct.yaml --charts "charts/${{ matrix.chart }}" + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + lint_success: + needs: + - lint-chart + if: | + always() + name: Lint successful + runs-on: ["self-hosted", "X64"] + steps: + - name: Check lint matrix status + if: ${{ (inputs.chartsToLint != '' && inputs.chartsToLint != '[]') && (needs.lint-chart.result != 'success') }} + run: exit 1 \ No newline at end of file