From c641943535395b85508caadd9d02eef9b22fed74 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 14 Jan 2024 20:50:52 +0100 Subject: [PATCH 001/357] add translation actions --- .github/workflows/auto_translate.yml | 43 ++++++ .github/workflows/propose_translation.yml | 58 ++++++++ .github/workflows/publish_alpha.yml | 122 ++++++++++++++++ scripts/bump_alpha.py | 21 +++ scripts/bump_build.py | 24 ++++ scripts/bump_major.py | 30 ++++ scripts/bump_minor.py | 27 ++++ scripts/prepare_skillstore.py | 83 +++++++++++ scripts/release_skillstore.py | 46 +++++++ scripts/remove_alpha.py | 15 ++ scripts/translate.py | 161 ++++++++++++++++++++++ 11 files changed, 630 insertions(+) create mode 100644 .github/workflows/auto_translate.yml create mode 100644 .github/workflows/propose_translation.yml create mode 100644 .github/workflows/publish_alpha.yml create mode 100644 scripts/bump_alpha.py create mode 100644 scripts/bump_build.py create mode 100644 scripts/bump_major.py create mode 100644 scripts/bump_minor.py create mode 100644 scripts/prepare_skillstore.py create mode 100644 scripts/release_skillstore.py create mode 100644 scripts/remove_alpha.py create mode 100644 scripts/translate.py diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml new file mode 100644 index 000000000..ca46688f7 --- /dev/null +++ b/.github/workflows/auto_translate.yml @@ -0,0 +1,43 @@ +name: Auto translate +on: + workflow_call: + inputs: + branch: + type: string + default: ${{ github.ref }} + locale_folder: + type: string + default: "locale" + +jobs: + autotranslate: + env: + API_KEY: ${{secrets.DL_API_KEY}} + LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v3 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Translate Tools + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + - name: Auto Translate + run: | + python action/github/scripts/translate.py + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: autotranslate + repository: action/package/ diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml new file mode 100644 index 000000000..de71f2ba2 --- /dev/null +++ b/.github/workflows/propose_translation.yml @@ -0,0 +1,58 @@ +name: Propose Translation +on: + workflow_call: + inputs: + branch: + type: string + default: ${{ github.ref_name }} + language: + type: string + default: "" + locale_folder: + type: string + default: "locale" + reviewer: + type: string + default: "emphasize" + + +jobs: + Propose_translation: + env: + TARGET_LANG: ${{ inputs.language }} + API_KEY: ${{ secrets.DL_API_KEY }} + LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v3 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Translate Tools + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + - name: Run Translate Script + run: | + python action/github/scripts/translate.py + cd ${{ github.workspace }}/action/package/ + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: autotranslate + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: Proposed ${{ inputs.language }} Translations + body: Translations for review + labels: translation + branch: staging/translation_${{ inputs.language }} + reviewers: ${{ inputs.reviewer }} diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml new file mode 100644 index 000000000..712c8d931 --- /dev/null +++ b/.github/workflows/publish_alpha.yml @@ -0,0 +1,122 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Alpha Build ...aX +on: + workflow_call: + inputs: + branch: + type: string + default: ${{ github.ref }} + version_file: + type: string + default: version.py + locale_folder: + type: string + default: "locale" + +jobs: + autotranslate: + runs-on: ubuntu-latest + env: + API_KEY: ${{secrets.DL_API_KEY}} + LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v3 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - uses: dorny/paths-filter@v2 + id: filter + with: + working-directory: action/package/ + filters: | + locales: + - '${{ inputs.locale_folder }}/en-us/**' + - '${{ inputs.locale_folder }}/../dialog/en-us/**' + - '${{ inputs.locale_folder }}/../vocab/en-us/**' + - name: Setup Python + if: steps.filter.outputs.locales == 'true' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Auto Translate + if: steps.filter.outputs.locales == 'true' + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + python action/github/scripts/translate.py + - name: Commit to dev + if: steps.filter.outputs.locales == 'true' + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: autotranslate + repository: action/package/ + + build_and_publish: + runs-on: ubuntu-latest + env: + VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v3 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Increment Version + id: version + run: | + python action/github/scripts/bump_alpha.py + echo "::set-output name=version::$(python action/package/setup.py --version)" + - name: "Generate release changelog" + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + output: action/package/ + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version to ${{ steps.version.outputs.version }} + repository: action/package/ + - name: Change working directory to release + run: cd ${{ github.workspace }}/action/package/ + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: true + commitish: ${{ inputs.branch }} + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Build Distribution Packages + run: | + cd action/package + python setup.py sdist bdist_wheel + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} + packages-dir: action/package/dist/ diff --git a/scripts/bump_alpha.py b/scripts/bump_alpha.py new file mode 100644 index 000000000..37e562c37 --- /dev/null +++ b/scripts/bump_alpha.py @@ -0,0 +1,21 @@ +import fileinput +import os + + +version_file = os.getenv("VERSION_FILE") +if not os.path.isfile(version_file): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") + +version_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_build.py b/scripts/bump_build.py new file mode 100644 index 000000000..f7f7091a9 --- /dev/null +++ b/scripts/bump_build.py @@ -0,0 +1,24 @@ +import fileinput +import os + + +version_file = os.getenv("VERSION_FILE") +if not os.path.isfile(version_file): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") + +version_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_major.py b/scripts/bump_major.py new file mode 100644 index 000000000..52f750fa8 --- /dev/null +++ b/scripts/bump_major.py @@ -0,0 +1,30 @@ +import fileinput +import os + + +version_file = os.getenv("VERSION_FILE") +if not os.path.isfile(version_file): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") + +version_var_name = "VERSION_MAJOR" +minor_var_name = "VERSION_MINOR" +build_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(minor_var_name): + print(f"{minor_var_name} = 0") + elif line.startswith(build_var_name): + print(f"{build_var_name} = 0") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_minor.py b/scripts/bump_minor.py new file mode 100644 index 000000000..fe61cc1a3 --- /dev/null +++ b/scripts/bump_minor.py @@ -0,0 +1,27 @@ +import fileinput +import os + + +version_file = os.getenv("VERSION_FILE") +if not os.path.isfile(version_file): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") + +version_var_name = "VERSION_MINOR" +build_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(build_var_name): + print(f"{build_var_name} = 0") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/prepare_skillstore.py b/scripts/prepare_skillstore.py new file mode 100644 index 000000000..f289e4f2a --- /dev/null +++ b/scripts/prepare_skillstore.py @@ -0,0 +1,83 @@ +import os +import json +from ovos_utils.bracket_expansion import expand_parentheses, expand_options +from ovos_skills_manager import SkillEntry + + +branch = os.getenv("BRANCH") +if not branch: + raise ValueError("environment variable `BRANCH` not set") +repo = os.getenv("REPO") +if not repo: + raise ValueError("environment variable `REPO` not set") +author = "OpenVoiceOS" + +url = f"https://github.com/{author}/{repo}@{branch}" + +skill = SkillEntry.from_github_url(url) +tmp_skills = "/tmp/osm_installed_skills" +skill_folder = f"{tmp_skills}/{skill.uuid}" + +BASE_FOLDER = os.getenv("BASE_FOLDER") +if not BASE_FOLDER: + raise ValueError("environment variable `LOCALE_FOLDER` not set") +elif not os.path.exists(BASE_FOLDER): + raise ValueError(f"environment variable `LOCALE_FOLDER` is not a folder: {BASE_FOLDER}") + + +desktop_dir = os.path.join(BASE_FOLDER, "res", "desktop") +android_ui = os.path.join(BASE_FOLDER, "ui", "+android") +os.makedirs(desktop_dir, exist_ok=True) + +readme = os.path.join(BASE_FOLDER, "readme.md") +jsonf = os.path.join(desktop_dir, "skill.json") +desktopf = os.path.join(desktop_dir, f"{repo}.desktop") +skill_code = os.path.join(BASE_FOLDER, "__init__.py") + +res_folder = os.path.join(BASE_FOLDER, "locale", "en-us") + + +def read_samples(path): + samples = [] + with open(path) as fi: + for _ in fi.read().split("\n"): + if _ and not _.strip().startswith("#"): + samples += expand_options(_) + return samples + +samples = [] +for root, folders, files in os.walk(res_folder): + for f in files: + if f.endswith(".intent"): + samples += read_samples(os.path.join(root, f)) +skill._data["examples"] = list(set(samples)) + +has_android = os.path.exists(android_ui) +with open(skill_code) as f: + has_homescreen = f"{repo}.{author}.home" in f.read() + +if not os.path.exists(readme): + with open(readme, "w") as f: + f.write(skill.generate_readme()) + +if has_homescreen and not os.path.exists(desktopf): + with open(desktopf, "w") as f: + f.write(skill.desktop_file) + +if not os.path.exists(jsonf): + data = skill.json + with open(jsonf, "w") as f: + if not has_android or not has_homescreen: + data.pop("android") + if not has_homescreen: + data.pop("desktop") + data["desktopFile"] = False +else: + with open(jsonf) as f: + data = json.load(f) + +# set dev branch +data["branch"] = "dev" + +with open(jsonf, "w") as f: + json.dump(data, f, indent=4) diff --git a/scripts/release_skillstore.py b/scripts/release_skillstore.py new file mode 100644 index 000000000..c0a7de151 --- /dev/null +++ b/scripts/release_skillstore.py @@ -0,0 +1,46 @@ +import json +import os + +BASE_FOLDER = os.getenv("BASE_FOLDER") +if not BASE_FOLDER: + raise ValueError("environment variable `LOCALE_FOLDER` not set") +elif not os.path.exists(BASE_FOLDER): + raise ValueError(f"environment variable `LOCALE_FOLDER` is not a folder: {BASE_FOLDER}") +VERSION_FILE = os.getenv("VERSION_FILE") +if not os.path.isfile(VERSION_FILE): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {VERSION_FILE}") + + +def get_version(): + """ Find the version of the package""" + major, minor, build, alpha = (None, None, None, None) + with open(VERSION_FILE) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if alpha and int(alpha) > 0: + version += f"a{alpha}" + return version + + +desktop_dir = os.path.join(BASE_FOLDER, "res", "desktop") +jsonf = os.path.join(desktop_dir, "skill.json") + +with open(jsonf) as f: + data = json.load(f) + +data["branch"] = "v" + get_version() + +with open(jsonf, "w") as f: + json.dump(data, f, indent=4) diff --git a/scripts/remove_alpha.py b/scripts/remove_alpha.py new file mode 100644 index 000000000..a33968e2b --- /dev/null +++ b/scripts/remove_alpha.py @@ -0,0 +1,15 @@ +import fileinput +import os + + +version_file = os.getenv("VERSION_FILE") +if not os.path.isfile(version_file): + raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") + +alpha_var_name = "VERSION_ALPHA" + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/translate.py b/scripts/translate.py new file mode 100644 index 000000000..f5cf7b56b --- /dev/null +++ b/scripts/translate.py @@ -0,0 +1,161 @@ +from os.path import dirname, join, isdir, exists +from pathlib import Path +import shutil +import os +import re +from ovos_utils.bracket_expansion import expand_options +from ovos_translate_plugin_deepl import DeepLTranslator + + +BASE_LANGS = {"de-de", "es-es", "fr-fr", "it-it", "nl-nl", "pt-pt"} +API_KEY = os.getenv("API_KEY") +if not API_KEY: + raise ValueError("environment variable `API_KEY` not set") +BASE_FOLDER = os.getenv("LOCALE_FOLDER") +if not BASE_FOLDER: + raise ValueError("environment variable `LOCALE_FOLDER` not set") +elif not isdir(BASE_FOLDER): + raise ValueError(f"environment variable `LOCALE_FOLDER` is not a folder: {BASE_FOLDER}") + +# old skill structure +old_voc_folder = join(dirname(BASE_FOLDER), "vocab") +old_dialog_folder = join(dirname(BASE_FOLDER), "dialog") +old_res_folder = [old_voc_folder, old_dialog_folder] + + +def get_target_languages(): + langs = set() + folder_to_check = [BASE_FOLDER] + old_res_folder + for folder in folder_to_check: + if not exists(folder): + continue + for subfolder in os.listdir(folder): + # only lancodes in the form of xx-xx + if len(subfolder) == 5 and subfolder[2] == "-": + langs.add(subfolder) + langs = langs.union(BASE_LANGS) + return langs + +single_lang = os.getenv("TARGET_LANG") +target_langs = (single_lang,) if single_lang \ + else get_target_languages() + +src_lang="en-us" +src_files={} +# note: regex/namedvalues are just copied, this cant be auto translated reliably +ext = [".voc", ".dialog", ".intent", ".entity", ".rx", ".value", ".word"] +untranslated = [".rx", ".value", ".entity"] + +tx = DeepLTranslator({"api_key": API_KEY}) + + +def file_location(f: str, base: str) -> bool: + for root, dirs, files in os.walk(base): + for file in files: + if f == file: + return join(root, file) + return None + +def translate(lines: list, target_lang: str) -> list: + translations = [] + for line in lines: + replacements = dict() + for num, var in enumerate(re.findall(r"(?:{{|{)[ a-zA-Z0-9_]*(?:}}|})", line)): + line = line.replace(var, f'@{num}', 1) + replacements[f'@{num}'] = var + try: + translated = tx.translate(line, target=target_lang, source=src_lang) + except Exception as e: + continue + for num, var in replacements.items(): + translated = translated.replace(num, var) + translations.append(translated) + + return translations + + +def entities(file: str) -> set: + vars = set() + if not exists(file): + return vars + + lines = get_lines(file) + for line in lines: + for var in re.findall(r"(?:{{|{)[ a-zA-Z0-9_]*(?:}}|})", line): + vars.add(var) + return vars + + +def get_lines(file: str): + with open(file, "r") as f: + # entity files often include #-placeholder + if file.endswith(".entity"): + lines = [exp for l in f.read().split("\n") for exp + in expand_options(l) if l] + else: + lines = [exp for l in f.read().split("\n") for exp + in expand_options(l) if l and not l.startswith("#")] + return lines + + +def migrate_locale(folder): + for lang in os.listdir(folder): + path = join(folder, lang) + for root, dirs, files in os.walk(path): + for file in files: + if file_location(file, join(BASE_FOLDER, lang)) is None: + rel_path = root.replace(folder, "").lstrip("/") + new_path = join(BASE_FOLDER, rel_path) + os.makedirs(new_path, exist_ok=True) + shutil.move(join(root, file), + join(new_path, file)) + shutil.rmtree(path) + shutil.rmtree(folder) + + +for folder in old_res_folder: + if not isdir(folder): + continue + migrate_locale(folder) + +src_folder = join(BASE_FOLDER, src_lang) +for root, dirs, files in os.walk(src_folder): + if src_lang not in root: + continue + for f in files: + if any(f.endswith(e) for e in ext): + file_path = join(root, f) + rel_path = file_path.replace(src_folder, "").lstrip("/") + src_files[rel_path] = file_path + +for lang in target_langs: + # service cant translate + if not tx.get_langcode(lang): + continue + for rel_path, src in src_files.items(): + filename = Path(rel_path).name + dst = file_location(filename, join(BASE_FOLDER, lang)) or \ + join(BASE_FOLDER, lang, rel_path) + if entities(src) != entities(dst): + if exists(dst): + os.remove(dst) + elif not exists(dst): + pass + else: + continue + os.makedirs(dirname(dst), exist_ok=True) + + lines = get_lines(src) + if any(filename.endswith(e) for e in untranslated): + tx_lines = lines + is_translated = False + else: + tx_lines = translate(lines, lang) + is_translated = True + if tx_lines: + tx_lines = list(set(tx_lines)) + with open(dst, "w") as f: + if is_translated: + f.write(f"# auto translated from {src_lang} to {lang}\n") + for translated in set(tx_lines): + f.write(translated + "\n") From 474221526274f9df086f5f6bff7f82bc716490db Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 15 Jan 2024 19:38:37 +0100 Subject: [PATCH 002/357] review/versioning --- .github/workflows/auto_translate.yml | 10 +++++++--- .github/workflows/propose_translation.yml | 11 +++++++---- .github/workflows/publish_alpha.yml | 19 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index ca46688f7..36a2f70eb 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -5,9 +5,13 @@ on: branch: type: string default: ${{ github.ref }} + python_version: + type: string + default: "3.8" locale_folder: type: string default: "locale" + jobs: autotranslate: @@ -17,19 +21,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} path: action/package/ - name: Checkout Scripts Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github path: action/github/ - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | python -m pip install ovos-translate-plugin-deepl ovos-utils diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index de71f2ba2..afd66f206 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -5,6 +5,9 @@ on: branch: type: string default: ${{ github.ref_name }} + python_version: + type: string + default: "3.8" language: type: string default: "" @@ -25,19 +28,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} path: action/package/ - name: Checkout Scripts Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github path: action/github/ - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | python -m pip install ovos-translate-plugin-deepl ovos-utils diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index 712c8d931..c3857a59b 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -10,6 +10,9 @@ on: version_file: type: string default: version.py + python_version: + type: string + default: "3.8" locale_folder: type: string default: "locale" @@ -22,12 +25,12 @@ jobs: LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} path: action/package/ - name: Checkout Scripts Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github path: action/github/ @@ -42,9 +45,9 @@ jobs: - '${{ inputs.locale_folder }}/../vocab/en-us/**' - name: Setup Python if: steps.filter.outputs.locales == 'true' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ inputs.python_version }} - name: Auto Translate if: steps.filter.outputs.locales == 'true' run: | @@ -63,19 +66,19 @@ jobs: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} path: action/package/ - name: Checkout Scripts Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github path: action/github/ - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ inputs.python_version }} - name: Increment Version id: version run: | From 09ab3b650f822bb55b24e94aa5acb3aa96ef8508 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 15 Jan 2024 19:46:48 +0100 Subject: [PATCH 003/357] review/requirements --- .github/workflows/auto_translate.yml | 2 +- .github/workflows/propose_translation.yml | 2 +- .github/workflows/publish_alpha.yml | 2 +- requirements/translation.txt | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 requirements/translation.txt diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index 36a2f70eb..67e88c6e4 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -36,7 +36,7 @@ jobs: python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | - python -m pip install ovos-translate-plugin-deepl ovos-utils + python -m pip install -r action/github/requirements/translation.txt - name: Auto Translate run: | python action/github/scripts/translate.py diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index afd66f206..939b379a2 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -43,7 +43,7 @@ jobs: python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | - python -m pip install ovos-translate-plugin-deepl ovos-utils + python -m pip install -r action/github/requirements/translation.txt - name: Run Translate Script run: | python action/github/scripts/translate.py diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index c3857a59b..eb7a0a0ab 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -51,7 +51,7 @@ jobs: - name: Auto Translate if: steps.filter.outputs.locales == 'true' run: | - python -m pip install ovos-translate-plugin-deepl ovos-utils + python -m pip install -r action/github/requirements/translation.txt python action/github/scripts/translate.py - name: Commit to dev if: steps.filter.outputs.locales == 'true' diff --git a/requirements/translation.txt b/requirements/translation.txt new file mode 100644 index 000000000..21196deef --- /dev/null +++ b/requirements/translation.txt @@ -0,0 +1,2 @@ +ovos-translate-plugin-deepl +ovos-utils \ No newline at end of file From a48eecfc241d3019d0a7713aff9226c96937e5cc Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 15 Jan 2024 20:15:08 +0100 Subject: [PATCH 004/357] review/descriptions --- .github/workflows/auto_translate.yml | 3 +++ .github/workflows/propose_translation.yml | 9 +++++++-- .github/workflows/publish_alpha.yml | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index 67e88c6e4..fd743baa8 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -3,12 +3,15 @@ on: workflow_call: inputs: branch: + description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref }} python_version: + description: 'The python version to use' type: string default: "3.8" locale_folder: + description: 'The base folder to look for localization files' type: string default: "locale" diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index 939b379a2..6da2f2588 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -3,18 +3,23 @@ on: workflow_call: inputs: branch: + description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref_name }} python_version: + description: 'The python version to use' type: string default: "3.8" language: + description: 'The language to translate to (ISO 3166-1 alpha-2 code; form "xx-xx")' type: string default: "" locale_folder: + description: 'The base folder to look for localization files' type: string default: "locale" - reviewer: + reviewers: + description: 'A comma or newline-separated string of reviewers' type: string default: "emphasize" @@ -58,4 +63,4 @@ jobs: body: Translations for review labels: translation branch: staging/translation_${{ inputs.language }} - reviewers: ${{ inputs.reviewer }} + reviewers: ${{ inputs.reviewers }} diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index eb7a0a0ab..7f36a6ed5 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -5,15 +5,19 @@ on: workflow_call: inputs: branch: + description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref }} version_file: + description: 'The file location to read the version from' type: string default: version.py python_version: + description: 'The python version to use' type: string default: "3.8" locale_folder: + description: 'The base folder to look for localization files' type: string default: "locale" From be4732a32f36da1bfa014eac0641d011e10e042a Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 15 Jan 2024 20:42:00 +0100 Subject: [PATCH 005/357] review/release_action --- .github/workflows/publish_alpha.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index 7f36a6ed5..aa5c08ff4 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -101,20 +101,18 @@ jobs: repository: action/package/ - name: Change working directory to release run: cd ${{ github.workspace }}/action/package/ - - name: Create Release + - name: Create Pre-release id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + uses: ncipollo/release-action@v1 with: - tag_name: V${{ steps.version.outputs.version }} - release_name: Release ${{ steps.version.outputs.version }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: V${{ steps.version.outputs.version }} + name: Release ${{ steps.version.outputs.version }} body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - draft: false + commit: ${{ inputs.branch }} prerelease: true - commitish: ${{ inputs.branch }} - name: Install Build Tools run: | python -m pip install build wheel @@ -122,7 +120,7 @@ jobs: run: | cd action/package python setup.py sdist bdist_wheel - - name: Publish to Test PyPI + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{secrets.PYPI_TOKEN}} From 636fbe308608e5e05b4aca8219438549136866d7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 16 Jan 2024 18:27:35 +0100 Subject: [PATCH 006/357] add semver actions --- .github/workflows/propose_semver_release.yml | 74 ++++++++++++++++++++ .github/workflows/publish_alpha.yml | 14 ++-- .github/workflows/publish_stable_release.yml | 67 ++++++++++++++++++ .github/workflows/pull_master.yml | 44 ++++++++++++ 4 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/propose_semver_release.yml create mode 100644 .github/workflows/publish_stable_release.yml create mode 100644 .github/workflows/pull_master.yml diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml new file mode 100644 index 000000000..76b4a6e75 --- /dev/null +++ b/.github/workflows/propose_semver_release.yml @@ -0,0 +1,74 @@ +name: Propose Stable Release +on: + workflow_call: + inputs: + branch: + type: string + default: "dev" + python_version: + description: 'The python version to use' + type: string + default: "3.8" + version_file: + description: 'The file location to read the version from' + type: string + default: version.py + release_type: + type: string + changelog_file: + type: string + default: "CHANGELOG.md" + + +jobs: + bump_version: + runs-on: "ubuntu-latest" + env: + VERSION_FILE: ${{ inputs.version_file }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + path: action/github/ + repository: OpenVoiceOS/.github + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Increment Version + id: version + run: | + python action/github/scripts/bump_"${{ inputs.release_type }}".py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version to ${{ env.version }} + repository: action/package/ + - name: Check Latest Release + id: latest_release + continue-on-error: true + uses: pozetroninc/github-action-get-latest-release@master + with: + repository: ${{ github.repository }} + excludes: draft,prerelease + - name: Generate Changelog + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + output: action/package/${{ inputs.changelog_file }} + futureRelease: ${{ env.VERSION }} + sinceTag: ${{ steps.latest_release.outputs.release }} + - name: Push Changelog + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update Changelog + repository: action/package/ + \ No newline at end of file diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index aa5c08ff4..3815e2ba4 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -20,6 +20,9 @@ on: description: 'The base folder to look for localization files' type: string default: "locale" + changelog_file: + type: string + default: "CHANGELOG.md" jobs: autotranslate: @@ -87,17 +90,18 @@ jobs: id: version run: | python action/github/scripts/bump_alpha.py - echo "::set-output name=version::$(python action/package/setup.py --version)" + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: "Generate release changelog" id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GITHUB_TOKEN }} - output: action/package/ + output: action/package/${{ inputs.changelog_file }} - name: Push Version Change uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: Increment Version to ${{ steps.version.outputs.version }} + commit_message: Increment Version to ${{ env.version }} repository: action/package/ - name: Change working directory to release run: cd ${{ github.workspace }}/action/package/ @@ -106,11 +110,11 @@ jobs: uses: ncipollo/release-action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - tag: V${{ steps.version.outputs.version }} + tag: V${{ env.version }} name: Release ${{ steps.version.outputs.version }} body: | Changes in this Release - ${{ steps.changelog.outputs.changelog }} + ${{ env.changelog }} commit: ${{ inputs.branch }} prerelease: true - name: Install Build Tools diff --git a/.github/workflows/publish_stable_release.yml b/.github/workflows/publish_stable_release.yml new file mode 100644 index 000000000..0b920d81f --- /dev/null +++ b/.github/workflows/publish_stable_release.yml @@ -0,0 +1,67 @@ +name: Publish Stable Build +on: + workflow_call: + secrets: + PYPI_TOKEN: + required: true + inputs: + python_version: + description: "Python version to use for the build process." + type: string + default: "3.8" + changelog_file: + description: "File to write the changelog to." + type: string + default: "CHANGELOG.md" + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: master + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python_version}} + - name: Get Version + run: | + VERSION=$(python setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Check Latest Release + id: latest_release + continue-on-error: true + uses: pozetroninc/github-action-get-latest-release@master + with: + repository: ${{ github.repository }} + excludes: draft,prerelease + - name: Generate Changelog + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + output: action/package/${{ inputs.changelog_file }} + sinceTag: ${{ steps.latest_release.outputs.release }} + - name: Create Release + id: create_release + uses: ncipollo/release-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + tag: V${{ env.version }} + name: Release ${{ env.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} diff --git a/.github/workflows/pull_master.yml b/.github/workflows/pull_master.yml new file mode 100644 index 000000000..a0883c2d5 --- /dev/null +++ b/.github/workflows/pull_master.yml @@ -0,0 +1,44 @@ +name: Semver release pull request to master +on: + workflow_call: + inputs: + pr_reviewer: + description: Pull request reviewers, comma-separated list (no spaces) + required: false + type: string + pr_assignee: + description: Pull request assignees, comma-separated list (no spaces) + required: false + type: string + source_branch: + description: Branch name to pull from, default is triggered branch + type: string + default: "${{ github.ref }}" + pr_title: + description: Pull request title + type: string + pr_body: + description: Pull request body + type: string + pr_draft: + description: Draft pull request + type: boolean + default: true + +jobs: + pull_changes: + runs-on: "ubuntu-latest" + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: pull request to master + uses: repo-sync/pull-request@v2 + # "master" is the `destination_branch` default value + with: + source_branch: ${{ inputs.source_branch }} + github_token: ${{ secrets.GITHUB_TOKEN }} + pr_reviewer: ${{ inputs.pr_reviewer }} + pr_assignee: ${{ inputs.pr_assignee }} + pr_draft: ${{ inputs.pr_draft }} + pr_title: ${{ inputs.pr_title }} + pr_body: ${{ inputs.pr_body }} \ No newline at end of file From 2e20518f979af9c880f3bfd82db0c530c389e554 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 16 Jan 2024 22:38:11 +0100 Subject: [PATCH 007/357] add skill / license / build tests --- .github/workflows/license_tests.yml | 85 ++++++ .github/workflows/python_build_tests.yml | 64 +++++ .github/workflows/skill_test_installation.yml | 67 +++++ .github/workflows/skill_test_intents.yml | 89 ++++++ .github/workflows/skill_test_ressources.yml | 68 +++++ .github/workflows/skill_tests.yml | 67 +++++ requirements/pip_skill_tests.txt | 0 requirements/pip_tests.txt | 3 + requirements/sys_deb_common_deps.txt | 7 + test/test_skill_intents.py | 264 ++++++++++++++++++ test/test_skill_osm_install.py | 31 ++ test/test_skill_ressources.py | 199 +++++++++++++ 12 files changed, 944 insertions(+) create mode 100644 .github/workflows/license_tests.yml create mode 100644 .github/workflows/python_build_tests.yml create mode 100644 .github/workflows/skill_test_installation.yml create mode 100644 .github/workflows/skill_test_intents.yml create mode 100644 .github/workflows/skill_test_ressources.yml create mode 100644 .github/workflows/skill_tests.yml create mode 100644 requirements/pip_skill_tests.txt create mode 100644 requirements/pip_tests.txt create mode 100644 requirements/sys_deb_common_deps.txt create mode 100644 test/test_skill_intents.py create mode 100644 test/test_skill_osm_install.py create mode 100644 test/test_skill_ressources.py diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 000000000..388bb4cb0 --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,85 @@ +name: License Tests +on: + workflow_call: + inputs: + runner: + description: 'Runner to use' + type: string + default: 'ubuntu-latest' + branch: + description: 'Branch to check out' + type: string + default: ${{ github.ref }} + python-version: + description: 'Python version to use' + type: string + default: '3.8' + system-deps: + description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + required: false + type: string + package-extras: + description: 'Optional extras to install the python package with' + required: false + type: string + packages-exclude: + description: 'Regex to exclude packages from the license check' + type: string + default: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct).*' + licenses-exclude: + description: 'Regex to exclude licenses from the license check' + type: string + default: '^(Mozilla|NeonAI License v1.0).*$' +jobs: + license_tests: + timeout-minutes: 15 + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + path: action/github/ + repository: OpenVoiceOS/.github + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + if [ -z ${{ inputs.system-deps }} ]; then + xargs sudo apt-get install -y < action/github/sys_deb_common_deps.txt + else + sudo apt-get install ${{ inputs.system-deps }} + fi + - name: Install repo + run: | + cd ${{ github.workspace }}/action/package + if [ -z ${{ inputs.package-extras }} ]; then + pip install . + else + pip install .[${{ inputs.package-extras }}] + fi + - name: Get explicit and transitive dependencies + run: | + pip freeze > requirements-all.txt + - name: Check licenses + id: license_check_report + uses: pilosus/action-pip-license-checker@v2 + with: + requirements: '${{ github.workspace }}/action/package/requirements-all.txt' + fail: 'Copyleft,Other,Error' + fails-only: true + exclude: ${{ inputs.packages-exclude }} + exclude-license: ${{ inputs.licenses-exclude }} + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" \ No newline at end of file diff --git a/.github/workflows/python_build_tests.yml b/.github/workflows/python_build_tests.yml new file mode 100644 index 000000000..c55fb3b63 --- /dev/null +++ b/.github/workflows/python_build_tests.yml @@ -0,0 +1,64 @@ +name: Python Build Tests +on: + workflow_call: + inputs: + runner: + description: 'Runner to use' + type: string + default: "ubuntu-latest" + python_matrix: + description: 'Python matrix (string) to use' + type: string + default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + test_manifest: + description: 'if to test MANIFEST.in' + required: false + type: boolean + default: false + manifest_ignored: + description: 'Files to ignore in MANIFEST.in' + required: false + type: string + default: "test/**" + test_pipaudit: + description: 'if to test with pip-audit' + type: boolean + default: false + test_relative_paths: + description: 'if to test relative paths' + type: boolean + default: true + pipaudit_ignored: + type: string + default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" +jobs: + py_build_tests: + timeout-minutes: 15 + strategy: + max-parallel: 2 + matrix: + python-version: ${{ fromJson(inputs.python_matrix) }} + runs-on: ${{inputs.runner}} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel sdist + - name: Test Manifest + if: ${{ inputs.test_manifest }} + uses: tj-actions/check-manifest@v1 + with: + args: --ignore ${{ inputs.manifest_ignored }} + - name: PIP Audit + if: ${{ inputs.test_pipaudit }} + uses: pypa/gh-action-pip-audit@v1.0.8 + with: + ignore-vulns: ${{ inputs.pipaudit_ignored }} diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml new file mode 100644 index 000000000..2b8e79ccf --- /dev/null +++ b/.github/workflows/skill_test_installation.yml @@ -0,0 +1,67 @@ +name: Skill Installation Tests +on: + workflow_call: + inputs: + runner: + description: 'Runner to use' + type: string + default: "ubuntu-latest" + branch: + description: 'Branch to use' + type: string + default: ${{ github.ref }} + python_matrix: + description: 'Python versions to test against' + type: string + default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + system_deps: + description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + required: false + type: string + pip_packages: + description: 'Python packages (whitespace delimited) to install instead of pip_tests.txt' + required: false + type: string + test_osm: + description: 'Whether to test OSM installation' + type: boolean + default: true +jobs: + test_osm_install: + runs-on: ${{inputs.runner}} + timeout-minutes: 5 + if: ${{ inputs.test_osm }} + strategy: + matrix: + python-version: ${{ fromJson(inputs.python_matrix) }} + steps: + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install System Dependencies + run: | + sudo apt update + if [ -z "${{inputs.system_deps}}" ]; then + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + else + sudo apt install -y ${{inputs.system_deps}} + fi + - name: Install Python Requirements + run: | + pip install --upgrade pip + if [ -z "${{inputs.pip_packages}}" ]; then + pip install -r ${{ github.workspace }}/action/skill/requirements/pip_tests.txt + else + pip install ${{inputs.pip_packages}} + fi + - name: Test OSM Installation + run: | + export TEST_REPO=${{ github.repository }} + export TEST_BRANCH=${{inputs.branch}} + pytest ${{ github.workspace }}/action/github/test/test_skill_osm_install.py diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml new file mode 100644 index 000000000..128af6271 --- /dev/null +++ b/.github/workflows/skill_test_intents.yml @@ -0,0 +1,89 @@ +name: Skill Intents Tests +on: + workflow_call: + inputs: + runner: + type: string + default: "ubuntu-latest" + timeout: + type: number + default: 15 + system_deps: + description: 'Optional system dependencies to install instead of requirements.txt' + required: false + type: string + skill_entrypoint: + type: string + default: action/skill/ + intent_file: + type: string + default: test/test_intents.yaml + test_padatious: + type: boolean + default: False + test_padacioso: + type: boolean + default: True + python_matrix: + description: 'Python matrix (string) to use' + type: string + default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: + description: 'Custom pip packages (whitespace delimited) to install instead of pip_skill_tests.txt' + required: false + type: string + +jobs: + test_intents_ovos: + runs-on: ${{inputs.runner}} + timeout-minutes: ${{inputs.timeout}} + strategy: + matrix: + python-version: ${{ fromJSON(inputs.python_matrix) }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + path: action/skill/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install System Dependencies + run: | + sudo apt update + if [ -z "${{inputs.system_deps}}" ]; then + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + else + sudo apt install -y ${{inputs.system_deps}} + fi + - name: Install Python Skill and Requirements + run: | + pip install --upgrade pip + pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 + if [ -z "${{inputs.pip_packages}}" ]; then + pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + else + pip install ${{inputs.pip_packages}} + fi + cd ${{ github.workspace }}/action/skill + pip install . + - name: Test Skill Intents Padacioso + if: ${{ inputs.test_padacioso }} + run: | + export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" + export INTENT_ENGINE="padacioso" + export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" + pytest action/github/test/test_skill_intents.py + - name: Test Skill Intents Padatious + if: ${{ inputs.test_padatious }} + run: | + export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" + export INTENT_ENGINE="padatious" + export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" + pytest action/github/test/test_skill_intents.py \ No newline at end of file diff --git a/.github/workflows/skill_test_ressources.yml b/.github/workflows/skill_test_ressources.yml new file mode 100644 index 000000000..b1b62e38c --- /dev/null +++ b/.github/workflows/skill_test_ressources.yml @@ -0,0 +1,68 @@ +name: Skill Ressource Tests +on: + workflow_call: + inputs: + runner: + description: 'The runner to use' + type: string + default: "ubuntu-latest" + system_deps: + description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + required: false + type: string + python_version: + description: 'The python version to use' + type: string + default: "3.8" + pip_packages: + description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + required: false + type: string + resource_testfile: + type: string + default: test/unittests/test_resources.yaml + skill_entrypoint: + type: string + default: action/skill/ +jobs: + test_resources: + runs-on: ${{inputs.runner}} + timeout-minutes: 5 + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + path: action/skill/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python_version}} + - name: Install System Dependencies + run: | + sudo apt update + if [ -z "${{inputs.system_deps}}" ]; then + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + else + sudo apt install -y ${{inputs.system_deps}} + fi + - name: Install Python Skill and Requirements + run: | + pip install --upgrade pip + pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 + if [ -z "${{inputs.pip_packages}}" ]; then + pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + else + pip install ${{inputs.pip_packages}} + fi + cd ${{ github.workspace }}/action/skill/ + pip install . + - name: Test Skill Resources + run: | + export RESOURCE_TEST_FILE="action/skill/${{inputs.resource_testfile}}" + export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" + pytest action/github/test/test_skill_resources.py \ No newline at end of file diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml new file mode 100644 index 000000000..e012bb29c --- /dev/null +++ b/.github/workflows/skill_tests.yml @@ -0,0 +1,67 @@ +name: Skill Unit Tests +on: + workflow_call: + inputs: + runner: + type: string + default: "ubuntu-latest" + system_deps: + description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + required: false + type: string + python_matrix: + description: 'Python matrix (string) to use' + type: string + default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: + description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + required: false + type: string + skill_testfile: + description: 'Skill test file to run relative to the base folder' + type: string + default: test/unittests/test_skill.py + +jobs: + skill_unit_tests: + strategy: + matrix: + python-version: ${{ fromJSON(inputs.python_matrix) }} + runs-on: ${{inputs.runner}} + timeout-minutes: 15 + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + path: action/skill/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + path: action/github/ + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install System Dependencies + run: | + sudo apt update + if [ -z "${{inputs.system_deps}}" ]; then + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + else + sudo apt install -y ${{inputs.system_deps}} + fi + - name: Install Python Skill and Requirements + run: | + pip install --upgrade pip + pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 + if [ -z "${{inputs.pip_packages}}" ]; then + pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + else + pip install ${{inputs.pip_packages}} + fi + cd ${{ github.workspace }}/action/skill/ + pip install . + - name: Test Skill + run: | + pytest ${{ github.workspace }}/action/skill/${{inputs.skill_testfile}} \ No newline at end of file diff --git a/requirements/pip_skill_tests.txt b/requirements/pip_skill_tests.txt new file mode 100644 index 000000000..e69de29bb diff --git a/requirements/pip_tests.txt b/requirements/pip_tests.txt new file mode 100644 index 000000000..044eaf15b --- /dev/null +++ b/requirements/pip_tests.txt @@ -0,0 +1,3 @@ +pytest +mock +ovos-skills-manager \ No newline at end of file diff --git a/requirements/sys_deb_common_deps.txt b/requirements/sys_deb_common_deps.txt new file mode 100644 index 000000000..5912d6e86 --- /dev/null +++ b/requirements/sys_deb_common_deps.txt @@ -0,0 +1,7 @@ +gcc +python3-dev +swig +libssl-dev +libfann-dev +portaudio19-dev +libpulse-dev \ No newline at end of file diff --git a/test/test_skill_intents.py b/test/test_skill_intents.py new file mode 100644 index 000000000..d5a6b2fd3 --- /dev/null +++ b/test/test_skill_intents.py @@ -0,0 +1,264 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +from os import getenv + +import importlib +import unittest +import yaml +from mock import Mock, patch + +from ovos_utils.log import LOG +from mycroft_bus_client import Message +from ovos_config.config import update_mycroft_config +from ovos_utils.messagebus import FakeBus +from mycroft.skills.intent_services.padatious_service import PadatiousMatcher + + +regex_only = getenv("INTENT_ENGINE") == "padacioso" +LOG.level = logging.DEBUG + + +class MockPadatiousMatcher(PadatiousMatcher): + include_med = True + include_low = False + + def __init__(self, *args, **kwargs): + PadatiousMatcher.__init__(self, *args, **kwargs) + LOG.debug("Creating test Padatious Matcher") + + def match_medium(self, utterances, lang=None, __=None): + if not self.include_med: + LOG.info(f"Skipping medium confidence check for {utterances}") + return None + PadatiousMatcher.match_medium(self, utterances, lang=lang) + + def match_low(self, utterances, lang=None, __=None): + if not self.include_low: + LOG.info(f"Skipping low confidence check for {utterances}") + return None + PadatiousMatcher.match_low(self, utterances, lang=lang) + + +def get_skill_class(): + from ovos_plugin_manager.skills import find_skill_plugins + plugins = find_skill_plugins() + plugin_id = getenv("TEST_SKILL_ID") + if plugin_id: + skill = plugins.get(plugin_id) + else: + assert len(plugins.values()) == 1 + skill = list(plugins.values())[0] + return skill + + +class TestSkillIntentMatching(unittest.TestCase): + test_intents = getenv("INTENT_TEST_FILE") + with open(test_intents) as f: + valid_intents = yaml.safe_load(f) + negative_intents = valid_intents.pop('unmatched intents', dict()) + common_query = valid_intents.pop("common query", dict()) + + # Ensure all tested languages are loaded + import ovos_config + update_mycroft_config({"secondary_langs": list(valid_intents.keys()), + "padatious": { + "regex_only": regex_only, + # TODO: below config patching ovos-core default config + # https://github.com/OpenVoiceOS/ovos-config/pull/78/files#r1439966369 + "intent_cache": "~/.local/share/mycroft/intent_cache", + "train_delay": 4, + "single_thread": False}}) + importlib.reload(ovos_config.config) + # Start the IntentService + bus = FakeBus() + from mycroft.skills.intent_service import IntentService + intent_service = IntentService(bus) + assert intent_service.padatious_service.is_regex_only == regex_only + + # Create the skill to test + # TODO: Refactor to use ovos-workshop + skill_class = get_skill_class() + test_skill_id = 'test_skill.test' + skill = skill_class(skill_id=test_skill_id, bus=bus) + assert skill.config_core["secondary_langs"] == list(valid_intents.keys()) + + last_message = None + + @classmethod + def setUpClass(cls) -> None: + def _on_message(msg): + cls.last_message = msg + + cls.bus.on("message", _on_message) + + def test_00_init(self): + for lang in self.valid_intents: + if hasattr(self.skill, "_native_langs"): + # ovos-workshop < 0.0.15 + self.assertIn(lang, self.skill._native_langs, lang) + else: + self.assertIn(lang, self.skill.native_langs, lang) + self.assertIn(lang, + self.intent_service.padatious_service.containers) + # intents = [intent[1]['name'] for intent in + # self.skill.intent_service.registered_intents if + # intent[1]['lang'] == lang] + # LOG.info(f"{lang} intents: {intents}") + # self.assertIsNotNone(intents, f"No intents registered for {lang}") + # for intent in self.valid_intents[lang]: + # # Validate IntentServiceInterface registration + # self.assertIn(f"{self.test_skill_id}:{intent}", intents, + # f"Intent not defined for {lang}") + + def test_intents(self): + for lang in self.valid_intents: + self.assertIsInstance(lang.split('-')[0], str) + self.assertIsInstance(lang.split('-')[1], str) + for intent, examples in self.valid_intents[lang].items(): + intent_event = f'{self.test_skill_id}:{intent}' + self.skill.events.remove(intent_event) + intent_handler = Mock() + self.skill.events.add(intent_event, intent_handler) + for utt in examples: + if isinstance(utt, dict): + data = list(utt.values())[0] + utt = list(utt.keys())[0] + else: + data = list() + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + try: + intent_handler.assert_called_once() + except AssertionError as e: + LOG.error(f"sent:{message.serialize()}") + LOG.error(f"received:{self.last_message}") + raise AssertionError(utt) from e + intent_message = intent_handler.call_args[0][0] + self.assertIsInstance(intent_message, Message, utt) + self.assertEqual(intent_message.msg_type, intent_event, utt) + for datum in data: + if isinstance(datum, dict): + name = list(datum.keys())[0] + value = list(datum.values())[0] + else: + name = datum + value = None + if name in intent_message.data: + # This is an entity + voc_id = name + else: + # We mocked the handler, data is munged + voc_id = f'{self.test_skill_id.replace(".", "_")}' \ + f'{name}' + self.assertIsInstance(intent_message.data.get(voc_id), + str, intent_message.data) + if value: + self.assertEqual(intent_message.data.get(voc_id), + value, utt) + intent_handler.reset_mock() + + @patch("mycroft.skills.intent_service.PadatiousMatcher", + new=MockPadatiousMatcher) + def test_negative_intents(self): + test_config = self.negative_intents.pop('config', None) + if test_config: + MockPadatiousMatcher.include_med = test_config.get('include_med', + True) + MockPadatiousMatcher.include_low = test_config.get('include_low', + False) + intent_failure = Mock() + self.intent_service.send_complete_intent_failure = intent_failure + + # # Skip any fallback/converse handling + # self.intent_service.fallback = Mock() + # self.intent_service.converse = Mock() + # if not self.common_query: + # # Skip common_qa unless explicitly testing a Common QA skill + # self.intent_service.common_qa = Mock() + + for lang in self.negative_intents.keys(): + for utt in self.negative_intents[lang]: + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + try: + intent_failure.assert_called_once_with(message) + intent_failure.reset_mock() + except AssertionError as e: + LOG.error(self.last_message) + raise AssertionError(utt) from e + + def test_common_query(self): + qa_callback = Mock() + qa_response = Mock() + self.skill.events.add('question:action', qa_callback) + self.skill.events.add('question:query.response', qa_response) + for lang in self.common_query.keys(): + for utt in self.common_query[lang]: + if isinstance(utt, dict): + data = list(utt.values())[0] + utt = list(utt.keys())[0] + else: + data = dict() + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + response = qa_response.call_args[0][0] + callback = qa_response.call_args[0][0] + self.assertIsInstance(response, Message) + self.assertTrue(response.data["phrase"] in utt) + self.assertEqual(response.data["skill_id"], self.skill.skill_id) + self.assertIn("callback_data", response.data.keys()) + self.assertIsInstance(response.data["conf"], float) + self.assertIsInstance(response.data["answer"], str) + + self.assertIsInstance(callback, Message) + self.assertEqual(callback.data['skill_id'], self.skill.skill_id) + self.assertEqual(callback.data['phrase'], + response.data['phrase']) + if not data: + continue + if isinstance(data.get('callback'), dict): + self.assertEqual(callback.data['callback_data'], + data['callback']) + elif isinstance(data.get('callback'), list): + self.assertEqual(set(callback.data['callback_data'].keys()), + set(data.get('callback'))) + if data.get('min_confidence'): + self.assertGreaterEqual(response.data['conf'], + data['min_confidence']) + if data.get('max_confidence'): + self.assertLessEqual(response.data['conf'], + data['max_confidence']) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/test/test_skill_osm_install.py b/test/test_skill_osm_install.py new file mode 100644 index 000000000..344279fb0 --- /dev/null +++ b/test/test_skill_osm_install.py @@ -0,0 +1,31 @@ +import unittest + +from os import environ +from os.path import exists, isdir, isfile, join +from shutil import rmtree + +from ovos_skills_manager import SkillEntry + + +class TestOSM(unittest.TestCase): + def test_osm_install(self): + branch = environ.get("TEST_BRANCH") + install_url = f"https://github.com/{environ.get('TEST_REPO')}@{branch}" + skill = SkillEntry.from_github_url(install_url) + tmp_skills = "/tmp/osm_installed_skills" + skill_folder = f"{tmp_skills}/{skill.uuid}" + + if exists(skill_folder): + rmtree(skill_folder) + + updated = skill.install(folder=tmp_skills, default_branch=branch) + self.assertEqual(updated, True) + self.assertTrue(isdir(skill_folder)) + self.assertTrue(isfile(join(skill_folder, "__init__.py"))) + + updated = skill.install(folder=tmp_skills, default_branch=branch) + self.assertEqual(updated, False) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/test/test_skill_ressources.py b/test/test_skill_ressources.py new file mode 100644 index 000000000..7f8214eac --- /dev/null +++ b/test/test_skill_ressources.py @@ -0,0 +1,199 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2022 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import json +import yaml +from os import getenv +from os.path import expanduser, isfile, isdir +from typing import Optional + +from ovos_utils.messagebus import FakeBus +from ovos_utils.log import LOG +from ovos_workshop.skills.base import BaseSkill +from ovos_workshop.skill_launcher import SkillLoader +from ovos_config.config import update_mycroft_config, Configuration +from ovos_plugin_manager.skills import find_skill_plugins + + +def get_skill_object(skill_entrypoint: str, bus: FakeBus, + skill_id: str, config_patch: Optional[dict] = None) -> BaseSkill: + """ + Get an initialized skill object by entrypoint with the requested skill_id. + @param skill_entrypoint: Skill plugin entrypoint or directory path + @param bus: FakeBus instance to bind to skill for testing + @param skill_id: skill_id to initialize skill with + @param config_patch: Configuration update to apply + @returns: Initialized skill object + """ + if config_patch: + user_config = update_mycroft_config(config_patch) + if user_config not in Configuration.xdg_configs: + Configuration.xdg_configs.append(user_config) + if isdir(skill_entrypoint): + LOG.info(f"Loading local skill: {skill_entrypoint}") + loader = SkillLoader(bus, skill_entrypoint, skill_id) + if loader.load(): + return loader.instance + plugins = find_skill_plugins() + if skill_entrypoint not in plugins: + raise ValueError(f"Requested skill not found: {skill_entrypoint}") + plugin = plugins[skill_entrypoint] + skill = plugin(bus=bus, skill_id=skill_id) + return skill + + +def load_resource_tests(test_file: str) -> dict: + """ + Load resource tests from a file + @param test_file: Test file to load + @returns: Loaded test spec + """ + test_file = expanduser(test_file) + if not isfile(test_file): + raise FileNotFoundError(test_file) + with open(test_file) as f: + resources = yaml.safe_load(f) + return resources + + +class TestSkillResources(unittest.TestCase): + # Static parameters + messages = list() + bus = FakeBus() + bus.run_forever() + test_skill_id = 'test_skill.test' + + # Define skill and resource spec to use in tests + resources = load_resource_tests(getenv("RESOURCE_TEST_FILE")) + skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT") + + # Specify valid languages to test + supported_languages = resources['languages'] + + # Specify skill intents as sets + adapt_intents = set(resources['intents']['adapt']) + padatious_intents = set(resources['intents']['padatious']) + + # regex entities, not necessarily filenames + regex = set(resources['regex']) + # vocab is lowercase .voc file basenames + vocab = set(resources['vocab']) + # dialog is .dialog file basenames (case-sensitive) + dialog = set(resources['dialog']) + + core_config_patch = {"secondary_langs": supported_languages} + + @classmethod + def setUpClass(cls) -> None: + cls.bus.on("message", cls._on_message) + + cls.skill = get_skill_object(skill_entrypoint=cls.skill_entrypoint, + bus=cls.bus, skill_id=cls.test_skill_id, + config_patch=cls.core_config_patch) + + cls.adapt_intents = {f'{cls.test_skill_id}:{intent}' + for intent in cls.adapt_intents} + cls.padatious_intents = {f'{cls.test_skill_id}:{intent}' + for intent in cls.padatious_intents} + + @classmethod + def _on_message(cls, message): + cls.messages.append(json.loads(message)) + + def test_skill_setup(self): + self.assertEqual(self.skill.skill_id, self.test_skill_id) + if hasattr(self.skill, "_core_lang"): + # ovos-workshop < 0.0.15 + self.assertEqual(set([self.skill._core_lang] + + self.skill._secondary_langs), + set(self.supported_languages), + f"expected={self.supported_languages}") + else: + self.assertEqual(set([self.skill.core_lang] + + self.skill.secondary_langs), + set(self.supported_languages), + f"expected={self.supported_languages}") + + def test_intent_registration(self): + registered_adapt = list() + registered_padatious = dict() + registered_vocab = dict() + registered_regex = dict() + for msg in self.messages: + if msg["type"] == "register_intent": + registered_adapt.append(msg["data"]["name"]) + elif msg["type"] == "padatious:register_intent": + lang = msg["data"]["lang"] + registered_padatious.setdefault(lang, list()) + registered_padatious[lang].append(msg["data"]["name"]) + elif msg["type"] == "register_vocab": + lang = msg["data"]["lang"] + if msg['data'].get('regex'): + registered_regex.setdefault(lang, dict()) + regex = msg["data"]["regex"].split( + '<', 1)[1].split('>', 1)[0].replace( + self.test_skill_id.replace('.', '_'), '') + registered_regex[lang].setdefault(regex, list()) + registered_regex[lang][regex].append(msg["data"]["regex"]) + else: + registered_vocab.setdefault(lang, dict()) + voc_filename = msg["data"]["entity_type"].replace( + self.test_skill_id.replace('.', '_'), '').lower() + registered_vocab[lang].setdefault(voc_filename, list()) + registered_vocab[lang][voc_filename].append( + msg["data"]["entity_value"]) + self.assertEqual(set(registered_adapt), self.adapt_intents, + registered_adapt) + for lang in self.supported_languages: + if self.padatious_intents: + self.assertEqual(set(registered_padatious[lang]), + self.padatious_intents, + registered_padatious[lang]) + if self.vocab: + self.assertEqual(set(registered_vocab[lang].keys()), + self.vocab, registered_vocab) + if self.regex: + self.assertEqual(set(registered_regex[lang].keys()), + self.regex, registered_regex) + for voc in self.vocab: + # Ensure every vocab file has at least one entry + self.assertGreater(len(registered_vocab[lang][voc]), 0) + for rx in self.regex: + # Ensure every rx file has exactly one entry + self.assertTrue(all((rx in line for line in + registered_regex[lang][rx])), self.regex) + + def test_dialog_files(self): + for lang in self.supported_languages: + dialogs = self.skill._lang_resources[lang].dialog_renderer.templates + for dialog in self.dialog: + self.assertIn(dialog, dialogs.keys(), + f"lang={lang}") + + # TODO: Consider adding tests for resource file existence \ No newline at end of file From 92befae3058b781445abb0b04cd3265e0dcb379d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 17 Jan 2024 23:54:17 +0100 Subject: [PATCH 008/357] add readme --- README.md | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 222 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d89e6d19c..1788f6ed3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,224 @@ +## Workflows/Actions +### Publish Alpha Release (example: `ovos-core`) +```yaml +name: Publish Alpha Build ...aX +on: + push: + branches: + - dev + paths-ignore: + - 'ovos_core/version.py' + - 'test/**' + - 'examples/**' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'readme.md' + - 'scripts/**' + workflow_dispatch: + +jobs: + build_and_publish: + uses: openvoiceos/.github/.github/workflows/publish_alpha_release.yml@main + with: + branch: dev # Branch to use, default: branch that triggered the action + version_file: ovos_core/version.py # File location of the version file, default: version.py + python_version: "3.8" # Python version (quoted) to use, default: 3.8 + locale_folder: ovos_core/locale # if there are localisation files the location of the base folder, default: locale + changelog_file: CHANGELOG.md # if the changlog file has a special name, default: CHANGELOG.md +``` +## Propose and Publish Stable (Build,Minor,Major) Release +Strategy: 2-fold +*Proposal (prepares bump and pull requests to master)* +```yaml +name: Propose Stable Build +on: + workflow_dispatch: + +jobs: + build_and_publish: + uses: openvoiceos/.github/.github/workflows/propose_stable_release.yml@main + with: + branch: dev # Branch to use, default: branch that triggered the action + python_version: '"3.10"' # Python version (quoted) to use, default: 3.8 + version_file: ovos_core/version.py # File location of the version file, default: version.py + release_type: build # build, minor, major + changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md +``` +*Publishing (from master)* +```yaml +name: Publish Stable Build +on: + workflow_dispatch: + +jobs: + build_and_publish: + uses: openvoiceos/.github/.github/workflows/publish_stable_release.yml@main + with: + python_version: '"3.10"' + changelog_file: ChAnGeLoG.md +``` +## Propose translatios +Introduce a new language localisation by proposing a translation via pull request. (creating new branch staging/translation_xx-xx) +```yaml +name: Propose Translations +on: + workflow_dispatch: + inputs: # multiple ways to set this up + translation: # predefined list of languages* + type: choice + options: + - "de-de" + ... + - "zh-cn" + # or + translations: # wait for dispatcher input for a langcode (xx-xx) + type: string + required: true + +jobs: + propose_translation: + uses: openvoiceos/.github/.github/workflows/propose_translation.yml@main + with: + branch: dev # Branch to use, default: branch that triggered the action + python_version: '"3.8"' + language: ${{ inputs.translation }} + locale_folder: ovos_core/locale/ # the location of the base localisation folder, default: locale + reviewers: "jarbasai,emphasize" # comma separated list of reviewers, default: emphasize +``` +* [available languages with deepl](https://support.deepl.com/hc/en-us/articles/360019925219-Languages-included-in-DeepL-Pro) + +## License testing +Tests validity of licenses of all packages (explicit and transitive). +```yaml +name: License testing +on: + + +jobs: + license_tests: + uses: openvoiceos/.github/.github/workflows/license_tests.yml@main + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + python-version: '"3.8"' # Python version (quoted) to use, default: 3.8 + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + package-extras: test # Optional extras to install the python package with + packages-exclude: '^(fann2|tqdm|bs4).*' # Custom regex to exclude packages from the license check + # default: '^(precise-runner|fann2|tqdm|bs4|nvidia|bitstruct).*' + licenses-exclude: ^(BSD-3).*$' # Custom regex to exclude licenses from the license check + # default: '^(Mozilla|NeonAI License v1.0).*$' +``` +## Build testing +Tests the build of the python package. +```yaml +name: Build testing +on: + + +jobs: + build_tests: + uses: openvoiceos/.github/.github/workflows/python_build_tests.yml@main + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + test_manifest: true # if to test with MANIFEST.in, default: false + manifest_ignored: "test/** qt5/**" # Files to ignore in MANIFEST.in, default: "test/**" + test_relative_paths: false # if to test with relative paths, default: true + test_pipaudit: true # if to test with pip-audit, default: false + pipaudit_ignored: "" # Vulnerabilities to ignore in pip-audit, + # default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" +``` +## Skills +## Skill Unit Tests +```yaml +name: Skill Unit Tests +on: + + +jobs: + build_tests: + uses: openvoiceos/.github/.github/workflows/skill_tests.yml@main + with: + runner: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py +``` +## Skill Installation Tests +```yaml +name: Skill Installation Tests +on: + + +jobs: + skill_installation_tests: + uses: openvoiceos/.github/.github/workflows/skill_test_installation.yml@main + with: + runner: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py + test_osm: true # if to test with osm, default: true + test_loader: true # if to test with skillloader, default: true +``` +## Skill Resource Tests +Tests the resources of a skill. This may be dialogs, vocabs, regex or intent resources +```yaml +name: Skill Ressource Tests +on: + + +jobs: + skill_resource_tests: + uses: openvoiceos/.github/.github/workflows/skill_test_resources.yml@main + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_version: '"3.10"' # Python version to use, default: "3.8" + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml +``` +## Skill Intent Tests +Tests if intents are successfully parsed by the skill +```yaml +name: Skill Intent Tests +on: + + +jobs: + skill_intent_tests: + uses: openvoiceos/.github/.github/workflows/skill_test_intents.yml@main + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + timeout: 15 # Timeout for the test, default: 15 + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml + test_padatious: true # if to test against padatious, default: false + test_padacioso: true # if to test against padacioso, default: true +``` + +------------- + ![OpenVoiceOS](https://openvoiceos.com/wp-content/uploads/2020/10/loading400.png) # OpenVoiceOS A community powered Linux distribution, purpose-built with buildroot to showcase the power of Open Source Voice AI for a range of devices. @@ -13,4 +234,4 @@ At this moment development is in very early stages and focussed on the Raspberry * [Forum thread @ Mycroft A.I.](https://community.mycroft.ai/t/openvoiceos-a-bare-minimal-production-type-of-os-based-on-buildroot/4708) * [Matrix Chat rooms](https://matrix.to/#/!XFpdtmgyCoPDxOMPpH:matrix.org?via=matrix.org) -Visit [openvoiceos.org](https://openvoiceos.org) to learn more! +Visit [openvoiceos.org](https://openvoiceos.org) to learn more! \ No newline at end of file From 35058b74feb021cbee010cb875407bd4467835dd Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 17 Jan 2024 23:54:51 +0100 Subject: [PATCH 009/357] adjustments --- .github/workflows/auto_translate.yml | 2 +- ...release.yml => propose_stable_release.yml} | 0 .github/workflows/propose_translation.yml | 2 +- ...sh_alpha.yml => publish_alpha_release.yml} | 2 +- .github/workflows/skill_test_intents.yml | 27 ++++++++++--------- ...essources.yml => skill_test_resources.yml} | 12 ++++++--- .github/workflows/skill_tests.yml | 5 ++++ requirements/pip_skill_tests.txt | 4 +++ .../{translation.txt => pip_translation.txt} | 0 9 files changed, 34 insertions(+), 20 deletions(-) rename .github/workflows/{propose_semver_release.yml => propose_stable_release.yml} (100%) rename .github/workflows/{publish_alpha.yml => publish_alpha_release.yml} (98%) rename .github/workflows/{skill_test_ressources.yml => skill_test_resources.yml} (92%) rename requirements/{translation.txt => pip_translation.txt} (100%) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index fd743baa8..5d8083416 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -39,7 +39,7 @@ jobs: python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | - python -m pip install -r action/github/requirements/translation.txt + python -m pip install -r action/github/requirements/pip_translation.txt - name: Auto Translate run: | python action/github/scripts/translate.py diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_stable_release.yml similarity index 100% rename from .github/workflows/propose_semver_release.yml rename to .github/workflows/propose_stable_release.yml diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index 6da2f2588..59c5761a7 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -48,7 +48,7 @@ jobs: python-version: ${{ inputs.python_version }} - name: Install Translate Tools run: | - python -m pip install -r action/github/requirements/translation.txt + python -m pip install -r action/github/requirements/pip_translation.txt - name: Run Translate Script run: | python action/github/scripts/translate.py diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha_release.yml similarity index 98% rename from .github/workflows/publish_alpha.yml rename to .github/workflows/publish_alpha_release.yml index 3815e2ba4..22add822c 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -58,7 +58,7 @@ jobs: - name: Auto Translate if: steps.filter.outputs.locales == 'true' run: | - python -m pip install -r action/github/requirements/translation.txt + python -m pip install -r action/github/requirements/pip_translation.txt python action/github/scripts/translate.py - name: Commit to dev if: steps.filter.outputs.locales == 'true' diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 128af6271..c0c9a0937 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -5,16 +5,25 @@ on: runner: type: string default: "ubuntu-latest" + branch: + description: 'Branch to test' + type: string + default: ${{ github.ref }} timeout: type: number default: 15 + python_matrix: + description: 'Python matrix (string) to use' + type: string + default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: + description: 'Custom pip packages (whitespace delimited) to install instead of pip_skill_tests.txt' + required: false + type: string system_deps: description: 'Optional system dependencies to install instead of requirements.txt' required: false type: string - skill_entrypoint: - type: string - default: action/skill/ intent_file: type: string default: test/test_intents.yaml @@ -24,17 +33,11 @@ on: test_padacioso: type: boolean default: True - python_matrix: - description: 'Python matrix (string) to use' - type: string - default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pip_packages: - description: 'Custom pip packages (whitespace delimited) to install instead of pip_skill_tests.txt' - required: false - type: string jobs: test_intents_ovos: + env: + TEST_SKILL_ENTRYPOINT: "action/skill/" runs-on: ${{inputs.runner}} timeout-minutes: ${{inputs.timeout}} strategy: @@ -76,14 +79,12 @@ jobs: - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | - export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" export INTENT_ENGINE="padacioso" export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" pytest action/github/test/test_skill_intents.py - name: Test Skill Intents Padatious if: ${{ inputs.test_padatious }} run: | - export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" export INTENT_ENGINE="padatious" export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" pytest action/github/test/test_skill_intents.py \ No newline at end of file diff --git a/.github/workflows/skill_test_ressources.yml b/.github/workflows/skill_test_resources.yml similarity index 92% rename from .github/workflows/skill_test_ressources.yml rename to .github/workflows/skill_test_resources.yml index b1b62e38c..f4133f522 100644 --- a/.github/workflows/skill_test_ressources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -6,6 +6,10 @@ on: description: 'The runner to use' type: string default: "ubuntu-latest" + branch: + description: 'The branch to use' + type: string + default: ${{ github.ref }} system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false @@ -21,17 +25,18 @@ on: resource_testfile: type: string default: test/unittests/test_resources.yaml - skill_entrypoint: - type: string - default: action/skill/ + jobs: test_resources: + env: + TEST_SKILL_ENTRYPOINT: "action/skill/" runs-on: ${{inputs.runner}} timeout-minutes: 5 steps: - name: Checkout Repository uses: actions/checkout@v4 with: + ref: ${{inputs.branch}} path: action/skill/ - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -64,5 +69,4 @@ jobs: - name: Test Skill Resources run: | export RESOURCE_TEST_FILE="action/skill/${{inputs.resource_testfile}}" - export TEST_SKILL_ENTRYPOINT="${{inputs.skill_entrypoint}}" pytest action/github/test/test_skill_resources.py \ No newline at end of file diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index e012bb29c..acc7e5a69 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -5,6 +5,10 @@ on: runner: type: string default: "ubuntu-latest" + branch: + description: 'Branch to use' + type: string + default: ${{ github.ref }} system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false @@ -33,6 +37,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: + ref: ${{inputs.branch}} path: action/skill/ - name: Checkout Scripts Repo uses: actions/checkout@v4 diff --git a/requirements/pip_skill_tests.txt b/requirements/pip_skill_tests.txt index e69de29bb..ba37c4d63 100644 --- a/requirements/pip_skill_tests.txt +++ b/requirements/pip_skill_tests.txt @@ -0,0 +1,4 @@ +pytest +pytest-cov +mock +ovos-core[skills]>=0.0.7 diff --git a/requirements/translation.txt b/requirements/pip_translation.txt similarity index 100% rename from requirements/translation.txt rename to requirements/pip_translation.txt From 5e83b715a3656580de73227f908623365631ff89 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 26 Jan 2024 15:45:27 +0100 Subject: [PATCH 010/357] last changes before test --- .github/workflows/notify_pr_matrix.yml | 26 +++ .github/workflows/propose_stable_release.yml | 17 ++ ...release.yml => publish_semver_release.yml} | 4 +- .github/workflows/skill_test_installation.yml | 30 ++- .github/workflows/skill_test_intents.yml | 14 +- .github/workflows/skill_test_resources.yml | 6 +- README.md | 212 +++++++++++------- requirements/pip_skill_tests.txt | 7 +- requirements/pip_tests.txt | 3 +- requirements/sys_deb_common_deps.txt | 2 +- requirements/version.py | 7 + test/test_skill_install.py | 101 +++++++++ test/test_skill_intents.py | 112 ++++----- test/test_skill_osm_install.py | 31 --- test/test_skill_ressources.py | 7 +- 15 files changed, 386 insertions(+), 193 deletions(-) create mode 100644 .github/workflows/notify_pr_matrix.yml rename .github/workflows/{publish_stable_release.yml => publish_semver_release.yml} (94%) create mode 100644 requirements/version.py create mode 100644 test/test_skill_install.py delete mode 100644 test/test_skill_osm_install.py diff --git a/.github/workflows/notify_pr_matrix.yml b/.github/workflows/notify_pr_matrix.yml new file mode 100644 index 000000000..578b1db00 --- /dev/null +++ b/.github/workflows/notify_pr_matrix.yml @@ -0,0 +1,26 @@ +name: Notify Matrix Chat + +# only triggers on pull request closed events +on: + workflow_call: + inputs: + pr_id: + type: number + required: true + +jobs: + notify_pr_matrix: + runs-on: ubuntu-latest + env: + PACKAGE: "$(basename ${{ github.repository }})" + steps: + - uses: actions/checkout@v4 + - name: Send message to Matrix bots channel + id: matrix-chat-message + uses: fadenb/matrix-chat-message@v0.0.6 + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + new ${{ env.PACKAGE }} PR merged! https://github.com/${{ github.repository }}/pull/${{ inputs.pr_id }} \ No newline at end of file diff --git a/.github/workflows/propose_stable_release.yml b/.github/workflows/propose_stable_release.yml index 76b4a6e75..dbe38c907 100644 --- a/.github/workflows/propose_stable_release.yml +++ b/.github/workflows/propose_stable_release.yml @@ -15,6 +15,8 @@ on: default: version.py release_type: type: string + description: 'The type of release to make' + required: true changelog_file: type: string default: "CHANGELOG.md" @@ -25,6 +27,7 @@ jobs: runs-on: "ubuntu-latest" env: VERSION_FILE: ${{ inputs.version_file }} + TARGET_BRANCH: ${{ inputs.release_type == 'build' && 'testing' || 'stable' }} steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -71,4 +74,18 @@ jobs: with: commit_message: Update Changelog repository: action/package/ + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" + title: "Propose ${{ env.TARGET_BRANCH }} Release" + body: | + This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. + Please review the changelog and merge if it looks good. + branch: ${{ env.TARGET_BRANCH }} + branch-suffix: ${{ env.TARGET_BRANCH }} + base: ${{ inputs.branch }} + draft: false + assignees: ${{ github.actor }} \ No newline at end of file diff --git a/.github/workflows/publish_stable_release.yml b/.github/workflows/publish_semver_release.yml similarity index 94% rename from .github/workflows/publish_stable_release.yml rename to .github/workflows/publish_semver_release.yml index 0b920d81f..42c9ddf83 100644 --- a/.github/workflows/publish_stable_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -16,12 +16,14 @@ on: jobs: build_and_publish: + env: + SOURCE_BRANCH: ${{ inputs.release_type == 'build' && 'testing' || 'stable' }} runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: master + ref: ${{ env.SOURCE_BRANCH }} fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 2b8e79ccf..e6c181363 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -22,19 +22,27 @@ on: description: 'Python packages (whitespace delimited) to install instead of pip_tests.txt' required: false type: string - test_osm: - description: 'Whether to test OSM installation' - type: boolean - default: true + skill_id: + description: 'Skill ID to test' + required: true + type: string + skill_location: + description: 'Relative path to skill' + type: string + default: '' + jobs: - test_osm_install: + test_skill_install: runs-on: ${{inputs.runner}} timeout-minutes: 5 - if: ${{ inputs.test_osm }} strategy: matrix: python-version: ${{ fromJson(inputs.python_matrix) }} steps: + - name: Checkout Skill Repo + uses: actions/checkout@v4 + with: + path: action/skill/ - name: Checkout Scripts Repo uses: actions/checkout@v4 with: @@ -56,12 +64,12 @@ jobs: run: | pip install --upgrade pip if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/skill/requirements/pip_tests.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_tests.txt else pip install ${{inputs.pip_packages}} fi - - name: Test OSM Installation + - name: Test Skill Installation run: | - export TEST_REPO=${{ github.repository }} - export TEST_BRANCH=${{inputs.branch}} - pytest ${{ github.workspace }}/action/github/test/test_skill_osm_install.py + export TEST_SKILL_PATH=${{ github.workspace }}/action/skill/${{inputs.skill_location}} + export TEST_SKILL_ENTRYPOINT_NAME=${{ inputs.skill_id }} + pytest ${{ github.workspace }}/action/github/test/test_skill_install.py diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index c0c9a0937..6d5d4adfb 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -24,20 +24,27 @@ on: description: 'Optional system dependencies to install instead of requirements.txt' required: false type: string + skill_id: + description: 'Skill ID to test' + required: true + type: string intent_file: + description: 'Intent file to test' type: string - default: test/test_intents.yaml + default: test/unittests/intent_tests.yaml test_padatious: + description: 'Whether to skip padatious tests' type: boolean default: False test_padacioso: + description: 'Whether to skip padacioso tests' type: boolean default: True jobs: test_intents_ovos: env: - TEST_SKILL_ENTRYPOINT: "action/skill/" + TEST_SKILL_ENTRYPOINT_NAME: ${{ inputs.skill_id }} runs-on: ${{inputs.runner}} timeout-minutes: ${{inputs.timeout}} strategy: @@ -76,6 +83,9 @@ jobs: fi cd ${{ github.workspace }}/action/skill pip install . + # when skill names are normalized + # - name: Set TEST_SKILL_ENTRYPOINT_NAME + # run: echo "TEST_SKILL_ENTRYPOINT_NAME=$(basename ${{ github.repository }} | sed 's/-/_/g').$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index f4133f522..c50e0fa98 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -22,6 +22,10 @@ on: description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' required: false type: string + skill_id: + description: 'Skill ID to test' + required: true + type: string resource_testfile: type: string default: test/unittests/test_resources.yaml @@ -29,7 +33,7 @@ on: jobs: test_resources: env: - TEST_SKILL_ENTRYPOINT: "action/skill/" + TEST_SKILL_ENTRYPOINT_NAME: ${{inputs.skill_id}} runs-on: ${{inputs.runner}} timeout-minutes: 5 steps: diff --git a/README.md b/README.md index 1788f6ed3..dfb54c5d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ ## Workflows/Actions +**You can also find the implementation of the workflows in the [`skill-template-repo`](https://github.com/OpenVoiceOS/skill-template-repo)** + ### Publish Alpha Release (example: `ovos-core`) ```yaml name: Publish Alpha Build ...aX @@ -22,43 +24,60 @@ on: jobs: build_and_publish: uses: openvoiceos/.github/.github/workflows/publish_alpha_release.yml@main - with: - branch: dev # Branch to use, default: branch that triggered the action - version_file: ovos_core/version.py # File location of the version file, default: version.py - python_version: "3.8" # Python version (quoted) to use, default: 3.8 - locale_folder: ovos_core/locale # if there are localisation files the location of the base folder, default: locale - changelog_file: CHANGELOG.md # if the changlog file has a special name, default: CHANGELOG.md + secrets: inherit + with: + branch: dev # Branch to use, default: branch that triggered the action + version_file: ovos_core/version.py # File location of the version file, default: version.py + python_version: "3.8" # Python version (quoted) to use, default: 3.8 + locale_folder: ovos_core/locale # if there are localisation files the location of the base folder, default: locale + changelog_file: CHANGELOG.md # if the changlog file has a special name, default: CHANGELOG.md ``` ## Propose and Publish Stable (Build,Minor,Major) Release Strategy: 2-fold -*Proposal (prepares bump and pull requests to master)* +*Proposal (prepares bump and pull requests to testing/stable)* ```yaml name: Propose Stable Build on: workflow_dispatch: + inputs: + release_type: + type: choice + options: + - "build" + - "minor" + - "major" jobs: build_and_publish: - uses: openvoiceos/.github/.github/workflows/propose_stable_release.yml@main - with: - branch: dev # Branch to use, default: branch that triggered the action - python_version: '"3.10"' # Python version (quoted) to use, default: 3.8 - version_file: ovos_core/version.py # File location of the version file, default: version.py - release_type: build # build, minor, major - changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md + uses: openvoiceos/.github/.github/workflows/propose_semver_release.yml@main + with: + branch: dev # Branch to use, default: branch that triggered the action + python_version: '"3.10"' # Python version (quoted) to use, default: 3.8 + version_file: ovos_core/version.py # File location of the version file, default: version.py + release_type: ${{inputs.release_type}} # build, minor, major + changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md ``` -*Publishing (from master)* +*Publishing* ```yaml name: Publish Stable Build on: workflow_dispatch: + inputs: + release_type: + type: choice + options: + - "build" + - "minor" + - "major" jobs: build_and_publish: uses: openvoiceos/.github/.github/workflows/publish_stable_release.yml@main - with: - python_version: '"3.10"' - changelog_file: ChAnGeLoG.md + secrets: inherit + with: + python_version: '"3.10"' + release_type: ${{inputs.release_type}} + changelog_file: ChAnGeLoG.md ``` ## Propose translatios Introduce a new language localisation by proposing a translation via pull request. (creating new branch staging/translation_xx-xx) @@ -81,12 +100,13 @@ on: jobs: propose_translation: uses: openvoiceos/.github/.github/workflows/propose_translation.yml@main - with: - branch: dev # Branch to use, default: branch that triggered the action - python_version: '"3.8"' - language: ${{ inputs.translation }} - locale_folder: ovos_core/locale/ # the location of the base localisation folder, default: locale - reviewers: "jarbasai,emphasize" # comma separated list of reviewers, default: emphasize + secrets: inherit + with: + branch: dev # Branch to use, default: branch that triggered the action + python_version: '"3.8"' + language: ${{ inputs.translation }} + locale_folder: ovos_core/locale/ # the location of the base localisation folder, default: locale + reviewers: "jarbasai,emphasize" # comma separated list of reviewers, default: emphasize ``` * [available languages with deepl](https://support.deepl.com/hc/en-us/articles/360019925219-Languages-included-in-DeepL-Pro) @@ -100,17 +120,17 @@ on: jobs: license_tests: uses: openvoiceos/.github/.github/workflows/license_tests.yml@main - with: - runner: ubuntu-latest # Runner to use, default: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - python-version: '"3.8"' # Python version (quoted) to use, default: 3.8 - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - package-extras: test # Optional extras to install the python package with - packages-exclude: '^(fann2|tqdm|bs4).*' # Custom regex to exclude packages from the license check - # default: '^(precise-runner|fann2|tqdm|bs4|nvidia|bitstruct).*' - licenses-exclude: ^(BSD-3).*$' # Custom regex to exclude licenses from the license check - # default: '^(Mozilla|NeonAI License v1.0).*$' + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + python-version: '"3.8"' # Python version (quoted) to use, default: 3.8 + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + package-extras: test # Optional extras to install the python package with + packages-exclude: '^(fann2|tqdm|bs4).*' # Custom regex to exclude packages from the license check + # default: '^(precise-runner|fann2|tqdm|bs4|nvidia|bitstruct).*' + licenses-exclude: ^(BSD-3).*$' # Custom regex to exclude licenses from the license check + # default: '^(Mozilla|NeonAI License v1.0).*$' ``` ## Build testing Tests the build of the python package. @@ -122,15 +142,15 @@ on: jobs: build_tests: uses: openvoiceos/.github/.github/workflows/python_build_tests.yml@main - with: - runner: ubuntu-latest # Runner to use, default: ubuntu-latest - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - test_manifest: true # if to test with MANIFEST.in, default: false - manifest_ignored: "test/** qt5/**" # Files to ignore in MANIFEST.in, default: "test/**" - test_relative_paths: false # if to test with relative paths, default: true - test_pipaudit: true # if to test with pip-audit, default: false - pipaudit_ignored: "" # Vulnerabilities to ignore in pip-audit, - # default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + test_manifest: true # if to test with MANIFEST.in, default: false + manifest_ignored: "test/** qt5/**" # Files to ignore in MANIFEST.in, default: "test/**" + test_relative_paths: false # if to test with relative paths, default: true + test_pipaudit: true # if to test with pip-audit, default: false + pipaudit_ignored: "" # Vulnerabilities to ignore in pip-audit, + # default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" ``` ## Skills ## Skill Unit Tests @@ -142,15 +162,15 @@ on: jobs: build_tests: uses: openvoiceos/.github/.github/workflows/skill_tests.yml@main - with: - runner: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py + with: + runner: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py ``` ## Skill Installation Tests ```yaml @@ -161,17 +181,18 @@ on: jobs: skill_installation_tests: uses: openvoiceos/.github/.github/workflows/skill_test_installation.yml@main - with: - runner: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py - test_osm: true # if to test with osm, default: true - test_loader: true # if to test with skillloader, default: true + with: + runner: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py + skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + test_osm: true # if to test with osm, default: true + test_loader: true # if to test with skillloader, default: true ``` ## Skill Resource Tests Tests the resources of a skill. This may be dialogs, vocabs, regex or intent resources @@ -183,15 +204,16 @@ on: jobs: skill_resource_tests: uses: openvoiceos/.github/.github/workflows/skill_test_resources.yml@main - with: - runner: ubuntu-latest # Runner to use, default: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_version: '"3.10"' # Python version to use, default: "3.8" - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_version: '"3.10"' # Python version to use, default: "3.8" + pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml ``` ## Skill Intent Tests Tests if intents are successfully parsed by the skill @@ -203,20 +225,38 @@ on: jobs: skill_intent_tests: uses: openvoiceos/.github/.github/workflows/skill_test_intents.yml@main - with: - runner: ubuntu-latest # Runner to use, default: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - timeout: 15 # Timeout for the test, default: 15 - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml - test_padatious: true # if to test against padatious, default: false - test_padacioso: true # if to test against padacioso, default: true + with: + runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + timeout: 15 # Timeout for the test, default: 15 + system-deps: "libfann-dev libfann2" # Custom system dependencies to install before running the license check + # default packages found at requirements/sys_deb_common_deps.txt + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml + skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + test_padatious: true # if to test against padatious, default: false + test_padacioso: true # if to test against padacioso, default: true +``` +## Notify Matrix on Pull Request +```yaml +name: Notify Matrix Chat + +# only trigger on pull request closed events +on: + pull_request: + types: [ closed ] + +jobs: + notify_pr_matrix: + # this job will only run if the PR has been merged + if: github.event.pull_request.merged == true + secrets: inherit + uses: openvoiceos/.github/.github/workflows/notify_pr_matrix.yml@feat/shared_actions1 + with: + pr_id: ${{ github.event.number }} ``` - ------------- ![OpenVoiceOS](https://openvoiceos.com/wp-content/uploads/2020/10/loading400.png) diff --git a/requirements/pip_skill_tests.txt b/requirements/pip_skill_tests.txt index ba37c4d63..0ea9f0319 100644 --- a/requirements/pip_skill_tests.txt +++ b/requirements/pip_skill_tests.txt @@ -1,4 +1,9 @@ pytest pytest-cov mock -ovos-core[skills]>=0.0.7 +ovos-core>=0.0.8a66 + +ovos-phal-plugin-connectivity-events~=0.0.1 +padatious>=0.4.8, < 0.5.0 +fann2>=1.0.7, < 1.1.0 +padaos>=0.1, < 0.2 \ No newline at end of file diff --git a/requirements/pip_tests.txt b/requirements/pip_tests.txt index 044eaf15b..2546e64f5 100644 --- a/requirements/pip_tests.txt +++ b/requirements/pip_tests.txt @@ -1,3 +1,4 @@ pytest +pytest-cov mock -ovos-skills-manager \ No newline at end of file +ovos-skills-manager diff --git a/requirements/sys_deb_common_deps.txt b/requirements/sys_deb_common_deps.txt index 5912d6e86..df7073817 100644 --- a/requirements/sys_deb_common_deps.txt +++ b/requirements/sys_deb_common_deps.txt @@ -4,4 +4,4 @@ swig libssl-dev libfann-dev portaudio19-dev -libpulse-dev \ No newline at end of file +libpulse-dev diff --git a/requirements/version.py b/requirements/version.py new file mode 100644 index 000000000..1ef7642f3 --- /dev/null +++ b/requirements/version.py @@ -0,0 +1,7 @@ +# version file used in all ovos python packages +# START_VERSION_BLOCK +VERSION_MAJOR = 0 +VERSION_MINOR = 0 +VERSION_BUILD = 1 +VERSION_ALPHA = 0 +# END_VERSION_BLOCK \ No newline at end of file diff --git a/test/test_skill_install.py b/test/test_skill_install.py new file mode 100644 index 000000000..4eca995f8 --- /dev/null +++ b/test/test_skill_install.py @@ -0,0 +1,101 @@ +import unittest +from typing import Optional +from os import environ +from os.path import isdir +from shutil import rmtree +from pathlib import Path + +from ovos_utils.log import LOG +from ovos_utils.messagebus import FakeBus +from ovos_config.config import update_mycroft_config, Configuration +from ovos_workshop.skills.base import BaseSkill +from ovos_workshop.skill_launcher import PluginSkillLoader, SkillLoader +from ovos_plugin_manager.skills import find_skill_plugins + + +def get_skill_object(bus: FakeBus, path: str = "", + skill_id: str = "", config_patch: Optional[dict] = None) -> BaseSkill: + """ + Get an initialized skill object laoded from path or using the plugin manager. + @param bus: FakeBus instance to bind to skill for testing + @param path: directory path the skill should be loaded from + @param skill_id: skill_id to initialize skill with + @param config_patch: Configuration update to apply + @returns: Initialized skill object + """ + if config_patch: + user_config = update_mycroft_config(config_patch) + if user_config not in Configuration.xdg_configs: + Configuration.xdg_configs.append(user_config) + if path: + if not isdir(path): + raise FileNotFoundError(path) + LOG.info(f"Loading local skill from: {path}") + loader = SkillLoader(bus, path, skill_id) + if loader.load(): + return loader.instance + plugins = find_skill_plugins() + if skill_id not in plugins: + raise ValueError(f"Requested skill not found: {skill_id}") + else: + LOG.info(f"Loading skill from plugin: {skill_id}") + plugin = plugins[skill_id] + skill = plugin(bus=bus, skill_id=skill_id) + return skill + + +# @unittest.skipIf(environ.get("TEST_OSM", "false") == "false", "TEST_OSM is false") +# class TestOSM(unittest.TestCase): +# def test_osm_install(self): +# branch = environ.get("TEST_BRANCH") +# install_url = f"https://github.com/{environ.get('TEST_REPO')}@{branch}" +# skill = SkillEntry.from_github_url(install_url) +# tmp_skills = "/tmp/osm_installed_skills" +# skill_folder = f"{tmp_skills}/{skill.uuid}" + +# if exists(skill_folder): +# rmtree(skill_folder) + +# updated = skill.install(folder=tmp_skills, default_branch=branch) +# self.assertEqual(updated, True) +# self.assertTrue(isdir(skill_folder)) +# self.assertTrue(isfile(join(skill_folder, "__init__.py"))) + +# updated = skill.install(folder=tmp_skills, default_branch=branch) +# self.assertEqual(updated, False) + + +class TestSkillLoading(unittest.TestCase): + @classmethod + def setUpClass(self): + self.skill_id = environ.get("TEST_SKILL_ENTRYPOINT_NAME") + self.path = str(Path(environ.get("TEST_SKILL_PATH"))) + + def test_from_plugin(self): + bus = FakeBus() + skill = get_skill_object(bus, skill_id=self.skill_id) + self.assertEqual(skill.bus, bus) + self.assertEqual(skill.root_dir, self.path) + + def test_from_loader(self): + bus = FakeBus() + skill = get_skill_object(bus, path=self.path) + self.assertEqual(skill.bus, bus) + self.assertEqual(skill.root_dir, self.path) + + def test_from_plugin_loader(self): + bus = FakeBus() + loader = PluginSkillLoader(bus, self.skill_id) + for skill_id, plug in find_skill_plugins().items(): + if skill_id == self.skill_id: + loader.load(plug) + break + else: + raise RuntimeError("plugin not found") + + self.assertEqual(loader.skill_id, self.skill_id) + self.assertEqual(loader.instance.bus, bus) + self.assertEqual(loader.instance.skill_id, self.skill_id) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/test/test_skill_intents.py b/test/test_skill_intents.py index d5a6b2fd3..e89def13f 100644 --- a/test/test_skill_intents.py +++ b/test/test_skill_intents.py @@ -1,47 +1,34 @@ -# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework -# All trademark and other rights reserved by their respective owners -# Copyright 2008-2022 Neongecko.com Inc. -# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, -# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo -# BSD-3 License -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - import logging from os import getenv +from os.path import isdir +from typing import Optional import importlib import unittest import yaml from mock import Mock, patch -from ovos_utils.log import LOG -from mycroft_bus_client import Message -from ovos_config.config import update_mycroft_config +from ovos_core.intent_services import PadatiousService, PadatiousMatcher +from ovos_bus_client import Message +from ovos_bus_client.session import Session, SessionManager +from ovos_config.config import update_mycroft_config, Configuration from ovos_utils.messagebus import FakeBus -from mycroft.skills.intent_services.padatious_service import PadatiousMatcher +from ovos_utils.log import LOG +from ovos_plugin_manager.skills import find_skill_plugins +from ovos_workshop.skill_launcher import SkillLoader +from ovos_workshop.skills.base import BaseSkill -regex_only = getenv("INTENT_ENGINE") == "padacioso" +PIPELINE = ["adapt_high", "adapt_medium", "adapt_low"] +use_padacioso = getenv("INTENT_ENGINE") == "padacioso" +if use_padacioso: + PIPELINE.extend(["padacioso_high", + "padacioso_medium", + "padacioso_low"]) +else: + PIPELINE.extend(["padatious_high", + "padatious_medium", + "padatious_low"]) LOG.level = logging.DEBUG @@ -66,15 +53,30 @@ def match_low(self, utterances, lang=None, __=None): PadatiousMatcher.match_low(self, utterances, lang=lang) -def get_skill_class(): - from ovos_plugin_manager.skills import find_skill_plugins +def get_skill_object(skill_entrypoint: str, bus: FakeBus, + skill_id: str, config_patch: Optional[dict] = None) -> BaseSkill: + """ + Get an initialized skill object by entrypoint with the requested skill_id. + @param skill_entrypoint: Skill plugin entrypoint or directory path + @param bus: FakeBus instance to bind to skill for testing + @param skill_id: skill_id to initialize skill with + @param config_patch: Configuration update to apply + @returns: Initialized skill object + """ + if config_patch: + user_config = update_mycroft_config(config_patch) + if user_config not in Configuration.xdg_configs: + Configuration.xdg_configs.append(user_config) + if isdir(skill_entrypoint): + LOG.info(f"Loading local skill: {skill_entrypoint}") + loader = SkillLoader(bus, skill_entrypoint, skill_id) + if loader.load(): + return loader.instance plugins = find_skill_plugins() - plugin_id = getenv("TEST_SKILL_ID") - if plugin_id: - skill = plugins.get(plugin_id) - else: - assert len(plugins.values()) == 1 - skill = list(plugins.values())[0] + if skill_entrypoint not in plugins: + raise ValueError(f"Requested skill not found: {skill_entrypoint}") + plugin = plugins[skill_entrypoint] + skill = plugin(bus=bus, skill_id=skill_id) return skill @@ -84,29 +86,28 @@ class TestSkillIntentMatching(unittest.TestCase): valid_intents = yaml.safe_load(f) negative_intents = valid_intents.pop('unmatched intents', dict()) common_query = valid_intents.pop("common query", dict()) + skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT_NAME") # Ensure all tested languages are loaded import ovos_config - update_mycroft_config({"secondary_langs": list(valid_intents.keys()), - "padatious": { - "regex_only": regex_only, - # TODO: below config patching ovos-core default config - # https://github.com/OpenVoiceOS/ovos-config/pull/78/files#r1439966369 - "intent_cache": "~/.local/share/mycroft/intent_cache", - "train_delay": 4, - "single_thread": False}}) + update_mycroft_config({"secondary_langs": list(valid_intents.keys())}) importlib.reload(ovos_config.config) + + # make the default session use the test pipeline + session = Session("default", pipeline=PIPELINE) + SessionManager.default_session = session + SessionManager.sessions = {"default": session} + # Start the IntentService bus = FakeBus() - from mycroft.skills.intent_service import IntentService + from ovos_core.intent_services import IntentService intent_service = IntentService(bus) - assert intent_service.padatious_service.is_regex_only == regex_only # Create the skill to test - # TODO: Refactor to use ovos-workshop - skill_class = get_skill_class() test_skill_id = 'test_skill.test' - skill = skill_class(skill_id=test_skill_id, bus=bus) + skill = get_skill_object(skill_entrypoint=skill_entrypoint, + bus=bus, + skill_id=test_skill_id) assert skill.config_core["secondary_langs"] == list(valid_intents.keys()) last_message = None @@ -185,7 +186,7 @@ def test_intents(self): value, utt) intent_handler.reset_mock() - @patch("mycroft.skills.intent_service.PadatiousMatcher", + @patch("ovos_core.intent_services.padacioso_service.PadaciosoService", new=MockPadatiousMatcher) def test_negative_intents(self): test_config = self.negative_intents.pop('config', None) @@ -194,6 +195,7 @@ def test_negative_intents(self): True) MockPadatiousMatcher.include_low = test_config.get('include_low', False) + intent_failure = Mock() self.intent_service.send_complete_intent_failure = intent_failure diff --git a/test/test_skill_osm_install.py b/test/test_skill_osm_install.py deleted file mode 100644 index 344279fb0..000000000 --- a/test/test_skill_osm_install.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest - -from os import environ -from os.path import exists, isdir, isfile, join -from shutil import rmtree - -from ovos_skills_manager import SkillEntry - - -class TestOSM(unittest.TestCase): - def test_osm_install(self): - branch = environ.get("TEST_BRANCH") - install_url = f"https://github.com/{environ.get('TEST_REPO')}@{branch}" - skill = SkillEntry.from_github_url(install_url) - tmp_skills = "/tmp/osm_installed_skills" - skill_folder = f"{tmp_skills}/{skill.uuid}" - - if exists(skill_folder): - rmtree(skill_folder) - - updated = skill.install(folder=tmp_skills, default_branch=branch) - self.assertEqual(updated, True) - self.assertTrue(isdir(skill_folder)) - self.assertTrue(isfile(join(skill_folder, "__init__.py"))) - - updated = skill.install(folder=tmp_skills, default_branch=branch) - self.assertEqual(updated, False) - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/test/test_skill_ressources.py b/test/test_skill_ressources.py index 7f8214eac..12ad078b2 100644 --- a/test/test_skill_ressources.py +++ b/test/test_skill_ressources.py @@ -91,7 +91,7 @@ class TestSkillResources(unittest.TestCase): # Define skill and resource spec to use in tests resources = load_resource_tests(getenv("RESOURCE_TEST_FILE")) - skill_entrypoint = getenv("TEST_SKILL_ENTRYPOINT") + skill_entrypoint_name = getenv("TEST_SKILL_ENTRYPOINT_NAME") # Specify valid languages to test supported_languages = resources['languages'] @@ -113,8 +113,9 @@ class TestSkillResources(unittest.TestCase): def setUpClass(cls) -> None: cls.bus.on("message", cls._on_message) - cls.skill = get_skill_object(skill_entrypoint=cls.skill_entrypoint, - bus=cls.bus, skill_id=cls.test_skill_id, + cls.skill = get_skill_object(skill_entrypoint=cls.skill_entrypoint_name, + bus=cls.bus, + skill_id=cls.test_skill_id, config_patch=cls.core_config_patch) cls.adapt_intents = {f'{cls.test_skill_id}:{intent}' From 4d8ef84481c719f4f961738896887ad0a0e322a6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 15:13:25 +0100 Subject: [PATCH 011/357] remove extra quotes --- .github/workflows/auto_translate.yml | 2 +- ...ropose_stable_release.yml => propose_semver_release.yml} | 6 +++--- .github/workflows/propose_translation.yml | 2 +- .github/workflows/publish_alpha_release.yml | 4 ++-- .github/workflows/publish_semver_release.yml | 2 +- .github/workflows/python_build_tests.yml | 4 ++-- .github/workflows/skill_test_installation.yml | 2 +- README.md | 6 +++--- 8 files changed, 14 insertions(+), 14 deletions(-) rename .github/workflows/{propose_stable_release.yml => propose_semver_release.yml} (97%) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index 5d8083416..523135ff0 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -13,7 +13,7 @@ on: locale_folder: description: 'The base folder to look for localization files' type: string - default: "locale" + default: locale jobs: diff --git a/.github/workflows/propose_stable_release.yml b/.github/workflows/propose_semver_release.yml similarity index 97% rename from .github/workflows/propose_stable_release.yml rename to .github/workflows/propose_semver_release.yml index dbe38c907..3aa294dae 100644 --- a/.github/workflows/propose_stable_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -4,7 +4,7 @@ on: inputs: branch: type: string - default: "dev" + default: dev python_version: description: 'The python version to use' type: string @@ -19,12 +19,12 @@ on: required: true changelog_file: type: string - default: "CHANGELOG.md" + default: CHANGELOG.md jobs: bump_version: - runs-on: "ubuntu-latest" + runs-on: ubuntu-latest env: VERSION_FILE: ${{ inputs.version_file }} TARGET_BRANCH: ${{ inputs.release_type == 'build' && 'testing' || 'stable' }} diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index 59c5761a7..ce45ebb47 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -17,7 +17,7 @@ on: locale_folder: description: 'The base folder to look for localization files' type: string - default: "locale" + default: locale reviewers: description: 'A comma or newline-separated string of reviewers' type: string diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index 22add822c..c23547f77 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -19,10 +19,10 @@ on: locale_folder: description: 'The base folder to look for localization files' type: string - default: "locale" + default: locale changelog_file: type: string - default: "CHANGELOG.md" + default: CHANGELOG.md jobs: autotranslate: diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index 42c9ddf83..6c7f0add4 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -12,7 +12,7 @@ on: changelog_file: description: "File to write the changelog to." type: string - default: "CHANGELOG.md" + default: CHANGELOG.md jobs: build_and_publish: diff --git a/.github/workflows/python_build_tests.yml b/.github/workflows/python_build_tests.yml index c55fb3b63..0be4f7f11 100644 --- a/.github/workflows/python_build_tests.yml +++ b/.github/workflows/python_build_tests.yml @@ -5,7 +5,7 @@ on: runner: description: 'Runner to use' type: string - default: "ubuntu-latest" + default: ubuntu-latest python_matrix: description: 'Python matrix (string) to use' type: string @@ -30,7 +30,7 @@ on: default: true pipaudit_ignored: type: string - default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" + default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012 PYSEC-2023-227" jobs: py_build_tests: timeout-minutes: 15 diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index e6c181363..61f89284f 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -5,7 +5,7 @@ on: runner: description: 'Runner to use' type: string - default: "ubuntu-latest" + default: ubuntu-latest branch: description: 'Branch to use' type: string diff --git a/README.md b/README.md index dfb54c5d3..00451c9b6 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ jobs: pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py - skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required test_osm: true # if to test with osm, default: true test_loader: true # if to test with skillloader, default: true ``` @@ -212,7 +212,7 @@ jobs: python_version: '"3.10"' # Python version to use, default: "3.8" pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml ``` ## Skill Intent Tests @@ -235,7 +235,7 @@ jobs: pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml - skill_id: "ovos_skill_x.openvoiceos" # Skill id of the testskill, required + skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true ``` From e850c84182c48abf1e98855147393090578ed11b Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 15:30:39 +0100 Subject: [PATCH 012/357] upgrade pip --- .github/workflows/publish_alpha_release.yml | 1 + .github/workflows/publish_semver_release.yml | 1 + .github/workflows/python_build_tests.yml | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index c23547f77..6bcde752f 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -119,6 +119,7 @@ jobs: prerelease: true - name: Install Build Tools run: | + pip install --upgrade pip python -m pip install build wheel - name: Build Distribution Packages run: | diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index 6c7f0add4..eda1b84eb 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -59,6 +59,7 @@ jobs: ${{ steps.changelog.outputs.changelog }} - name: Install Build Tools run: | + pip install --upgrade pip python -m pip install build wheel - name: Build Distribution Packages run: | diff --git a/.github/workflows/python_build_tests.yml b/.github/workflows/python_build_tests.yml index 0be4f7f11..5fc8234f8 100644 --- a/.github/workflows/python_build_tests.yml +++ b/.github/workflows/python_build_tests.yml @@ -30,7 +30,7 @@ on: default: true pipaudit_ignored: type: string - default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012 PYSEC-2023-227" + default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" jobs: py_build_tests: timeout-minutes: 15 @@ -48,6 +48,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Build Tools run: | + pip install --upgrade pip python -m pip install build wheel - name: Build Distribution Packages run: | From 4de43eeee51bb1c1b0244fa7647def2f4440a965 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 15:44:30 +0100 Subject: [PATCH 013/357] kick py 3.12 tests --- .github/workflows/license_tests.yml | 2 +- .github/workflows/python_build_tests.yml | 2 +- .github/workflows/skill_test_installation.yml | 2 +- .github/workflows/skill_test_intents.yml | 2 +- .github/workflows/skill_tests.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 388bb4cb0..49a0875d2 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -56,7 +56,7 @@ jobs: run: | sudo apt-get update if [ -z ${{ inputs.system-deps }} ]; then - xargs sudo apt-get install -y < action/github/sys_deb_common_deps.txt + xargs sudo apt-get install -y < action/github/requirements/sys_deb_common_deps.txt else sudo apt-get install ${{ inputs.system-deps }} fi diff --git a/.github/workflows/python_build_tests.yml b/.github/workflows/python_build_tests.yml index 5fc8234f8..1845e1ac2 100644 --- a/.github/workflows/python_build_tests.yml +++ b/.github/workflows/python_build_tests.yml @@ -9,7 +9,7 @@ on: python_matrix: description: 'Python matrix (string) to use' type: string - default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + default: '["3.8", "3.9", "3.10", "3.11"]' test_manifest: description: 'if to test MANIFEST.in' required: false diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 61f89284f..507a3e123 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -13,7 +13,7 @@ on: python_matrix: description: 'Python versions to test against' type: string - default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + default: '["3.8", "3.9", "3.10", "3.11"]' system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 6d5d4adfb..dac4111bd 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -15,7 +15,7 @@ on: python_matrix: description: 'Python matrix (string) to use' type: string - default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: description: 'Custom pip packages (whitespace delimited) to install instead of pip_skill_tests.txt' required: false diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index acc7e5a69..ffadc0baf 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -16,7 +16,7 @@ on: python_matrix: description: 'Python matrix (string) to use' type: string - default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' required: false From 6a456cdbff603b1edd585b53150006d230697d9d Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 16:11:01 +0100 Subject: [PATCH 014/357] fix rel requirements path --- .github/workflows/license_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 49a0875d2..a0e9fdd1d 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -56,7 +56,7 @@ jobs: run: | sudo apt-get update if [ -z ${{ inputs.system-deps }} ]; then - xargs sudo apt-get install -y < action/github/requirements/sys_deb_common_deps.txt + xargs sudo apt-get install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt else sudo apt-get install ${{ inputs.system-deps }} fi From a8cbbbf24f381b951e3d6ee9373a725d1b9aeeea Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 16:28:21 +0100 Subject: [PATCH 015/357] temporarily checkout SA feature branch --- .github/workflows/auto_translate.yml | 1 + .github/workflows/license_tests.yml | 1 + .github/workflows/propose_semver_release.yml | 1 + .github/workflows/propose_translation.yml | 1 + .github/workflows/publish_alpha_release.yml | 2 ++ .github/workflows/skill_test_installation.yml | 1 + .github/workflows/skill_test_intents.yml | 1 + .github/workflows/skill_test_resources.yml | 1 + .github/workflows/skill_tests.yml | 1 + 9 files changed, 10 insertions(+) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index 523135ff0..2381c0f93 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -32,6 +32,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index a0e9fdd1d..3c7a30baa 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -44,6 +44,7 @@ jobs: uses: actions/checkout@v4 with: path: action/github/ + ref: feat/shared_actions1 repository: OpenVoiceOS/.github - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 3aa294dae..7479cbdc5 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -38,6 +38,7 @@ jobs: uses: actions/checkout@v4 with: path: action/github/ + ref: feat/shared_actions1 repository: OpenVoiceOS/.github - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index ce45ebb47..c8c45f423 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -41,6 +41,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index 6bcde752f..066cbb908 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -40,6 +40,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - uses: dorny/paths-filter@v2 id: filter @@ -81,6 +82,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 507a3e123..1ec74a7aa 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -47,6 +47,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index dac4111bd..f5476de0c 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -59,6 +59,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index c50e0fa98..59ea35694 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -46,6 +46,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Set up python uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index ffadc0baf..13b5c0d6d 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -43,6 +43,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: feat/shared_actions1 path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 From 601f5ccf24f3a003601efac42d59f1c137b50895 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 16:33:34 +0100 Subject: [PATCH 016/357] license checker rel path --- .github/workflows/license_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 3c7a30baa..39386206b 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -76,7 +76,7 @@ jobs: id: license_check_report uses: pilosus/action-pip-license-checker@v2 with: - requirements: '${{ github.workspace }}/action/package/requirements-all.txt' + requirements: 'action/package/requirements-all.txt' fail: 'Copyleft,Other,Error' fails-only: true exclude: ${{ inputs.packages-exclude }} From 1030007a9124fb15380080cd8d2c232c3b01c832 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 16:36:57 +0100 Subject: [PATCH 017/357] licence checker no path --- .github/workflows/license_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 39386206b..32c8ac8d2 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -76,7 +76,7 @@ jobs: id: license_check_report uses: pilosus/action-pip-license-checker@v2 with: - requirements: 'action/package/requirements-all.txt' + requirements: 'requirements-all.txt' fail: 'Copyleft,Other,Error' fails-only: true exclude: ${{ inputs.packages-exclude }} From 40db1453d615e93b2812ac328074c9e83278eec7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 17:04:52 +0100 Subject: [PATCH 018/357] adjust naming --- .github/workflows/skill_test_intents.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index f5476de0c..7e4a5c7a1 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -28,7 +28,7 @@ on: description: 'Skill ID to test' required: true type: string - intent_file: + intent_testfile: description: 'Intent file to test' type: string default: test/unittests/intent_tests.yaml diff --git a/README.md b/README.md index 00451c9b6..2d255d2d0 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ on: jobs: - build_tests: + skill_tests: uses: openvoiceos/.github/.github/workflows/skill_tests.yml@main with: runner: ubuntu-latest From 196363d393d5e9450b0399ad09052abd3be48e0d Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 17:15:13 +0100 Subject: [PATCH 019/357] adjust skill requirement path --- .github/workflows/skill_test_intents.yml | 2 +- .github/workflows/skill_test_resources.yml | 2 +- .github/workflows/skill_tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 7e4a5c7a1..8663aaadd 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -78,7 +78,7 @@ jobs: pip install --upgrade pip pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt else pip install ${{inputs.pip_packages}} fi diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 59ea35694..1b04c86d5 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -65,7 +65,7 @@ jobs: pip install --upgrade pip pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt else pip install ${{inputs.pip_packages}} fi diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index 13b5c0d6d..50963850a 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -62,7 +62,7 @@ jobs: pip install --upgrade pip pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/skill/requirements/pip_skill_tests.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt else pip install ${{inputs.pip_packages}} fi From b8e3d5e9a3083ef3332ce181b156cd1f2ce30860 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 18:45:21 +0100 Subject: [PATCH 020/357] skill wf adjustments --- .github/workflows/skill_test_installation.yml | 5 +++-- .github/workflows/skill_test_intents.yml | 7 ++++--- .github/workflows/skill_test_resources.yml | 10 +++++----- .github/workflows/skill_tests.yml | 4 ++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 1ec74a7aa..f54abb2e8 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -19,7 +19,7 @@ on: required: false type: string pip_packages: - description: 'Python packages (whitespace delimited) to install instead of pip_tests.txt' + description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' required: false type: string skill_id: @@ -42,6 +42,7 @@ jobs: - name: Checkout Skill Repo uses: actions/checkout@v4 with: + ref: ${{inputs.branch}} path: action/skill/ - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -65,7 +66,7 @@ jobs: run: | pip install --upgrade pip if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_tests.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt else pip install ${{inputs.pip_packages}} fi diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 8663aaadd..73558595e 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -51,9 +51,10 @@ jobs: matrix: python-version: ${{ fromJSON(inputs.python_matrix) }} steps: - - name: Checkout Repository + - name: Checkout Skill Repo uses: actions/checkout@v4 with: + ref: ${{inputs.branch}} path: action/skill/ - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -91,11 +92,11 @@ jobs: if: ${{ inputs.test_padacioso }} run: | export INTENT_ENGINE="padacioso" - export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_file }}" pytest action/github/test/test_skill_intents.py - name: Test Skill Intents Padatious if: ${{ inputs.test_padatious }} run: | export INTENT_ENGINE="padatious" - export INTENT_TEST_FILE="action/skill/${{inputs.intent_file}}" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_file }}" pytest action/github/test/test_skill_intents.py \ No newline at end of file diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 1b04c86d5..85f5c45a8 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -37,7 +37,7 @@ jobs: runs-on: ${{inputs.runner}} timeout-minutes: 5 steps: - - name: Checkout Repository + - name: Checkout Skill Repo uses: actions/checkout@v4 with: ref: ${{inputs.branch}} @@ -69,9 +69,9 @@ jobs: else pip install ${{inputs.pip_packages}} fi - cd ${{ github.workspace }}/action/skill/ - pip install . + cd ${{ github.workspace }}/action/skill + pip install . - name: Test Skill Resources run: | - export RESOURCE_TEST_FILE="action/skill/${{inputs.resource_testfile}}" - pytest action/github/test/test_skill_resources.py \ No newline at end of file + export RESOURCE_TEST_FILE="${{ github.workspace }}/action/skill/${{inputs.resource_testfile}}" + pytest ${{ github.workspace }}/action/github/test/test_skill_resources.py \ No newline at end of file diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index 50963850a..ca06b731e 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -34,7 +34,7 @@ jobs: runs-on: ${{inputs.runner}} timeout-minutes: 15 steps: - - name: Checkout Repository + - name: Checkout Skill Repo uses: actions/checkout@v4 with: ref: ${{inputs.branch}} @@ -66,7 +66,7 @@ jobs: else pip install ${{inputs.pip_packages}} fi - cd ${{ github.workspace }}/action/skill/ + cd ${{ github.workspace }}/action/skill pip install . - name: Test Skill run: | From ba504f18dae4f434c4c7bc595e31bc37865bebc7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 19:08:20 +0100 Subject: [PATCH 021/357] skill adjustments --- .github/workflows/skill_test_intents.yml | 4 ++-- test/{test_skill_ressources.py => test_skill_resources.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename test/{test_skill_ressources.py => test_skill_resources.py} (100%) diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 73558595e..addade8cd 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -92,11 +92,11 @@ jobs: if: ${{ inputs.test_padacioso }} run: | export INTENT_ENGINE="padacioso" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_file }}" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" pytest action/github/test/test_skill_intents.py - name: Test Skill Intents Padatious if: ${{ inputs.test_padatious }} run: | export INTENT_ENGINE="padatious" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_file }}" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" pytest action/github/test/test_skill_intents.py \ No newline at end of file diff --git a/test/test_skill_ressources.py b/test/test_skill_resources.py similarity index 100% rename from test/test_skill_ressources.py rename to test/test_skill_resources.py From 415aef6f091987cd11424d801c0aabc19f0576fb Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 19:14:12 +0100 Subject: [PATCH 022/357] log available skills --- test/test_skill_install.py | 2 +- test/test_skill_intents.py | 2 +- test/test_skill_resources.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_skill_install.py b/test/test_skill_install.py index 4eca995f8..9cc7a6b9f 100644 --- a/test/test_skill_install.py +++ b/test/test_skill_install.py @@ -36,7 +36,7 @@ def get_skill_object(bus: FakeBus, path: str = "", return loader.instance plugins = find_skill_plugins() if skill_id not in plugins: - raise ValueError(f"Requested skill not found: {skill_id}") + raise ValueError(f"Requested skill not found: {skill_id}; available skills: {list(plugins.keys())}") else: LOG.info(f"Loading skill from plugin: {skill_id}") plugin = plugins[skill_id] diff --git a/test/test_skill_intents.py b/test/test_skill_intents.py index e89def13f..d0c4718c3 100644 --- a/test/test_skill_intents.py +++ b/test/test_skill_intents.py @@ -74,7 +74,7 @@ def get_skill_object(skill_entrypoint: str, bus: FakeBus, return loader.instance plugins = find_skill_plugins() if skill_entrypoint not in plugins: - raise ValueError(f"Requested skill not found: {skill_entrypoint}") + raise ValueError(f"Requested skill not found: {skill_entrypoint}; available skills: {list(plugins.keys())}") plugin = plugins[skill_entrypoint] skill = plugin(bus=bus, skill_id=skill_id) return skill diff --git a/test/test_skill_resources.py b/test/test_skill_resources.py index 12ad078b2..f35675f3e 100644 --- a/test/test_skill_resources.py +++ b/test/test_skill_resources.py @@ -62,7 +62,7 @@ def get_skill_object(skill_entrypoint: str, bus: FakeBus, return loader.instance plugins = find_skill_plugins() if skill_entrypoint not in plugins: - raise ValueError(f"Requested skill not found: {skill_entrypoint}") + raise ValueError(f"Requested skill not found: {skill_entrypoint}; available skills: {list(plugins.keys())}") plugin = plugins[skill_entrypoint] skill = plugin(bus=bus, skill_id=skill_id) return skill From dd04c9407d746887896343782aa6260db670bb12 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 19:27:58 +0100 Subject: [PATCH 023/357] add missed skill install --- .github/workflows/skill_test_installation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index f54abb2e8..63a4c5b11 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -62,7 +62,7 @@ jobs: else sudo apt install -y ${{inputs.system_deps}} fi - - name: Install Python Requirements + - name: Install Python Skill and Requirements run: | pip install --upgrade pip if [ -z "${{inputs.pip_packages}}" ]; then @@ -70,6 +70,8 @@ jobs: else pip install ${{inputs.pip_packages}} fi + cd ${{ github.workspace }}/action/skill + pip install . - name: Test Skill Installation run: | export TEST_SKILL_PATH=${{ github.workspace }}/action/skill/${{inputs.skill_location}} From 403a43bd96a1877798f59be827c00ed5847f3ac4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 27 Jan 2024 19:53:53 +0100 Subject: [PATCH 024/357] make action branch configurable --- .github/workflows/auto_translate.yml | 6 +++++- .github/workflows/license_tests.yml | 6 +++++- .github/workflows/propose_semver_release.yml | 6 +++++- .github/workflows/propose_translation.yml | 6 +++++- .github/workflows/publish_alpha_release.yml | 8 ++++++-- .github/workflows/skill_test_installation.yml | 6 +++++- .github/workflows/skill_test_intents.yml | 6 +++++- .github/workflows/skill_test_resources.yml | 6 +++++- .github/workflows/skill_tests.yml | 6 +++++- README.md | 2 +- 10 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index 2381c0f93..ecb450b8a 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -6,6 +6,10 @@ on: description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main python_version: description: 'The python version to use' type: string @@ -32,7 +36,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} path: action/github/ - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 32c8ac8d2..94320d6ab 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -10,6 +10,10 @@ on: description: 'Branch to check out' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main python-version: description: 'Python version to use' type: string @@ -44,7 +48,7 @@ jobs: uses: actions/checkout@v4 with: path: action/github/ - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} repository: OpenVoiceOS/.github - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 7479cbdc5..ed8fc501a 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -5,6 +5,10 @@ on: branch: type: string default: dev + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main python_version: description: 'The python version to use' type: string @@ -38,7 +42,7 @@ jobs: uses: actions/checkout@v4 with: path: action/github/ - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} repository: OpenVoiceOS/.github - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index c8c45f423..dc5deb902 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -6,6 +6,10 @@ on: description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref_name }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main python_version: description: 'The python version to use' type: string @@ -41,7 +45,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} path: action/github/ - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index 066cbb908..89e31b0a1 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -8,6 +8,10 @@ on: description: 'The branch to checkout and run the action in' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main version_file: description: 'The file location to read the version from' type: string @@ -40,7 +44,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} path: action/github/ - uses: dorny/paths-filter@v2 id: filter @@ -82,7 +86,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{ inputs.action_branch }} path: action/github/ - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 63a4c5b11..75df8886a 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -10,6 +10,10 @@ on: description: 'Branch to use' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main python_matrix: description: 'Python versions to test against' type: string @@ -48,7 +52,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{inputs.action_branch}} path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index addade8cd..049394a2a 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -9,6 +9,10 @@ on: description: 'Branch to test' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main timeout: type: number default: 15 @@ -60,7 +64,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{inputs.action_branch}} path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 85f5c45a8..33c0ce2c9 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -10,6 +10,10 @@ on: description: 'The branch to use' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false @@ -46,7 +50,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{inputs.action_branch}} path: action/github/ - name: Set up python uses: actions/setup-python@v5 diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml index ca06b731e..72c0f2110 100644 --- a/.github/workflows/skill_tests.yml +++ b/.github/workflows/skill_tests.yml @@ -9,6 +9,10 @@ on: description: 'Branch to use' type: string default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false @@ -43,7 +47,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: feat/shared_actions1 + ref: ${{inputs.action_branch}} path: action/github/ - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/README.md b/README.md index 2d255d2d0..f83ab468e 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ jobs: # this job will only run if the PR has been merged if: github.event.pull_request.merged == true secrets: inherit - uses: openvoiceos/.github/.github/workflows/notify_pr_matrix.yml@feat/shared_actions1 + uses: openvoiceos/.github/.github/workflows/notify_pr_matrix.yml@main with: pr_id: ${{ github.event.number }} ``` From 54fdc9724eb9bc398512cfddec1bef36e7000a05 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 28 Jan 2024 14:24:14 +0100 Subject: [PATCH 025/357] fix publish alpha wf --- .github/workflows/publish_alpha_release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index 89e31b0a1..a16e14829 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -52,9 +52,9 @@ jobs: working-directory: action/package/ filters: | locales: - - '${{ inputs.locale_folder }}/en-us/**' - - '${{ inputs.locale_folder }}/../dialog/en-us/**' - - '${{ inputs.locale_folder }}/../vocab/en-us/**' + - '${{ env.LOCALE_FOLDER }}/en-us/**' + - '${{ env.LOCALE_FOLDER }}/../dialog/en-us/**' + - '${{ env.LOCALE_FOLDER }}/../vocab/en-us/**' - name: Setup Python if: steps.filter.outputs.locales == 'true' uses: actions/setup-python@v5 @@ -107,7 +107,7 @@ jobs: - name: Push Version Change uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: Increment Version to ${{ env.version }} + commit_message: Increment Version to ${{ env.VERSION }} repository: action/package/ - name: Change working directory to release run: cd ${{ github.workspace }}/action/package/ @@ -116,11 +116,11 @@ jobs: uses: ncipollo/release-action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - tag: V${{ env.version }} - name: Release ${{ steps.version.outputs.version }} + tag: V${{ env.VERSION }} + name: Release ${{ env.VERSION }} body: | Changes in this Release - ${{ env.changelog }} + ${{ steps.changelog.outputs.changelog }} commit: ${{ inputs.branch }} prerelease: true - name: Install Build Tools From 8a0c8ce20957257937b9da769e37e51b44e3d82c Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 28 Jan 2024 14:38:27 +0100 Subject: [PATCH 026/357] revert locale filter path change --- .github/workflows/publish_alpha_release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index a16e14829..edfff1181 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -52,9 +52,9 @@ jobs: working-directory: action/package/ filters: | locales: - - '${{ env.LOCALE_FOLDER }}/en-us/**' - - '${{ env.LOCALE_FOLDER }}/../dialog/en-us/**' - - '${{ env.LOCALE_FOLDER }}/../vocab/en-us/**' + - '${{ inputs.locale_folder }}/en-us/**' + - '${{ inputs.locale_folder }}/../dialog/en-us/**' + - '${{ inputs.locale_folder }}/../vocab/en-us/**' - name: Setup Python if: steps.filter.outputs.locales == 'true' uses: actions/setup-python@v5 From 0d1b8f6919c70787e163e00e787e28ba9c8478a1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 28 Jan 2024 15:10:07 +0100 Subject: [PATCH 027/357] fix propose tx pkg path --- .github/workflows/propose_translation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index dc5deb902..2f0fcac3d 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -56,12 +56,12 @@ jobs: python -m pip install -r action/github/requirements/pip_translation.txt - name: Run Translate Script run: | - python action/github/scripts/translate.py - cd ${{ github.workspace }}/action/package/ + python ${{ github.workspace }}/action/github/scripts/translate.py - name: Create Pull Request uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} + path: action/package/ commit-message: autotranslate author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> title: Proposed ${{ inputs.language }} Translations From 9c59d54e5cf3fbd1b43d1f1c5e9df660d18d8ee6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 28 Jan 2024 15:29:07 +0100 Subject: [PATCH 028/357] adjust excluded pkgs/lics from license test --- .github/workflows/license_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 94320d6ab..b004b5707 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -29,11 +29,11 @@ on: packages-exclude: description: 'Regex to exclude packages from the license check' type: string - default: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct).*' + default: '^(precise-runner|fann2|tqdm|bs4|nvidia|bitstruct).*' licenses-exclude: description: 'Regex to exclude licenses from the license check' type: string - default: '^(Mozilla|NeonAI License v1.0).*$' + default: '^(Mozilla).*$' jobs: license_tests: timeout-minutes: 15 From 4814d49a9d063cae58d601aa470066ab72eb99ed Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 28 Jan 2024 15:36:10 +0100 Subject: [PATCH 029/357] lower target langs --- scripts/translate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/translate.py b/scripts/translate.py index f5cf7b56b..31c957302 100644 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -129,6 +129,7 @@ def migrate_locale(folder): src_files[rel_path] = file_path for lang in target_langs: + lang = lang.lower() # service cant translate if not tx.get_langcode(lang): continue From 779bf6af3c39a6105899fe4f4bf4319019ebe938 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 15:26:54 +0100 Subject: [PATCH 030/357] adjust semver wfs --- .github/workflows/propose_semver_release.yml | 22 +++--- .github/workflows/publish_semver_release.yml | 28 +++++--- README.md | 4 +- requirements/pip_base.txt | 2 + scripts/semver_cycle_start.py | 74 ++++++++++++++++++++ 5 files changed, 107 insertions(+), 23 deletions(-) create mode 100644 requirements/pip_base.txt create mode 100644 scripts/semver_cycle_start.py diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index ed8fc501a..508512d81 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -1,4 +1,4 @@ -name: Propose Stable Release +name: Propose SemVer Release on: workflow_call: inputs: @@ -31,7 +31,9 @@ jobs: runs-on: ubuntu-latest env: VERSION_FILE: ${{ inputs.version_file }} - TARGET_BRANCH: ${{ inputs.release_type == 'build' && 'testing' || 'stable' }} + TARGET_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} + GITHUB_REPOSITORY: ${{ github.repository }} + RELEASE_TYPE: ${{ inputs.release_type }} steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -48,6 +50,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} + - name: Install Python Dependencies + run: | + pip install -r action/github/requirements/pip_base.txt - name: Increment Version id: version run: | @@ -57,15 +62,12 @@ jobs: - name: Push Version Change uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: Increment Version to ${{ env.version }} + commit_message: Increment Version to ${{ env.VERSION }} repository: action/package/ - name: Check Latest Release - id: latest_release - continue-on-error: true - uses: pozetroninc/github-action-get-latest-release@master - with: - repository: ${{ github.repository }} - excludes: draft,prerelease + run: | + SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Generate Changelog id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 @@ -73,7 +75,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.VERSION }} - sinceTag: ${{ steps.latest_release.outputs.release }} + sinceTag: ${{ env.SINCE_TAG }} - name: Push Changelog uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index eda1b84eb..7bd9f348b 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -1,10 +1,14 @@ -name: Publish Stable Build +name: Publish SemVer Build on: workflow_call: secrets: PYPI_TOKEN: required: true inputs: + release_type: + type: string + description: 'The type of release to make' + required: true python_version: description: "Python version to use for the build process." type: string @@ -17,7 +21,9 @@ on: jobs: build_and_publish: env: - SOURCE_BRANCH: ${{ inputs.release_type == 'build' && 'testing' || 'stable' }} + SOURCE_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} + GITHUB_REPOSITORY: ${{ github.repository }} + RELEASE_TYPE: ${{ inputs.release_type }} runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -29,31 +35,31 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{inputs.python_version}} + - name: Install Python Dependencies + run: | + pip install -r action/github/requirements/pip_base.txt - name: Get Version run: | VERSION=$(python setup.py --version) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release - id: latest_release - continue-on-error: true - uses: pozetroninc/github-action-get-latest-release@master - with: - repository: ${{ github.repository }} - excludes: draft,prerelease + run: | + SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Generate Changelog id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GITHUB_TOKEN }} output: action/package/${{ inputs.changelog_file }} - sinceTag: ${{ steps.latest_release.outputs.release }} + sinceTag: ${{ env.SINCE_TAG }} - name: Create Release id: create_release uses: ncipollo/release-action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - tag: V${{ env.version }} - name: Release ${{ env.version }} + tag: V${{ env.VERSION }} + name: Release ${{ env.VERSION }} body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} diff --git a/README.md b/README.md index f83ab468e..44750605c 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ on: release_type: type: choice options: - - "build" + - "patch" - "minor" - "major" @@ -66,7 +66,7 @@ on: release_type: type: choice options: - - "build" + - "patch" - "minor" - "major" diff --git a/requirements/pip_base.txt b/requirements/pip_base.txt new file mode 100644 index 000000000..9bbbfc51b --- /dev/null +++ b/requirements/pip_base.txt @@ -0,0 +1,2 @@ +PyGithub +semver diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py new file mode 100644 index 000000000..c88c26155 --- /dev/null +++ b/scripts/semver_cycle_start.py @@ -0,0 +1,74 @@ +from os import getenv +import re + +from github import Github +import semver + +""" +Get the start of the release cycle for the current release type. (patch, minor, major) +(the release cycle starts with the end version of the last cylce) +If `LAST_RELEASE` is set, the last release will be returned. +If `FIRST_RELEASE` is set, the first release will be returned. +""" + +GITHUB_REPOSITORY = getenv("GITHUB_REPOSITORY") +RELEASE_TYPE = getenv("RELEASE_TYPE") +FIRST_RELEASE = bool(getenv("FIRST_RELEASE")) +LAST_RELEASE = bool(getenv("LAST_RELEASE")) + +if any(req is None for req in [GITHUB_REPOSITORY, RELEASE_TYPE]): + raise ValueError("Missing required environment variable(s)") + +repo = Github(getenv("GITHUB_TOKEN")).get_repo(GITHUB_REPOSITORY) +latest_version = None +start_cycle_id = 0 + +def get_semver(tag: str) -> semver.Version: + # hack for v prefix + tag = tag.lstrip("v").lstrip("V") + + # hack for alpha releases + if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): + tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) + + if not semver.Version.is_valid(tag): + return None + return semver.Version.parse(tag) + + +def in_cycle(v: semver.Version) -> bool: + if RELEASE_TYPE == "patch": + return v.patch == latest_version.patch and not \ + (v.minor < latest_version.minor) and not \ + (v.major < latest_version.major) + elif RELEASE_TYPE == "minor": + return v.minor == latest_version.minor and not \ + (v.major < latest_version.major) + elif RELEASE_TYPE == "major": + return v.major == latest_version.major + + +releases = repo.get_releases() +if not releases: + raise Exception("No releases found") + +for id, release in enumerate(releases): + version = get_semver(release.tag_name) + if id == 0: + latest_version = version + if LAST_RELEASE: + break + continue + + if not version: + continue + elif in_cycle(version): + start_cycle_id = id + +if latest_version is None: + exit(0) +elif start_cycle_id < releases.totalCount - 1 and not \ + (FIRST_RELEASE or LAST_RELEASE): + start_cycle_id += 1 + +print(releases[start_cycle_id].tag_name) From ef5bd3bcf2d46d5e7194b103c608f00e48e84eba Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 15:37:40 +0100 Subject: [PATCH 031/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 2 +- scripts/{bump_build.py => bump_patch.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{bump_build.py => bump_patch.py} (100%) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 508512d81..8953aa931 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -56,7 +56,7 @@ jobs: - name: Increment Version id: version run: | - python action/github/scripts/bump_"${{ inputs.release_type }}".py + python action/github/scripts/bump_${{ inputs.release_type }}.py VERSION=$(python action/package/setup.py --version) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Push Version Change diff --git a/scripts/bump_build.py b/scripts/bump_patch.py similarity index 100% rename from scripts/bump_build.py rename to scripts/bump_patch.py From 3639159544fade97eb30f53a4185ec851db0a840 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 15:43:52 +0100 Subject: [PATCH 032/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 8953aa931..e6802c883 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -30,7 +30,7 @@ jobs: bump_version: runs-on: ubuntu-latest env: - VERSION_FILE: ${{ inputs.version_file }} + VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} TARGET_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} GITHUB_REPOSITORY: ${{ github.repository }} RELEASE_TYPE: ${{ inputs.release_type }} From 74eb738141cbbeff4bc16f9f2423ed7f585af1b0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 15:49:01 +0100 Subject: [PATCH 033/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index e6802c883..381db87b4 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -85,6 +85,7 @@ jobs: uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} + path: action/package/ commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" title: "Propose ${{ env.TARGET_BRANCH }} Release" body: | From a69b637c27c55405d0a09b89876980f9cceaf2ae Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 15:54:46 +0100 Subject: [PATCH 034/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 381db87b4..b5ea4d15f 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -92,7 +92,6 @@ jobs: This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. branch: ${{ env.TARGET_BRANCH }} - branch-suffix: ${{ env.TARGET_BRANCH }} base: ${{ inputs.branch }} draft: false assignees: ${{ github.actor }} From afbaa4615628ff9a087fcd381ad0ba1bfddad815 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 16:44:13 +0100 Subject: [PATCH 035/357] adjust target branch handling --- .github/workflows/propose_semver_release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index b5ea4d15f..4e481e0da 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -81,6 +81,14 @@ jobs: with: commit_message: Update Changelog repository: action/package/ + - name: Check and Create Target Branch + run: | + TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) + git checkout -b ${{ env.TARGET_BRANCH }} $FIRST_COMMIT + git push origin ${{ env.TARGET_BRANCH }} + fi - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From 3ac4ffdf0d7b8bdac759c3070b52d6b82967f5a8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 17:42:49 +0100 Subject: [PATCH 036/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 4e481e0da..e54537141 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -85,6 +85,7 @@ jobs: run: | TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) git checkout -b ${{ env.TARGET_BRANCH }} $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} From c7418c17934fb79f862713bad306ea0eadf10eca Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 17:46:06 +0100 Subject: [PATCH 037/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index e54537141..5661e93d2 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -81,11 +81,13 @@ jobs: with: commit_message: Update Changelog repository: action/package/ + - name: Set up Git Authentication + run: | + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - name: Check and Create Target Branch run: | TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) git checkout -b ${{ env.TARGET_BRANCH }} $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} From 4a30f081dbbd2390d232437faf01f0497dbfd28f Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 17:51:55 +0100 Subject: [PATCH 038/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 5661e93d2..52c964c64 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -76,22 +76,24 @@ jobs: output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.VERSION }} sinceTag: ${{ env.SINCE_TAG }} - - name: Push Changelog - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update Changelog - repository: action/package/ - name: Set up Git Authentication run: | + cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - name: Check and Create Target Branch run: | + cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) git checkout -b ${{ env.TARGET_BRANCH }} $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi + - name: Push Changelog + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update Changelog + repository: action/package/ - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From fef8443af8113f274330a8c09c495994ba4f2860 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 18:21:13 +0100 Subject: [PATCH 039/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 52c964c64..dae247a0c 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -86,7 +86,8 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) - git checkout -b ${{ env.TARGET_BRANCH }} $FIRST_COMMIT + git checkout $FIRST_COMMIT + git checkout -b ${{ env.TARGET_BRANCH }} git push origin ${{ env.TARGET_BRANCH }} fi - name: Push Changelog From 37f509ff1cdf59256fb401a03aed82afbd186c10 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 19:08:02 +0100 Subject: [PATCH 040/357] adjust semver wf --- .github/workflows/propose_semver_release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index dae247a0c..174a9d8d2 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -85,9 +85,8 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) - git checkout $FIRST_COMMIT - git checkout -b ${{ env.TARGET_BRANCH }} + FIRST_COMMIT=$(git rev-list --reverse HEAD | head -n 1) + git branch ${{ env.TARGET_BRANCH }} $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi - name: Push Changelog From f67f5cfdc8f634b9d7ceac42760ef91169e48a1d Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 19:34:36 +0100 Subject: [PATCH 041/357] test --- .github/workflows/propose_semver_release.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 174a9d8d2..976254dfb 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -89,23 +89,5 @@ jobs: git branch ${{ env.TARGET_BRANCH }} $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi - - name: Push Changelog - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update Changelog - repository: action/package/ - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - path: action/package/ - commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" - title: "Propose ${{ env.TARGET_BRANCH }} Release" - body: | - This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. - Please review the changelog and merge if it looks good. - branch: ${{ env.TARGET_BRANCH }} - base: ${{ inputs.branch }} - draft: false - assignees: ${{ github.actor }} + \ No newline at end of file From 937e0375f73d2d627108c8be8200792bb5de7f3b Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 20:00:30 +0100 Subject: [PATCH 042/357] test --- .github/workflows/propose_semver_release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 976254dfb..f5288aa10 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -86,8 +86,7 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --reverse HEAD | head -n 1) - git branch ${{ env.TARGET_BRANCH }} $FIRST_COMMIT + git branch ${{ env.TARGET_BRANCH }} "$FIRST_COMMIT" git push origin ${{ env.TARGET_BRANCH }} fi - \ No newline at end of file From cbf255e119ffabc6545bc6991b0869b1849f2ddf Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 20:24:25 +0100 Subject: [PATCH 043/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index f5288aa10..701567ac1 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -86,7 +86,7 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --reverse HEAD | head -n 1) - git branch ${{ env.TARGET_BRANCH }} "$FIRST_COMMIT" + git checkout -b ${{ env.TARGET_BRANCH }} "$FIRST_COMMIT" git push origin ${{ env.TARGET_BRANCH }} fi \ No newline at end of file From affbf9cc9f3721e107bb99f88c2d7dfb07e246c4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 21:11:21 +0100 Subject: [PATCH 044/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 701567ac1..733e7c86b 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -87,6 +87,6 @@ jobs: if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --reverse HEAD | head -n 1) git checkout -b ${{ env.TARGET_BRANCH }} "$FIRST_COMMIT" - git push origin ${{ env.TARGET_BRANCH }} + git push -v -u origin ${{ env.TARGET_BRANCH }} fi \ No newline at end of file From 8f68d651b4602f9ac5d45434619b768b629aec8a Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 21:21:32 +0100 Subject: [PATCH 045/357] test --- .github/workflows/propose_semver_release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 733e7c86b..ac2d76829 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -85,8 +85,10 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - FIRST_COMMIT=$(git rev-list --reverse HEAD | head -n 1) - git checkout -b ${{ env.TARGET_BRANCH }} "$FIRST_COMMIT" - git push -v -u origin ${{ env.TARGET_BRANCH }} + FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) + git checkout --orphan ${{ env.TARGET_BRANCH }} + git reset + git cherry-pick $FIRST_COMMIT + git push origin ${{ env.TARGET_BRANCH }} fi \ No newline at end of file From 9ec7aa1416ebb2cbd9596ec6c34f766a926cf525 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 21:32:54 +0100 Subject: [PATCH 046/357] test --- .github/workflows/propose_semver_release.yml | 49 ++++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index ac2d76829..79cf142f0 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -53,6 +53,22 @@ jobs: - name: Install Python Dependencies run: | pip install -r action/github/requirements/pip_base.txt + - name: Set up Git Authentication + run: | + cd ${{ github.workspace }}/action/package/ + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} + - name: Check and Create Target Branch + run: | + cd ${{ github.workspace }}/action/package/ + TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) + git checkout --orphan ${{ env.TARGET_BRANCH }} + git reset + git clean -fdx + git cherry-pick $FIRST_COMMIT + git push origin ${{ env.TARGET_BRANCH }} + fi - name: Increment Version id: version run: | @@ -76,19 +92,22 @@ jobs: output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.VERSION }} sinceTag: ${{ env.SINCE_TAG }} - - name: Set up Git Authentication - run: | - cd ${{ github.workspace }}/action/package/ - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - - name: Check and Create Target Branch - run: | - cd ${{ github.workspace }}/action/package/ - TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) - git checkout --orphan ${{ env.TARGET_BRANCH }} - git reset - git cherry-pick $FIRST_COMMIT - git push origin ${{ env.TARGET_BRANCH }} - fi + - name: Push Changelog + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update Changelog + repository: action/package/ + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: action/package/ + commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" + title: "Propose ${{ env.TARGET_BRANCH }} Release" + body: | + This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. + Please review the changelog and merge if it looks good. + branch: ${{ env.TARGET_BRANCH }} + draft: false + assignees: ${{ github.actor }} \ No newline at end of file From 2d614d2b36a0db98d4cd0727dec29349b4c371be Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 21:43:54 +0100 Subject: [PATCH 047/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 79cf142f0..6bca8f5fb 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -65,7 +65,7 @@ jobs: FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) git checkout --orphan ${{ env.TARGET_BRANCH }} git reset - git clean -fdx + git clean -fdx -e .git/ git cherry-pick $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi From 913ae29293e258b96bbe5819de2a1f6c7994f1c5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 21:49:38 +0100 Subject: [PATCH 048/357] test --- .github/workflows/propose_semver_release.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 6bca8f5fb..3defc77d1 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -63,9 +63,13 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) + GIT_USER_NAME=$(git config --get user.name) + GIT_USER_EMAIL=$(git config --get user.email) git checkout --orphan ${{ env.TARGET_BRANCH }} git reset - git clean -fdx -e .git/ + git clean -fdx + git config user.email "$GIT_USER_EMAIL" + git config user.name "$GIT_USER_NAME" git cherry-pick $FIRST_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi From 6cc0f5fb68cc8335a30ef110cd97345dfae5b08a Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 22:21:10 +0100 Subject: [PATCH 049/357] test --- .github/workflows/propose_semver_release.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 3defc77d1..777c673a0 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,16 +62,10 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + git checkout -b ${{ env.TARGET_BRANCH }} FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) - GIT_USER_NAME=$(git config --get user.name) - GIT_USER_EMAIL=$(git config --get user.email) - git checkout --orphan ${{ env.TARGET_BRANCH }} - git reset - git clean -fdx - git config user.email "$GIT_USER_EMAIL" - git config user.name "$GIT_USER_NAME" - git cherry-pick $FIRST_COMMIT - git push origin ${{ env.TARGET_BRANCH }} + git reset --hard $FIRST_COMMIT + git push -u origin ${{ env.TARGET_BRANCH }} fi - name: Increment Version id: version From 38f8901ca31d9429f48d8b04a8e8e62251b27ee9 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 22:30:55 +0100 Subject: [PATCH 050/357] test --- .github/workflows/propose_semver_release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 777c673a0..dc893ecf8 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -65,7 +65,8 @@ jobs: git checkout -b ${{ env.TARGET_BRANCH }} FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) git reset --hard $FIRST_COMMIT - git push -u origin ${{ env.TARGET_BRANCH }} + git push -u origin ${{ env.TARGET_BRANCH }} --force + git checkout ${{ inputs.branch }} fi - name: Increment Version id: version From 75ddd5b824610f634209dc8b91d560a236936a33 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 29 Jan 2024 23:54:48 +0100 Subject: [PATCH 051/357] test --- .github/workflows/propose_semver_release.yml | 41 -------------------- 1 file changed, 41 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index dc893ecf8..f62985d06 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -68,45 +68,4 @@ jobs: git push -u origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi - - name: Increment Version - id: version - run: | - python action/github/scripts/bump_${{ inputs.release_type }}.py - VERSION=$(python action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Increment Version to ${{ env.VERSION }} - repository: action/package/ - - name: Check Latest Release - run: | - SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: Generate Changelog - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.VERSION }} - sinceTag: ${{ env.SINCE_TAG }} - - name: Push Changelog - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update Changelog - repository: action/package/ - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - path: action/package/ - commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" - title: "Propose ${{ env.TARGET_BRANCH }} Release" - body: | - This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. - Please review the changelog and merge if it looks good. - branch: ${{ env.TARGET_BRANCH }} - draft: false - assignees: ${{ github.actor }} \ No newline at end of file From 00ffc3e155316195c7e5816f280933976c67dce4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:16:27 +0100 Subject: [PATCH 052/357] test --- .github/workflows/propose_semver_release.yml | 50 ++++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index f62985d06..818793933 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,10 +62,50 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git checkout -b ${{ env.TARGET_BRANCH }} - FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) - git reset --hard $FIRST_COMMIT - git push -u origin ${{ env.TARGET_BRANCH }} --force - git checkout ${{ inputs.branch }} + git checkout --orphan ${{ env.TARGET_BRANCH }} + git rm --cached -r . + git clean -fd + git push origin ${{ env.TARGET_BRANCH }} --force fi + - name: Increment Version + id: version + run: | + python action/github/scripts/bump_${{ inputs.release_type }}.py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version to ${{ env.VERSION }} + repository: action/package/ + - name: Check Latest Release + run: | + SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: Generate Changelog + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + output: action/package/${{ inputs.changelog_file }} + futureRelease: ${{ env.VERSION }} + sinceTag: ${{ env.SINCE_TAG }} + - name: Push Changelog + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update Changelog + repository: action/package/ + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: action/package/ + commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" + title: "Propose ${{ env.TARGET_BRANCH }} Release" + body: | + This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. + Please review the changelog and merge if it looks good. + branch: ${{ env.TARGET_BRANCH }} + draft: false + assignees: ${{ github.actor }} \ No newline at end of file From 29dd94f19ccb2350134fe04910d8490efd862905 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:23:14 +0100 Subject: [PATCH 053/357] test --- .github/workflows/propose_semver_release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 818793933..2197c85bb 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -63,8 +63,7 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} - git rm --cached -r . - git clean -fd + git commit --allow-empty -m "Initial empty commit" git push origin ${{ env.TARGET_BRANCH }} --force fi - name: Increment Version From e97073ccf8c42d90a2a28c9a806572c43dd2eb60 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:30:25 +0100 Subject: [PATCH 054/357] test --- .github/workflows/propose_semver_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 2197c85bb..46b72539f 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -63,6 +63,8 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "GitHub Actions Bot" git commit --allow-empty -m "Initial empty commit" git push origin ${{ env.TARGET_BRANCH }} --force fi From 68d841227cf6c1199d620af8210126b66b868d92 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:32:26 +0100 Subject: [PATCH 055/357] test --- .github/workflows/propose_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 46b72539f..1f483216b 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -67,6 +67,7 @@ jobs: git config user.name "GitHub Actions Bot" git commit --allow-empty -m "Initial empty commit" git push origin ${{ env.TARGET_BRANCH }} --force + git checkout ${{ inputs.branch }} fi - name: Increment Version id: version From 613ea986a24048af599e96fbdb7d3b85f93459ba Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:37:08 +0100 Subject: [PATCH 056/357] test --- .github/workflows/propose_semver_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 1f483216b..5a52fef8e 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -63,6 +63,8 @@ jobs: TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} + git rm --cached -r . + git clean -fd git config user.email "github-actions[bot]@users.noreply.github.com" git config user.name "GitHub Actions Bot" git commit --allow-empty -m "Initial empty commit" From 6117a53d271a9c2fc00aa0a43802ee5a6160e842 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:43:32 +0100 Subject: [PATCH 057/357] test --- .github/workflows/propose_semver_release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 5a52fef8e..cc9dc0a94 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,12 +62,9 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git checkout --orphan ${{ env.TARGET_BRANCH }} - git rm --cached -r . - git clean -fd - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "GitHub Actions Bot" - git commit --allow-empty -m "Initial empty commit" + git checkout -b ${{ env.TARGET_BRANCH }} + EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) + git reset --hard $EMPTY_COMMIT git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi From c11903db71c1917830b47bacea738f7fa867f0bf Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:46:55 +0100 Subject: [PATCH 058/357] test --- .github/workflows/propose_semver_release.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index cc9dc0a94..b4d5929a6 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,10 +62,12 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git checkout -b ${{ env.TARGET_BRANCH }} - EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) - git reset --hard $EMPTY_COMMIT - git push origin ${{ env.TARGET_BRANCH }} --force + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "GitHub Actions Bot" + git checkout -b ${{ env.TARGET_BRANCH }} + EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) + git reset --hard $EMPTY_COMMIT + git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi - name: Increment Version From 25855bae8281d1001039a8d0c01e6f660243f4f1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:50:33 +0100 Subject: [PATCH 059/357] test --- .github/workflows/propose_semver_release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index b4d5929a6..3a842f44f 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -61,13 +61,13 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "GitHub Actions Bot" - git checkout -b ${{ env.TARGET_BRANCH }} - EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) - git reset --hard $EMPTY_COMMIT - git push origin ${{ env.TARGET_BRANCH }} --force + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "GitHub Actions Bot" + git checkout -b ${{ env.TARGET_BRANCH }} + EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) + git reset --hard $EMPTY_COMMIT + git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi - name: Increment Version From bdabd2100a2300cd92c5c65416474da4a073516e Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:59:00 +0100 Subject: [PATCH 060/357] test --- .github/workflows/propose_semver_release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 3a842f44f..43fa29ce9 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,13 +62,12 @@ jobs: cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + git update-ref refs/heads/${{ env.TARGET_BRANCH }} $(git hash-object -t tree /dev/null) + git checkout ${{ env.TARGET_BRANCH }} git config user.email "github-actions[bot]@users.noreply.github.com" git config user.name "GitHub Actions Bot" - git checkout -b ${{ env.TARGET_BRANCH }} - EMPTY_COMMIT=$(git commit-tree -m "Initial empty commit" 4b825dc642cb6eb9a060e54bf8d69288fbee4904) - git reset --hard $EMPTY_COMMIT + git commit --allow-empty -m "Initial empty commit" git push origin ${{ env.TARGET_BRANCH }} --force - git checkout ${{ inputs.branch }} fi - name: Increment Version id: version From 30aea2d6d38132925e9819a4d0ffe9fe39ff9001 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 00:59:17 +0100 Subject: [PATCH 061/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 43fa29ce9..dff9953c2 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -61,7 +61,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git update-ref refs/heads/${{ env.TARGET_BRANCH }} $(git hash-object -t tree /dev/null) git checkout ${{ env.TARGET_BRANCH }} git config user.email "github-actions[bot]@users.noreply.github.com" From 49ef06bd04c635dbf27195ead5b513adda8f8636 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:06:03 +0100 Subject: [PATCH 062/357] test --- .github/workflows/propose_semver_release.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index dff9953c2..684bcbffa 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -55,19 +55,18 @@ jobs: pip install -r action/github/requirements/pip_base.txt - name: Set up Git Authentication run: | - cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - - name: Check and Create Target Branch + - name: Create Empty Branch run: | - cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git update-ref refs/heads/${{ env.TARGET_BRANCH }} $(git hash-object -t tree /dev/null) - git checkout ${{ env.TARGET_BRANCH }} + git checkout --orphan ${{ env.TARGET_BRANCH }} + git rm --cached -r . + git clean -fd git config user.email "github-actions[bot]@users.noreply.github.com" git config user.name "GitHub Actions Bot" git commit --allow-empty -m "Initial empty commit" - git push origin ${{ env.TARGET_BRANCH }} --force + git push origin ${{ env.TARGET_BRANCH }} fi - name: Increment Version id: version From 6a8b5f4a5b259b71a71f2f48f18579afcbdb41d2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:09:21 +0100 Subject: [PATCH 063/357] test --- .github/workflows/propose_semver_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 684bcbffa..6d12b1d95 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -55,9 +55,11 @@ jobs: pip install -r action/github/requirements/pip_base.txt - name: Set up Git Authentication run: | + cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - name: Create Empty Branch run: | + cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} From 781f7cde41314c2fe8a1dcbb4a9c4078c8d95977 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:30:13 +0100 Subject: [PATCH 064/357] test --- .github/workflows/propose_semver_release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 6d12b1d95..5ee1c775a 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -60,14 +60,13 @@ jobs: - name: Create Empty Branch run: | cd ${{ github.workspace }}/action/package/ + INITIAL_COMMIT=$(git rev-list --max-parents=0 HEAD) TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} git rm --cached -r . git clean -fd - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "GitHub Actions Bot" - git commit --allow-empty -m "Initial empty commit" + git cherry-pick $INITIAL_COMMIT git push origin ${{ env.TARGET_BRANCH }} fi - name: Increment Version From eac8de6e103b37145044e6dbce77c475277a71ac Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:31:24 +0100 Subject: [PATCH 065/357] test --- .github/workflows/propose_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 5ee1c775a..ea70b4a60 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -68,6 +68,7 @@ jobs: git clean -fd git cherry-pick $INITIAL_COMMIT git push origin ${{ env.TARGET_BRANCH }} + git checkout ${{ inputs.branch }} fi - name: Increment Version id: version From 5e3e2c89bb83175b4e77c5fd9f4c9484a53d11c2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:36:50 +0100 Subject: [PATCH 066/357] test --- .github/workflows/propose_semver_release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index ea70b4a60..ff94e4c98 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -61,11 +61,15 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ INITIAL_COMMIT=$(git rev-list --max-parents=0 HEAD) + AUTHOR_NAME=$(git show -s --format='%an' $INITIAL_COMMIT) + AUTHOR_EMAIL=$(git show -s --format='%ae' $INITIAL_COMMIT) TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then git checkout --orphan ${{ env.TARGET_BRANCH }} git rm --cached -r . git clean -fd + git config user.name "$AUTHOR_NAME" + git config user.email "$AUTHOR_EMAIL" git cherry-pick $INITIAL_COMMIT git push origin ${{ env.TARGET_BRANCH }} git checkout ${{ inputs.branch }} From 08628d1cf323a8832b01998c13b2b9d10616b0b7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 01:39:41 +0100 Subject: [PATCH 067/357] test --- .github/workflows/propose_semver_release.yml | 43 +------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index ff94e4c98..3047b00b3 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -71,48 +71,7 @@ jobs: git config user.name "$AUTHOR_NAME" git config user.email "$AUTHOR_EMAIL" git cherry-pick $INITIAL_COMMIT - git push origin ${{ env.TARGET_BRANCH }} + git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi - - name: Increment Version - id: version - run: | - python action/github/scripts/bump_${{ inputs.release_type }}.py - VERSION=$(python action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Increment Version to ${{ env.VERSION }} - repository: action/package/ - - name: Check Latest Release - run: | - SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: Generate Changelog - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.VERSION }} - sinceTag: ${{ env.SINCE_TAG }} - - name: Push Changelog - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update Changelog - repository: action/package/ - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - path: action/package/ - commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" - title: "Propose ${{ env.TARGET_BRANCH }} Release" - body: | - This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. - Please review the changelog and merge if it looks good. - branch: ${{ env.TARGET_BRANCH }} - draft: false - assignees: ${{ github.actor }} \ No newline at end of file From 9698a56029ecf2f318aaf811656dcece6870aa4d Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 02:07:38 +0100 Subject: [PATCH 068/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 3047b00b3..7a45966be 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -71,7 +71,7 @@ jobs: git config user.name "$AUTHOR_NAME" git config user.email "$AUTHOR_EMAIL" git cherry-pick $INITIAL_COMMIT - git push origin ${{ env.TARGET_BRANCH }} --force + git push -u origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi \ No newline at end of file From 6edb7efc58bf5935b87770c74e53ad14fe2bdbaa Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 02:33:39 +0100 Subject: [PATCH 069/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 7a45966be..b20950e17 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -60,7 +60,7 @@ jobs: - name: Create Empty Branch run: | cd ${{ github.workspace }}/action/package/ - INITIAL_COMMIT=$(git rev-list --max-parents=0 HEAD) + INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) AUTHOR_NAME=$(git show -s --format='%an' $INITIAL_COMMIT) AUTHOR_EMAIL=$(git show -s --format='%ae' $INITIAL_COMMIT) TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) From e6c344cb6ea4e4cef471ff40fe839b4f6cd1a470 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 02:38:39 +0100 Subject: [PATCH 070/357] test --- .github/workflows/propose_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index b20950e17..14f909893 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -60,6 +60,7 @@ jobs: - name: Create Empty Branch run: | cd ${{ github.workspace }}/action/package/ + git fetch origin ${{ inputs.branch }} INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) AUTHOR_NAME=$(git show -s --format='%an' $INITIAL_COMMIT) AUTHOR_EMAIL=$(git show -s --format='%ae' $INITIAL_COMMIT) From 14a4577a62ffbf87ae3539f6ea8a6349d9d29e96 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 02:47:27 +0100 Subject: [PATCH 071/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 14f909893..e53a96a8f 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -60,7 +60,7 @@ jobs: - name: Create Empty Branch run: | cd ${{ github.workspace }}/action/package/ - git fetch origin ${{ inputs.branch }} + git fetch --unshallow origin ${{ inputs.branch }} INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) AUTHOR_NAME=$(git show -s --format='%an' $INITIAL_COMMIT) AUTHOR_EMAIL=$(git show -s --format='%ae' $INITIAL_COMMIT) From 3c59beb8372d252ca231bf510399def87cb94e62 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 02:59:33 +0100 Subject: [PATCH 072/357] test --- .github/workflows/propose_semver_release.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index e53a96a8f..ad877e0ee 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -62,17 +62,11 @@ jobs: cd ${{ github.workspace }}/action/package/ git fetch --unshallow origin ${{ inputs.branch }} INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) - AUTHOR_NAME=$(git show -s --format='%an' $INITIAL_COMMIT) - AUTHOR_EMAIL=$(git show -s --format='%ae' $INITIAL_COMMIT) TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git checkout --orphan ${{ env.TARGET_BRANCH }} - git rm --cached -r . - git clean -fd - git config user.name "$AUTHOR_NAME" - git config user.email "$AUTHOR_EMAIL" - git cherry-pick $INITIAL_COMMIT - git push -u origin ${{ env.TARGET_BRANCH }} --force + git checkout $INITIAL_COMMIT + git checkout -b ${{ env.TARGET_BRANCH }} + git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi \ No newline at end of file From a6cd84a63d727c01328361eeb4710a6cb16e1d39 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 03:17:16 +0100 Subject: [PATCH 073/357] test --- .github/workflows/propose_semver_release.yml | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index ad877e0ee..b500bbe3e 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -69,4 +69,45 @@ jobs: git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi + - name: Increment Version + id: version + run: | + python action/github/scripts/bump_${{ inputs.release_type }}.py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version to ${{ env.VERSION }} + repository: action/package/ + - name: Check Latest Release + run: | + SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: Generate Changelog + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + output: action/package/${{ inputs.changelog_file }} + futureRelease: ${{ env.VERSION }} + sinceTag: ${{ env.SINCE_TAG }} + - name: Push Changelog + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update Changelog + repository: action/package/ + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: action/package/ + commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" + title: "Propose ${{ env.TARGET_BRANCH }} Release" + body: | + This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. + Please review the changelog and merge if it looks good. + branch: ${{ env.TARGET_BRANCH }} + draft: false + assignees: ${{ github.actor }} \ No newline at end of file From 6e569ada08097c5d84131177b1e9d8cdf0114bb6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 03:49:23 +0100 Subject: [PATCH 074/357] test --- .github/workflows/propose_semver_release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index b500bbe3e..aafa98278 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -57,7 +57,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - - name: Create Empty Branch + - name: Create Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ git fetch --unshallow origin ${{ inputs.branch }} @@ -107,7 +107,8 @@ jobs: body: | This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. - branch: ${{ env.TARGET_BRANCH }} + branch: staging_${{ env.TARGET_BRANCH }} + base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} \ No newline at end of file From 473e30788a151e356029dd051f9e09fb89de8a56 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 04:49:07 +0100 Subject: [PATCH 075/357] test --- .github/workflows/propose_semver_release.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index aafa98278..458a81f1c 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -97,6 +97,11 @@ jobs: with: commit_message: Update Changelog repository: action/package/ + - name: Checkout testing Branch + uses: actions/checkout@v4 + with: + ref: ${{ env.TARGET_BRANCH }} + path: action/package/ - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: @@ -108,7 +113,7 @@ jobs: This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. branch: staging_${{ env.TARGET_BRANCH }} - base: ${{ env.TARGET_BRANCH }} + base: ${{ inputs.branch }} draft: false assignees: ${{ github.actor }} \ No newline at end of file From 2b0754e3358847a36e65e82b17f7315461bc3207 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 05:23:56 +0100 Subject: [PATCH 076/357] test --- .github/workflows/propose_semver_release.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 458a81f1c..eea4fa7de 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -97,11 +97,6 @@ jobs: with: commit_message: Update Changelog repository: action/package/ - - name: Checkout testing Branch - uses: actions/checkout@v4 - with: - ref: ${{ env.TARGET_BRANCH }} - path: action/package/ - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: @@ -112,8 +107,8 @@ jobs: body: | This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. - branch: staging_${{ env.TARGET_BRANCH }} - base: ${{ inputs.branch }} + branch: ${{ inputs.branch }} + base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} \ No newline at end of file From 7affd06e46e4cc37aad5c01a943ed996c3911130 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 05:39:40 +0100 Subject: [PATCH 077/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index eea4fa7de..aafa98278 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -107,7 +107,7 @@ jobs: body: | This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. - branch: ${{ inputs.branch }} + branch: staging_${{ env.TARGET_BRANCH }} base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} From c91c3014cf7b4b99b479e24511bce9fe92b6c939 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 13:19:27 +0100 Subject: [PATCH 078/357] test --- .github/workflows/propose_semver_release.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index aafa98278..0c0d389a5 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -97,6 +97,15 @@ jobs: with: commit_message: Update Changelog repository: action/package/ + - name: Checkout Testing Branch + uses: actions/checkout@v4 + with: + ref: ${{ env.TARGET_BRANCH }} + path: action/package/ + - name: Pull Latest Changes + run: | + cd ${{ github.workspace }}/action/package/ + git pull origin ${{ inputs.branch }} - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: @@ -111,4 +120,3 @@ jobs: base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} - \ No newline at end of file From f600230757da06b478ebc85060d4d60e2a799241 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:09:27 +0100 Subject: [PATCH 079/357] test --- .github/workflows/propose_semver_release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 0c0d389a5..f23edb396 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -105,7 +105,8 @@ jobs: - name: Pull Latest Changes run: | cd ${{ github.workspace }}/action/package/ - git pull origin ${{ inputs.branch }} + git fetch origin ${{ inputs.branch }} + git merge --strategy-option ours FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From a9f3b04cd3cd92aeb4db90ed446e63ae1879328c Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:17:15 +0100 Subject: [PATCH 080/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index f23edb396..aff29d061 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -106,7 +106,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git fetch origin ${{ inputs.branch }} - git merge --strategy-option ours FETCH_HEAD + git merge --allow-unrelated-histories --strategy-option ours FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From 0de791b851906e65f22942315782928ff47d39f2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:28:14 +0100 Subject: [PATCH 081/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index aff29d061..fadf99f11 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -106,7 +106,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git fetch origin ${{ inputs.branch }} - git merge --allow-unrelated-histories --strategy-option ours FETCH_HEAD + git reset --hard FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From e319a0cd226f903080d4350f510da595e8fe9a95 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:29:51 +0100 Subject: [PATCH 082/357] test --- .github/workflows/propose_semver_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index fadf99f11..f3b5cbaf4 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -105,6 +105,8 @@ jobs: - name: Pull Latest Changes run: | cd ${{ github.workspace }}/action/package/ + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com git fetch origin ${{ inputs.branch }} git reset --hard FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} From 246811efa27a5f31f22ef3a0a17288c615d9114f Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:32:28 +0100 Subject: [PATCH 083/357] test --- .github/workflows/propose_semver_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index f3b5cbaf4..8e59486fc 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -106,7 +106,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com + git config user.email "github-actions[bot]@users.noreply.github.com" git fetch origin ${{ inputs.branch }} git reset --hard FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} From 4668248a9d3bfac55691e361649f589489e943dd Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 14:46:08 +0100 Subject: [PATCH 084/357] test --- .github/workflows/propose_semver_release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 8e59486fc..45f16561c 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -102,13 +102,13 @@ jobs: with: ref: ${{ env.TARGET_BRANCH }} path: action/package/ - - name: Pull Latest Changes + - name: Rebase Latest Changes run: | cd ${{ github.workspace }}/action/package/ git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git fetch origin ${{ inputs.branch }} - git reset --hard FETCH_HEAD + git rebase FETCH_HEAD - name: Create Pull Request to ${{ env.TARGET_BRANCH }} uses: peter-evans/create-pull-request@v5 with: From e1f7c5f3938ec52cc534ab1fb467a130cbbbf2e8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 17:00:48 +0100 Subject: [PATCH 085/357] test --- .github/workflows/notify_pr_matrix.yml | 4 +-- .github/workflows/propose_semver_release.yml | 6 ++-- .../propose_semver_release_merged.yml | 35 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/propose_semver_release_merged.yml diff --git a/.github/workflows/notify_pr_matrix.yml b/.github/workflows/notify_pr_matrix.yml index 578b1db00..69278403c 100644 --- a/.github/workflows/notify_pr_matrix.yml +++ b/.github/workflows/notify_pr_matrix.yml @@ -11,8 +11,6 @@ on: jobs: notify_pr_matrix: runs-on: ubuntu-latest - env: - PACKAGE: "$(basename ${{ github.repository }})" steps: - uses: actions/checkout@v4 - name: Send message to Matrix bots channel @@ -23,4 +21,4 @@ jobs: token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' message: | - new ${{ env.PACKAGE }} PR merged! https://github.com/${{ github.repository }}/pull/${{ inputs.pr_id }} \ No newline at end of file + new ${{ github.repository }} PR merged! https://github.com/${{ github.repository }}/pull/${{ inputs.pr_id }} \ No newline at end of file diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 45f16561c..5494323bd 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -114,12 +114,12 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} path: action/package/ - commit-message: "Propose ${{ env.TARGET_BRANCH }} Release" - title: "Propose ${{ env.TARGET_BRANCH }} Release" + commit-message: "Propose ${{ inputs.release_type }} release" + title: "Propose ${{ inputs.release_type }} release" body: | This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. Please review the changelog and merge if it looks good. - branch: staging_${{ env.TARGET_BRANCH }} + branch: staging_${{ inputs.release_type }}_release base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} diff --git a/.github/workflows/propose_semver_release_merged.yml b/.github/workflows/propose_semver_release_merged.yml new file mode 100644 index 000000000..a760feb91 --- /dev/null +++ b/.github/workflows/propose_semver_release_merged.yml @@ -0,0 +1,35 @@ +name: Rebase dev after SemVer bump + +# only triggers on pull request closed events +on: + workflow_call: + +jobs: + rebase_dev: + runs-on: ubuntu-latest + steps: + - name: Set branch name + id: branch + run: | + BRANCH_NAME=$(echo "${{ github.event.pull_request.head.commit.message }}" | grep -o -E 'patch|minor|major') + if [ "$BRANCH_NAME" == "patch" ]; then + echo "BRANCH=testing" >> $GITHUB_ENV + else + echo "BRANCH=stable" >> $GITHUB_ENV + fi + - name: Checkout + uses: actions/checkout@v4 + with: + ref: "dev" + fetch-depth: 0 + - name: Set up Git Authentication + run: | + cd ${{ github.workspace }}/action/package/ + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} + - name: Rebase testing onto dev + run: | + git fetch origin ${{ env.BRANCH }} + git rebase origin/${{ env.BRANCH }} + - name: Push changes + run: | + git push origin dev \ No newline at end of file From 7c5e1b7e089099bfbd59da8c5f4f756ac35d9510 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 17:22:14 +0100 Subject: [PATCH 086/357] test --- .github/workflows/propose_semver_release_merged.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose_semver_release_merged.yml b/.github/workflows/propose_semver_release_merged.yml index a760feb91..56622b3f5 100644 --- a/.github/workflows/propose_semver_release_merged.yml +++ b/.github/workflows/propose_semver_release_merged.yml @@ -11,7 +11,7 @@ jobs: - name: Set branch name id: branch run: | - BRANCH_NAME=$(echo "${{ github.event.pull_request.head.commit.message }}" | grep -o -E 'patch|minor|major') + BRANCH_NAME=$(echo "${{ github.event.pull_request.title }}" | grep -o -E 'patch|minor|major') if [ "$BRANCH_NAME" == "patch" ]; then echo "BRANCH=testing" >> $GITHUB_ENV else From 5d57be85e113d4080b3584873072e7995f852491 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 17:25:57 +0100 Subject: [PATCH 087/357] test --- .github/workflows/propose_semver_release_merged.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/propose_semver_release_merged.yml b/.github/workflows/propose_semver_release_merged.yml index 56622b3f5..90dcedd42 100644 --- a/.github/workflows/propose_semver_release_merged.yml +++ b/.github/workflows/propose_semver_release_merged.yml @@ -24,7 +24,6 @@ jobs: fetch-depth: 0 - name: Set up Git Authentication run: | - cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - name: Rebase testing onto dev run: | From 44139a5576b5484dbb158e69af0a4a5f482bfbb5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 17:57:20 +0100 Subject: [PATCH 088/357] test --- .github/workflows/propose_semver_release.yml | 2 +- .github/workflows/publish_semver_release.yml | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml index 5494323bd..0ddf2288e 100644 --- a/.github/workflows/propose_semver_release.yml +++ b/.github/workflows/propose_semver_release.yml @@ -52,7 +52,7 @@ jobs: python-version: ${{ inputs.python_version }} - name: Install Python Dependencies run: | - pip install -r action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - name: Set up Git Authentication run: | cd ${{ github.workspace }}/action/package/ diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index 7bd9f348b..14fd86c14 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -5,6 +5,10 @@ on: PYPI_TOKEN: required: true inputs: + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main release_type: type: string description: 'The type of release to make' @@ -30,21 +34,28 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.SOURCE_BRANCH }} - fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + fetch-depth: 0 + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{inputs.python_version}} - name: Install Python Dependencies run: | - pip install -r action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - name: Get Version run: | - VERSION=$(python setup.py --version) + VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release run: | - SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Generate Changelog id: changelog From 6095ec711dc8dfa77ff762796621c768986cac47 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 18:44:55 +0100 Subject: [PATCH 089/357] test --- .../propose_semver_release_merged.yml | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/propose_semver_release_merged.yml b/.github/workflows/propose_semver_release_merged.yml index 90dcedd42..c18582797 100644 --- a/.github/workflows/propose_semver_release_merged.yml +++ b/.github/workflows/propose_semver_release_merged.yml @@ -3,20 +3,18 @@ name: Rebase dev after SemVer bump # only triggers on pull request closed events on: workflow_call: + inputs: + release_type: + type: string + description: 'The type of release to make' + required: true jobs: rebase_dev: + env: + SOURCE_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} runs-on: ubuntu-latest steps: - - name: Set branch name - id: branch - run: | - BRANCH_NAME=$(echo "${{ github.event.pull_request.title }}" | grep -o -E 'patch|minor|major') - if [ "$BRANCH_NAME" == "patch" ]; then - echo "BRANCH=testing" >> $GITHUB_ENV - else - echo "BRANCH=stable" >> $GITHUB_ENV - fi - name: Checkout uses: actions/checkout@v4 with: @@ -27,8 +25,8 @@ jobs: git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - name: Rebase testing onto dev run: | - git fetch origin ${{ env.BRANCH }} - git rebase origin/${{ env.BRANCH }} + git fetch origin ${{ env.SOURCE_BRANCH }} + git rebase origin/${{ env.SOURCE_BRANCH }} - name: Push changes run: | git push origin dev \ No newline at end of file From a021c5d81a94e4e06ded1efaaca3e79ede1bf50d Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 18:56:13 +0100 Subject: [PATCH 090/357] test --- .github/workflows/publish_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index 14fd86c14..31c31015f 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -80,6 +80,7 @@ jobs: python -m pip install build wheel - name: Build Distribution Packages run: | + cd ${{ github.workspace }}/action/package python setup.py sdist bdist_wheel - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From 389905755f58ba72aaecf3bc0314a4bf0f878c48 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 30 Jan 2024 19:06:03 +0100 Subject: [PATCH 091/357] test --- .github/workflows/publish_semver_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/publish_semver_release.yml index 31c31015f..51218e38c 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/publish_semver_release.yml @@ -86,3 +86,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{secrets.PYPI_TOKEN}} + packages-dir: action/package/dist/ From a59e15eba67a0043c549d3472ad11be481110364 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 31 Jan 2024 21:30:19 +0100 Subject: [PATCH 092/357] codecov ready unittests --- .github/workflows/pytest_file_or_dir.yml | 153 +++++++++++++++++++++++ .github/workflows/skill_tests.yml | 77 ------------ README.md | 53 +++++--- 3 files changed, 192 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/pytest_file_or_dir.yml delete mode 100644 .github/workflows/skill_tests.yml diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml new file mode 100644 index 000000000..3f8225fe3 --- /dev/null +++ b/.github/workflows/pytest_file_or_dir.yml @@ -0,0 +1,153 @@ +name: Unit Tests +on: + workflow_call: + inputs: + runner: + type: string + default: "ubuntu-latest" + branch: + description: 'Branch to use' + type: string + default: ${{ github.ref }} + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main + system_deps: + description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + required: false + type: string + python_matrix: + description: 'Python matrix (string) to use' + type: string + default: '["3.8", "3.9", "3.10", "3.11"]' + pip_packages: + description: 'Additional Python packages (whitespace delimited) to install' + type: string + default: '' + install_extras: + description: 'comma delimited extras to install the package with (eg: "test,dev")' + type: string + required: false + timout_minutes: + description: 'Timeout in minutes for the job' + type: number + default: 15 + package_location: + description: 'name of the package directory relative to the base folder' + type: string + test_location: + description: 'test file or directory to run relative to the base folder' + type: string + default: test/unittests + is_skill: + description: 'Whether this is an ovos skill or not' + type: boolean + default: false + codecov: + description: 'Whether to record the test code coverage' + type: boolean + default: true + append_coverage: + description: 'Whether to append coverage to codecov, used for consecutive runs' + type: boolean + default: false + upload_coverage: + description: 'Whether to upload coverage to codecov' + type: boolean + default: true + +jobs: + unit_tests: + strategy: + matrix: + python-version: ${{ fromJSON(inputs.python_matrix) }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timout_minutes }} + steps: + - name: Checkout Package Repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install System Dependencies + run: | + sudo apt update + if [ -z "${{ inputs.system_deps }}" ]; then + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + else + sudo apt install -y ${{inputs.system_deps}} + fi + - name: upgrade pip + run: | + pip install --upgrade pip + - name: Install Skill Requirements + if: ${{ inputs.is_skill }} == 'true' + run: | + if [ -z "${{ inputs.pip_packages }}" ]; then + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt + fi + pip install wheel "cython<3.0.0" # TODO: cython patching + - name: Install Additional Python Requirements + if: ${{ inputs.pip_packages }} != '' + run: | + pip install ${{ inputs.pip_packages }} + - name: Install Package + run: | + cd ${{ github.workspace }}/action/package + if [ -n "${{ inputs.install_extras }}" ]; then + pip install -e .[${{ inputs.install_extras }}] + else + pip install -e . + fi + - name: download test coverage artifact + if: ${{ inputs.codecov }} == 'true' && ${{ inputs.append_coverage }} == 'true' + uses: actions/download-artifact@v4 + with: + name: coverage_${{ github.repository_id }} + path: ${{ github.workspace }}/action/package/ + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Test package with ${{ matrix.python-version }} + run: | + FIRST_PYTHON_VERSION=${{ fromJSON(inputs.python_matrix)[0] }} + cd ${{ github.workspace }}/action/package/ + if [ "${{ inputs.codecov }}" = "true" ] && [ "${{ matrix.python-version }}" = "$FIRST_PYTHON_VERSION" ]; then + echo "Running tests with codecov" + if [ "${{ inputs.append_coverage }}" = "true" ]; then + pytest --cov-append --cov=${{ inputs.package_location}} --cov-report=xml ${{ inputs.test_location }} + else + pytest --cov=${{ inputs.package_location}} --cov-report=xml ${{ inputs.test_location }} + fi + else + echo "Running tests without codecov" + pytest ${{ inputs.test_location }} + fi + - name: upload test coverage artifact + if: ${{ inputs.codecov }} == 'true' && ${{ inputs.upload_coverage }} == 'false' + uses: actions/upload-artifact@v4 + with: + name: coverage_${{ github.repository_id }} + overwrite: true + retention-days: 1 + if-no-files-found: ignore + path: ${{ github.workspace }}/action/package/coverage.xml + - name: Upload coverage + if: ${{ inputs.codecov }} == 'true' && ${{ inputs.upload_coverage }} == 'true' + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + files: ${{ github.workspace }}/action/package/coverage.xml,!${{ github.workspace }}/cache + flags: unittests + name: ovos + verbose: true \ No newline at end of file diff --git a/.github/workflows/skill_tests.yml b/.github/workflows/skill_tests.yml deleted file mode 100644 index 72c0f2110..000000000 --- a/.github/workflows/skill_tests.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Skill Unit Tests -on: - workflow_call: - inputs: - runner: - type: string - default: "ubuntu-latest" - branch: - description: 'Branch to use' - type: string - default: ${{ github.ref }} - action_branch: - description: 'The shared action branch to checkout' - type: string - default: main - system_deps: - description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' - required: false - type: string - python_matrix: - description: 'Python matrix (string) to use' - type: string - default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: - description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - required: false - type: string - skill_testfile: - description: 'Skill test file to run relative to the base folder' - type: string - default: test/unittests/test_skill.py - -jobs: - skill_unit_tests: - strategy: - matrix: - python-version: ${{ fromJSON(inputs.python_matrix) }} - runs-on: ${{inputs.runner}} - timeout-minutes: 15 - steps: - - name: Checkout Skill Repo - uses: actions/checkout@v4 - with: - ref: ${{inputs.branch}} - path: action/skill/ - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{inputs.action_branch}} - path: action/github/ - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install System Dependencies - run: | - sudo apt update - if [ -z "${{inputs.system_deps}}" ]; then - xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt install -y ${{inputs.system_deps}} - fi - - name: Install Python Skill and Requirements - run: | - pip install --upgrade pip - pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 - if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - else - pip install ${{inputs.pip_packages}} - fi - cd ${{ github.workspace }}/action/skill - pip install . - - name: Test Skill - run: | - pytest ${{ github.workspace }}/action/skill/${{inputs.skill_testfile}} \ No newline at end of file diff --git a/README.md b/README.md index 44750605c..7b4d54ad6 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ jobs: uses: openvoiceos/.github/.github/workflows/python_build_tests.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' test_manifest: true # if to test with MANIFEST.in, default: false manifest_ignored: "test/** qt5/**" # Files to ignore in MANIFEST.in, default: "test/**" test_relative_paths: false # if to test with relative paths, default: true @@ -152,26 +152,52 @@ jobs: pipaudit_ignored: "" # Vulnerabilities to ignore in pip-audit, # default: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012" ``` -## Skills -## Skill Unit Tests +## Unit Tests (file or directory) ```yaml name: Skill Unit Tests on: jobs: - skill_tests: - uses: openvoiceos/.github/.github/workflows/skill_tests.yml@main + unit_tests: + uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main with: runner: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action + action_branch: custom/branch # Shared action branch to use, default: main system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' + pip_packages: "pytest pytest-cov" # Optional Python packages (whitespace delimited) to install + timeout_minutes: 15 # Timeout in minutes for the job, default: 15 + package_location: ovos_core # Package location relative to the root (in this case: package "ovos-core") + # (not needed if codevoc isn't used) + install_extras: lgpl,mycroft # Optional comma-separated extras to install the python package with + test_location: test/unittests # Test file (or directory) to run, default: test/unittests + is_skill: true # Whether this is an ovos skill, default: false + codecov: true # Whether to record the test code coverage, default: true + upload_coverage: true # Whether to upload the coverage to codecov server, default: false + # should upload only if there are no following jobs that need coverage + + # showcase with multiple jobs that should add to the coverage test + next_test_that_needs_coverage: + needs: unit_tests + uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main + with: + ... + append_coverage: true # Whether to append coverage to the previous job, default: false + # the artifact will be downloaded, appended and uploaded again + ... + and_another_test_that_needs_coverage: + needs: [ unit_tests, next_test_that_needs_coverage ] + uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main + with: + ... + append_coverage: true + upload_coverage: true # Whether to upload the coverage to codecov server + ... ``` +## Skills ## Skill Installation Tests ```yaml name: Skill Installation Tests @@ -184,15 +210,14 @@ jobs: with: runner: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action + action_branch: custom/branch # Shared action branch to use, default: main system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_testfile: test/unittest/test_skl.py # Skill test file to run, default: test/unittest/test_skill.py skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required - test_osm: true # if to test with osm, default: true - test_loader: true # if to test with skillloader, default: true + skill_location: "skill" # Skill location relative to the root (can usually be omitted) ``` ## Skill Resource Tests Tests the resources of a skill. This may be dialogs, vocabs, regex or intent resources @@ -231,7 +256,7 @@ jobs: timeout: 15 # Timeout for the test, default: 15 system-deps: "libfann-dev libfann2" # Custom system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11", "3.12"]' + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml From c20c11a246daf4abef91eeb9227a015b606da93d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 31 Jan 2024 22:41:41 +0100 Subject: [PATCH 093/357] codecov adjustments --- .github/workflows/pytest_file_or_dir.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 3f8225fe3..221d7a8eb 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -80,6 +80,16 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: set codecov run flag + id: codecov + run: | + if [ "${{ matrix.python-version }}" = "$FIRST_PYTHON_VERSION" ] && [ "${{ inputs.codecov }}" = "true" ]; then + echo "should_run=true" >> $env:GITHUB_OUTPUT + else + echo "should_run=false" >> $env:GITHUB_OUTPUT + fi + env: + FIRST_PYTHON_VERSION: ${{ fromJSON(inputs.python_matrix)[0] }} - name: Install System Dependencies run: | sudo apt update @@ -111,7 +121,7 @@ jobs: pip install -e . fi - name: download test coverage artifact - if: ${{ inputs.codecov }} == 'true' && ${{ inputs.append_coverage }} == 'true' + if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.append_coverage }} == 'true' uses: actions/download-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -119,9 +129,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Test package with ${{ matrix.python-version }} run: | - FIRST_PYTHON_VERSION=${{ fromJSON(inputs.python_matrix)[0] }} cd ${{ github.workspace }}/action/package/ - if [ "${{ inputs.codecov }}" = "true" ] && [ "${{ matrix.python-version }}" = "$FIRST_PYTHON_VERSION" ]; then + if [ ${{ steps.codecov.outputs.should_run }} == 'true' ]; then echo "Running tests with codecov" if [ "${{ inputs.append_coverage }}" = "true" ]; then pytest --cov-append --cov=${{ inputs.package_location}} --cov-report=xml ${{ inputs.test_location }} @@ -133,7 +142,7 @@ jobs: pytest ${{ inputs.test_location }} fi - name: upload test coverage artifact - if: ${{ inputs.codecov }} == 'true' && ${{ inputs.upload_coverage }} == 'false' + if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.upload_coverage }} == 'false' uses: actions/upload-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -142,7 +151,7 @@ jobs: if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml - name: Upload coverage - if: ${{ inputs.codecov }} == 'true' && ${{ inputs.upload_coverage }} == 'true' + if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.upload_coverage }} == 'true' uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} From 2433633dd28827d57e399b26214d401263619409 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 15:27:25 +0100 Subject: [PATCH 094/357] add supported py versions check --- .../workflows/check_supported_py_versions.yml | 25 ++++++++++++++ scripts/check_supported_py_versions.py | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/workflows/check_supported_py_versions.yml create mode 100644 scripts/check_supported_py_versions.py diff --git a/.github/workflows/check_supported_py_versions.yml b/.github/workflows/check_supported_py_versions.yml new file mode 100644 index 000000000..5f955e8f3 --- /dev/null +++ b/.github/workflows/check_supported_py_versions.yml @@ -0,0 +1,25 @@ +name: Check Supported Python Versions + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + update-versions: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + - name: Run Python Version Check Script + run: + python scripts/check_supported_py_versions.py + - name: Commit and Push if Changes + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add supported_py_versions.json + git commit -m "Update Supported Python Versions" || exit 0 # Exit gracefully if no changes + git push \ No newline at end of file diff --git a/scripts/check_supported_py_versions.py b/scripts/check_supported_py_versions.py new file mode 100644 index 000000000..769d6b8f8 --- /dev/null +++ b/scripts/check_supported_py_versions.py @@ -0,0 +1,33 @@ +import json +from datetime import datetime + +# Manually define the non-EOL Python versions and their EOL dates +# Update this as new releases come out or old versions reach EOL +python_versions = { + "3.8": "2024-10-01", + "3.9": "2025-10-01", + "3.10": "2026-10-01", + "3.11": "2027-10-01", + #"3.12": "2028-10-01" +} + +# Get the current date +current_date = datetime.now().date() + +# Filter out versions that are past their EOL +non_eol_versions = {version: eol_date for version, eol_date in python_versions.items() + if datetime.strptime(eol_date, "%Y-%m-%d").date() > current_date} + +# Prepare the data for the JSON file +data = { + "schemaVersion": 1, + "label": "python", + "message": ", ".join(non_eol_versions.keys()), + "color": "blue" +} + +# Write the data to a JSON file +with open("supported_py_versions.json", "w") as file: + json.dump(data, file) + +print("Updated python-versions.json with non-EOL Python versions.") \ No newline at end of file From d80cf4f0c8e618423f8e9ea1a5835c04b7ee967e Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 15:59:35 +0100 Subject: [PATCH 095/357] test --- .github/workflows/pytest_file_or_dir.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 221d7a8eb..bbe06a7e0 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -24,7 +24,7 @@ on: pip_packages: description: 'Additional Python packages (whitespace delimited) to install' type: string - default: '' + required: false install_extras: description: 'comma delimited extras to install the package with (eg: "test,dev")' type: string @@ -64,6 +64,8 @@ jobs: python-version: ${{ fromJSON(inputs.python_matrix) }} runs-on: ${{ inputs.runner }} timeout-minutes: ${{ inputs.timout_minutes }} + env: + FIRST_PYTHON_VERSION: ${{ fromJSON(inputs.python_matrix)[0] }} steps: - name: Checkout Package Repo uses: actions/checkout@v4 @@ -88,8 +90,6 @@ jobs: else echo "should_run=false" >> $env:GITHUB_OUTPUT fi - env: - FIRST_PYTHON_VERSION: ${{ fromJSON(inputs.python_matrix)[0] }} - name: Install System Dependencies run: | sudo apt update @@ -109,7 +109,7 @@ jobs: fi pip install wheel "cython<3.0.0" # TODO: cython patching - name: Install Additional Python Requirements - if: ${{ inputs.pip_packages }} != '' + if: ${{ inputs.pip_packages != null }} run: | pip install ${{ inputs.pip_packages }} - name: Install Package From c8be5f0fd46626ea1296bd482c88fc8bea0881ba Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 16:42:22 +0100 Subject: [PATCH 096/357] test --- .github/workflows/pytest_file_or_dir.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index bbe06a7e0..a90fa527e 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -86,9 +86,11 @@ jobs: id: codecov run: | if [ "${{ matrix.python-version }}" = "$FIRST_PYTHON_VERSION" ] && [ "${{ inputs.codecov }}" = "true" ]; then - echo "should_run=true" >> $env:GITHUB_OUTPUT + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "codecov will run" else - echo "should_run=false" >> $env:GITHUB_OUTPUT + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "codecov will not run" fi - name: Install System Dependencies run: | @@ -121,7 +123,7 @@ jobs: pip install -e . fi - name: download test coverage artifact - if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.append_coverage }} == 'true' + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.append_coverage == 'true' }} uses: actions/download-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -142,7 +144,7 @@ jobs: pytest ${{ inputs.test_location }} fi - name: upload test coverage artifact - if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.upload_coverage }} == 'false' + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage == 'false' }} uses: actions/upload-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -151,7 +153,7 @@ jobs: if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml - name: Upload coverage - if: ${{ steps.codecov.outputs.should_run }} == 'true' && ${{ inputs.upload_coverage }} == 'true' + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage == 'true' }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} From 390910c29bc6f56f5d9e4a8dc25be78a231ca0e0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 18:36:22 +0100 Subject: [PATCH 097/357] test --- .github/workflows/pytest_file_or_dir.yml | 21 ++++++++++----------- README.md | 2 -- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index a90fa527e..ed06a6312 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -29,13 +29,10 @@ on: description: 'comma delimited extras to install the package with (eg: "test,dev")' type: string required: false - timout_minutes: + timeout_minutes: description: 'Timeout in minutes for the job' type: number default: 15 - package_location: - description: 'name of the package directory relative to the base folder' - type: string test_location: description: 'test file or directory to run relative to the base folder' type: string @@ -63,7 +60,7 @@ jobs: matrix: python-version: ${{ fromJSON(inputs.python_matrix) }} runs-on: ${{ inputs.runner }} - timeout-minutes: ${{ inputs.timout_minutes }} + timeout-minutes: ${{ inputs.timeout_minutes }} env: FIRST_PYTHON_VERSION: ${{ fromJSON(inputs.python_matrix)[0] }} steps: @@ -104,7 +101,7 @@ jobs: run: | pip install --upgrade pip - name: Install Skill Requirements - if: ${{ inputs.is_skill }} == 'true' + if: ${{ inputs.is_skill }} run: | if [ -z "${{ inputs.pip_packages }}" ]; then pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt @@ -117,13 +114,15 @@ jobs: - name: Install Package run: | cd ${{ github.workspace }}/action/package + PACKAGE_NAME=$(python setup.py --name) if [ -n "${{ inputs.install_extras }}" ]; then pip install -e .[${{ inputs.install_extras }}] else pip install -e . fi + echo "PACKAGE_NAME=$PACKAGE_NAME" >> "$GITHUB_ENV" - name: download test coverage artifact - if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.append_coverage == 'true' }} + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.append_coverage }} uses: actions/download-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -135,16 +134,16 @@ jobs: if [ ${{ steps.codecov.outputs.should_run }} == 'true' ]; then echo "Running tests with codecov" if [ "${{ inputs.append_coverage }}" = "true" ]; then - pytest --cov-append --cov=${{ inputs.package_location}} --cov-report=xml ${{ inputs.test_location }} + pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else - pytest --cov=${{ inputs.package_location}} --cov-report=xml ${{ inputs.test_location }} + pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} fi else echo "Running tests without codecov" pytest ${{ inputs.test_location }} fi - name: upload test coverage artifact - if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage == 'false' }} + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: actions/upload-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -153,7 +152,7 @@ jobs: if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml - name: Upload coverage - if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage == 'true' }} + if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index 7b4d54ad6..1a9a8eb1f 100644 --- a/README.md +++ b/README.md @@ -170,8 +170,6 @@ jobs: python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Optional Python packages (whitespace delimited) to install timeout_minutes: 15 # Timeout in minutes for the job, default: 15 - package_location: ovos_core # Package location relative to the root (in this case: package "ovos-core") - # (not needed if codevoc isn't used) install_extras: lgpl,mycroft # Optional comma-separated extras to install the python package with test_location: test/unittests # Test file (or directory) to run, default: test/unittests is_skill: true # Whether this is an ovos skill, default: false From 36b474fef56215b4106a0b5ac2af693de41a9937 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 19:45:11 +0100 Subject: [PATCH 098/357] test --- .github/workflows/pytest_file_or_dir.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index ed06a6312..3318d84e0 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -156,8 +156,9 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false files: ${{ github.workspace }}/action/package/coverage.xml,!${{ github.workspace }}/cache flags: unittests name: ovos - verbose: true \ No newline at end of file + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 6150c6fa2e0dcd045c66dddd72373a9aca4b9cf5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 20:23:20 +0100 Subject: [PATCH 099/357] test --- .github/workflows/pytest_file_or_dir.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 3318d84e0..ff20f27d9 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -143,7 +143,7 @@ jobs: pytest ${{ inputs.test_location }} fi - name: upload test coverage artifact - if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} + if: ${{ steps.codecov.outputs.should_run == 'true' && !inputs.upload_coverage }} uses: actions/upload-artifact@v4 with: name: coverage_${{ github.repository_id }} @@ -160,5 +160,3 @@ jobs: flags: unittests name: ovos verbose: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 2f93505b4565d708c654815324177b90ebd54ae8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 21:03:22 +0100 Subject: [PATCH 100/357] test --- .github/workflows/pytest_file_or_dir.yml | 15 ++++++++------- README.md | 4 ++-- requirements/pip_skill_tests.txt | 5 ++++- requirements/pip_tests.txt | 4 +++- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index ff20f27d9..925076ce8 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -100,14 +100,15 @@ jobs: - name: upgrade pip run: | pip install --upgrade pip - - name: Install Skill Requirements - if: ${{ inputs.is_skill }} + - name: Install Base Skill Requirements + if: ${{ inputs.is_skill && !inputs.pip_packages }} run: | - if [ -z "${{ inputs.pip_packages }}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - fi - pip install wheel "cython<3.0.0" # TODO: cython patching - - name: Install Additional Python Requirements + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt + - name: Install Base Requirements + if: ${{ !inputs.is_skill && !inputs.pip_packages }} + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_tests.txt + - name: Install Custom Python Requirements if: ${{ inputs.pip_packages != null }} run: | pip install ${{ inputs.pip_packages }} diff --git a/README.md b/README.md index 1a9a8eb1f..27d2c31d8 100644 --- a/README.md +++ b/README.md @@ -168,9 +168,9 @@ jobs: system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Optional Python packages (whitespace delimited) to install + pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of the commonly used timeout_minutes: 15 # Timeout in minutes for the job, default: 15 - install_extras: lgpl,mycroft # Optional comma-separated extras to install the python package with + install_extras: lgpl,mycroft # Comma-separated extras to install the python package with test_location: test/unittests # Test file (or directory) to run, default: test/unittests is_skill: true # Whether this is an ovos skill, default: false codecov: true # Whether to record the test code coverage, default: true diff --git a/requirements/pip_skill_tests.txt b/requirements/pip_skill_tests.txt index 0ea9f0319..eab8c7cdc 100644 --- a/requirements/pip_skill_tests.txt +++ b/requirements/pip_skill_tests.txt @@ -1,9 +1,12 @@ +coveralls pytest pytest-cov mock +wheel +cython ovos-core>=0.0.8a66 ovos-phal-plugin-connectivity-events~=0.0.1 padatious>=0.4.8, < 0.5.0 fann2>=1.0.7, < 1.1.0 -padaos>=0.1, < 0.2 \ No newline at end of file +padaos>=0.1, < 0.2 diff --git a/requirements/pip_tests.txt b/requirements/pip_tests.txt index 2546e64f5..660c5e277 100644 --- a/requirements/pip_tests.txt +++ b/requirements/pip_tests.txt @@ -1,4 +1,6 @@ +coveralls pytest pytest-cov mock -ovos-skills-manager +wheel +cython From cbf6c6d7b5cc89562127086f3b9e43302cf19d95 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:21:59 +0100 Subject: [PATCH 101/357] test --- .github/workflows/pytest_file_or_dir.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 925076ce8..729f70fc3 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -157,7 +157,8 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ github.workspace }}/action/package/coverage.xml,!${{ github.workspace }}/cache + files: coverage.xml + working-directory: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From f304955403f0895c2aa2c4b6623b5dc34523eb34 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:29:15 +0100 Subject: [PATCH 102/357] test --- .github/workflows/pytest_file_or_dir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 729f70fc3..30226fa79 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -157,7 +157,7 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml + files: ${{ github.workspace }}/action/package/coverage.xml working-directory: ${{ github.workspace }}/action/package flags: unittests name: ovos From e1e031a41db97acbe926f46c7e1d897ccdd4d60e Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:34:56 +0100 Subject: [PATCH 103/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 30226fa79..985228a4e 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -158,7 +158,6 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ github.workspace }}/action/package/coverage.xml - working-directory: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From eae7a1b782e7dc74b06378c6df8004f8c40e30aa Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:39:59 +0100 Subject: [PATCH 104/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 985228a4e..152a40c91 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -158,6 +158,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ github.workspace }}/action/package/coverage.xml + root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From eff1395662fe367a891e0d398240595bffa42135 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:50:46 +0100 Subject: [PATCH 105/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 152a40c91..8e5a947a6 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -139,6 +139,7 @@ jobs: else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} fi + cat coverage.xml else echo "Running tests without codecov" pytest ${{ inputs.test_location }} From ccdaf151acccf5d734deeb604e4da9883ad60050 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 22:58:53 +0100 Subject: [PATCH 106/357] test --- .github/workflows/pytest_file_or_dir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 8e5a947a6..ac8c8f18e 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -138,8 +138,8 @@ jobs: pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} + cat coverage.xml fi - cat coverage.xml else echo "Running tests without codecov" pytest ${{ inputs.test_location }} From cfe7106daf9d44afd075a1d53cbf98867e96c2d2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:01:40 +0100 Subject: [PATCH 107/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index ac8c8f18e..642fc8ec1 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -159,7 +159,6 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ github.workspace }}/action/package/coverage.xml - root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From 1ab51584c4698c113a5ed2aaea9761a165a7016c Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:03:57 +0100 Subject: [PATCH 108/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 642fc8ec1..985228a4e 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -138,7 +138,6 @@ jobs: pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} - cat coverage.xml fi else echo "Running tests without codecov" From edbeb4ea6f2240670a394550e00328814ba5eaf6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:10:35 +0100 Subject: [PATCH 109/357] test --- .github/workflows/pytest_file_or_dir.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 985228a4e..88cff2fb7 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -143,6 +143,11 @@ jobs: echo "Running tests without codecov" pytest ${{ inputs.test_location }} fi + - name: Display coverage report + if: ${{ steps.codecov.outputs.should_run == 'true' }} + run: | + cd ${{ github.workspace }}/action/package/ + coverage report - name: upload test coverage artifact if: ${{ steps.codecov.outputs.should_run == 'true' && !inputs.upload_coverage }} uses: actions/upload-artifact@v4 From a5b8eecbcf355f41391ed9fc671e8c94168be835 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:20:47 +0100 Subject: [PATCH 110/357] test --- .github/workflows/pytest_file_or_dir.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 88cff2fb7..d7e8c0338 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -143,11 +143,6 @@ jobs: echo "Running tests without codecov" pytest ${{ inputs.test_location }} fi - - name: Display coverage report - if: ${{ steps.codecov.outputs.should_run == 'true' }} - run: | - cd ${{ github.workspace }}/action/package/ - coverage report - name: upload test coverage artifact if: ${{ steps.codecov.outputs.should_run == 'true' && !inputs.upload_coverage }} uses: actions/upload-artifact@v4 @@ -157,6 +152,11 @@ jobs: retention-days: 1 if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml + - name: Display coverage report + if: ${{ steps.codecov.outputs.should_run == 'true' }} + run: | + cd ${{ github.workspace }}/action/package/ + coverage report - name: Upload coverage if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: codecov/codecov-action@v4 From e2b40e5936c96ebe0156fbb42e7d29053a18bad5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:30:32 +0100 Subject: [PATCH 111/357] test --- .github/workflows/pytest_file_or_dir.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index d7e8c0338..4e2d11319 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -143,6 +143,7 @@ jobs: echo "Running tests without codecov" pytest ${{ inputs.test_location }} fi + coverage report - name: upload test coverage artifact if: ${{ steps.codecov.outputs.should_run == 'true' && !inputs.upload_coverage }} uses: actions/upload-artifact@v4 @@ -152,11 +153,6 @@ jobs: retention-days: 1 if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml - - name: Display coverage report - if: ${{ steps.codecov.outputs.should_run == 'true' }} - run: | - cd ${{ github.workspace }}/action/package/ - coverage report - name: Upload coverage if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: codecov/codecov-action@v4 From 4b858d0c1a5678ca02d49fb0163e7cc8682e33ec Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:33:27 +0100 Subject: [PATCH 112/357] test --- .github/workflows/pytest_file_or_dir.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 4e2d11319..65661fb56 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -69,6 +69,7 @@ jobs: with: ref: ${{ inputs.branch }} path: action/package/ + fetch-depth: 0 - name: Checkout Scripts Repo uses: actions/checkout@v4 with: @@ -138,12 +139,12 @@ jobs: pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} + coverage report fi else echo "Running tests without codecov" pytest ${{ inputs.test_location }} fi - coverage report - name: upload test coverage artifact if: ${{ steps.codecov.outputs.should_run == 'true' && !inputs.upload_coverage }} uses: actions/upload-artifact@v4 From e335091874d12838505732fc046d68183e47d166 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 1 Feb 2024 23:37:13 +0100 Subject: [PATCH 113/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 65661fb56..1f5ab47b5 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -139,7 +139,6 @@ jobs: pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} - coverage report fi else echo "Running tests without codecov" From afbc4601b2341e90c0e24b34971e227b41b0b41d Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 00:23:47 +0100 Subject: [PATCH 114/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 1f5ab47b5..54921aa5f 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -159,6 +159,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ github.workspace }}/action/package/coverage.xml + root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From 4426abd52116939f6a23ff3e252971ed622b3659 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 00:27:58 +0100 Subject: [PATCH 115/357] test --- .github/workflows/pytest_file_or_dir.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 54921aa5f..c8ff73028 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -139,6 +139,7 @@ jobs: pytest --cov-append --cov=./ --cov-report=xml ${{ inputs.test_location }} else pytest --cov=./ --cov-report=xml ${{ inputs.test_location }} + coverage report fi else echo "Running tests without codecov" From 7e82fbb8db7b03ad61dfad78c6ba3c8c296b3b80 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 00:36:17 +0100 Subject: [PATCH 116/357] test --- .github/workflows/pytest_file_or_dir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index c8ff73028..6496d060b 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -159,7 +159,7 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ github.workspace }}/action/package/coverage.xml + files: coverage.xml root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos From f274f7e99478812fd4c724519b91de95be738c5b Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 00:42:35 +0100 Subject: [PATCH 117/357] test --- .github/workflows/pytest_file_or_dir.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 6496d060b..366e5efbc 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -154,13 +154,16 @@ jobs: retention-days: 1 if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml + - name: copy report to artifact + if: ${{ steps.codecov.outputs.should_run == 'true' }} + run: | + cp ${{ github.workspace }}/action/package/coverage.xml ${{ github.workspace }} - name: Upload coverage if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml - root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From 97bfa3cabcc11f976989533e99b928753461da1d Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 00:46:52 +0100 Subject: [PATCH 118/357] test --- .github/workflows/pytest_file_or_dir.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 366e5efbc..6496d060b 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -154,16 +154,13 @@ jobs: retention-days: 1 if-no-files-found: ignore path: ${{ github.workspace }}/action/package/coverage.xml - - name: copy report to artifact - if: ${{ steps.codecov.outputs.should_run == 'true' }} - run: | - cp ${{ github.workspace }}/action/package/coverage.xml ${{ github.workspace }} - name: Upload coverage if: ${{ steps.codecov.outputs.should_run == 'true' && inputs.upload_coverage }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml + root_dir: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From 635fc1f00dc7550cedea0db59905de20256335c5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 01:24:01 +0100 Subject: [PATCH 119/357] test --- .github/workflows/pytest_file_or_dir.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 6496d060b..aef17620d 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -161,6 +161,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml root_dir: ${{ github.workspace }}/action/package + directory: ${{ github.workspace }}/action/package + working-directory: ${{ github.workspace }}/action/package flags: unittests name: ovos verbose: true From 10252ebfc003326f96f2bfc9f0249d4f99569548 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 16:02:30 +0100 Subject: [PATCH 120/357] toss commented osm test --- test/test_skill_install.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/test/test_skill_install.py b/test/test_skill_install.py index 9cc7a6b9f..63965ba52 100644 --- a/test/test_skill_install.py +++ b/test/test_skill_install.py @@ -44,27 +44,6 @@ def get_skill_object(bus: FakeBus, path: str = "", return skill -# @unittest.skipIf(environ.get("TEST_OSM", "false") == "false", "TEST_OSM is false") -# class TestOSM(unittest.TestCase): -# def test_osm_install(self): -# branch = environ.get("TEST_BRANCH") -# install_url = f"https://github.com/{environ.get('TEST_REPO')}@{branch}" -# skill = SkillEntry.from_github_url(install_url) -# tmp_skills = "/tmp/osm_installed_skills" -# skill_folder = f"{tmp_skills}/{skill.uuid}" - -# if exists(skill_folder): -# rmtree(skill_folder) - -# updated = skill.install(folder=tmp_skills, default_branch=branch) -# self.assertEqual(updated, True) -# self.assertTrue(isdir(skill_folder)) -# self.assertTrue(isfile(join(skill_folder, "__init__.py"))) - -# updated = skill.install(folder=tmp_skills, default_branch=branch) -# self.assertEqual(updated, False) - - class TestSkillLoading(unittest.TestCase): @classmethod def setUpClass(self): From e74e45d230799692eb5b128e6c87c49018cdf465 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 19:16:35 +0100 Subject: [PATCH 121/357] prototype package handling --- .github/workflows/pytest_file_or_dir.yml | 37 +++++++++++++++--------- README.md | 16 +++++++--- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index aef17620d..da51379d6 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -25,6 +25,10 @@ on: description: 'Additional Python packages (whitespace delimited) to install' type: string required: false + pip_install_dirs: + description: 'Additional local Python package directories to install' + type: string + required: false install_extras: description: 'comma delimited extras to install the package with (eg: "test,dev")' type: string @@ -90,29 +94,36 @@ jobs: echo "should_run=false" >> "$GITHUB_OUTPUT" echo "codecov will not run" fi - - name: Install System Dependencies + - name: Install Common System Dependencies run: | sudo apt update - if [ -z "${{ inputs.system_deps }}" ]; then - xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt install -y ${{inputs.system_deps}} - fi + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Install Custom System Dependencies + if: ${{ inputs.system_deps != null }} + run: | + sudo apt install -y ${{ inputs.system_deps }} - name: upgrade pip run: | pip install --upgrade pip - - name: Install Base Skill Requirements - if: ${{ inputs.is_skill && !inputs.pip_packages }} + - name: Install Common Python Requirements run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - - name: Install Base Requirements - if: ${{ !inputs.is_skill && !inputs.pip_packages }} - run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_tests.txt + if [ "${{ inputs.is_skill }}" = "true" ]; then + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt + else + pip install -r ${{ github.workspace }}/action/github/requirements/pip_tests.txt + fi - name: Install Custom Python Requirements if: ${{ inputs.pip_packages != null }} run: | pip install ${{ inputs.pip_packages }} + - name: Install Python Package Directories + if: ${{ inputs.pip_install_dirs != null }} + run: | + IFS=$'\n' read -d '' -r -a dirs <<< "${{ inputs.pip_install_dirs }}" + for dir in "${dirs[@]}" + do + pip install -e "$dir" + done - name: Install Package run: | cd ${{ github.workspace }}/action/package diff --git a/README.md b/README.md index 27d2c31d8..144d82094 100644 --- a/README.md +++ b/README.md @@ -165,15 +165,23 @@ jobs: runner: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of the commonly used timeout_minutes: 15 # Timeout in minutes for the job, default: 15 + system-deps: "libfann-dev libfann2" # Additional system dependencies to install before running the license check + # commonly installed: `gcc python3-dev swig libssl-dev libfann-dev portaudio19-dev libpulse-dev` + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' + pip_packages: "pytest pytest-cov" # Additional python packages (whitespace delimited) to install + # commonly installed: + # Skill: see https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt + # Other: see https://github.com/OpenVoiceOS/.github/requirements/pip_tests.txt + pip_install_dirs: | # Additional directories to install python packages from + /path/to/package1 + /path/to/package2 install_extras: lgpl,mycroft # Comma-separated extras to install the python package with test_location: test/unittests # Test file (or directory) to run, default: test/unittests is_skill: true # Whether this is an ovos skill, default: false codecov: true # Whether to record the test code coverage, default: true + # below (append/upload_coverage) can be omitted if codecov is false + upload_coverage: true # Whether to upload the coverage to codecov server, default: false # should upload only if there are no following jobs that need coverage From 87c2f5107ca045ad7eb6abc68662b32f0c79babb Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 2 Feb 2024 19:31:01 +0100 Subject: [PATCH 122/357] ensure correct working dir --- .github/workflows/pytest_file_or_dir.yml | 1 + README.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index da51379d6..44237db11 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -119,6 +119,7 @@ jobs: - name: Install Python Package Directories if: ${{ inputs.pip_install_dirs != null }} run: | + cd ${{ github.workspace }}/action/package/ IFS=$'\n' read -d '' -r -a dirs <<< "${{ inputs.pip_install_dirs }}" for dir in "${dirs[@]}" do diff --git a/README.md b/README.md index 144d82094..70ba4795a 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,8 @@ jobs: # Skill: see https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt # Other: see https://github.com/OpenVoiceOS/.github/requirements/pip_tests.txt pip_install_dirs: | # Additional directories to install python packages from - /path/to/package1 - /path/to/package2 + relpath/to/package1 + relpath/to/package2 install_extras: lgpl,mycroft # Comma-separated extras to install the python package with test_location: test/unittests # Test file (or directory) to run, default: test/unittests is_skill: true # Whether this is an ovos skill, default: false From 17b643d903e3b07e6216ef054d5b793296acd642 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 6 Feb 2024 17:36:24 +0100 Subject: [PATCH 123/357] resource testing consolidation --- .github/workflows/skill_test_intents.yml | 13 +- README.md | 2 - requirements/sys_deb_common_deps.txt | 2 + test/test_skill_resourcesV2.py | 468 +++++++++++++++++++++++ 4 files changed, 473 insertions(+), 12 deletions(-) create mode 100644 test/test_skill_resourcesV2.py diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml index 049394a2a..5f98be9b8 100644 --- a/.github/workflows/skill_test_intents.yml +++ b/.github/workflows/skill_test_intents.yml @@ -28,10 +28,6 @@ on: description: 'Optional system dependencies to install instead of requirements.txt' required: false type: string - skill_id: - description: 'Skill ID to test' - required: true - type: string intent_testfile: description: 'Intent file to test' type: string @@ -47,8 +43,6 @@ on: jobs: test_intents_ovos: - env: - TEST_SKILL_ENTRYPOINT_NAME: ${{ inputs.skill_id }} runs-on: ${{inputs.runner}} timeout-minutes: ${{inputs.timeout}} strategy: @@ -89,9 +83,8 @@ jobs: fi cd ${{ github.workspace }}/action/skill pip install . - # when skill names are normalized - # - name: Set TEST_SKILL_ENTRYPOINT_NAME - # run: echo "TEST_SKILL_ENTRYPOINT_NAME=$(basename ${{ github.repository }} | sed 's/-/_/g').$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + PKG_NAME=$(python setup.py --name) + export TEST_SKILL_PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | @@ -103,4 +96,4 @@ jobs: run: | export INTENT_ENGINE="padatious" export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" - pytest action/github/test/test_skill_intents.py \ No newline at end of file + pytest action/github/test/test_skill_intents.py diff --git a/README.md b/README.md index 70ba4795a..0bb8f6fcc 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,6 @@ jobs: python_version: '"3.10"' # Python version to use, default: "3.8" pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml ``` ## Skill Intent Tests @@ -266,7 +265,6 @@ jobs: pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml - skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true ``` diff --git a/requirements/sys_deb_common_deps.txt b/requirements/sys_deb_common_deps.txt index df7073817..24c07e0a9 100644 --- a/requirements/sys_deb_common_deps.txt +++ b/requirements/sys_deb_common_deps.txt @@ -5,3 +5,5 @@ libssl-dev libfann-dev portaudio19-dev libpulse-dev +libicu-dev +pkg-config diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py new file mode 100644 index 000000000..4c1c24c23 --- /dev/null +++ b/test/test_skill_resourcesV2.py @@ -0,0 +1,468 @@ +import logging +from os import getenv +from os.path import isdir +from dataclasses import dataclass +from pathlib import Path +from functools import reduce +import operator +import random +from copy import deepcopy +from typing import Optional, List + +import importlib +import unittest +import yaml +from mock import Mock, patch + +from ovos_core.intent_services import PadatiousMatcher +from ovos_bus_client import Message +from ovos_bus_client.session import Session, SessionManager +from ovos_config.config import update_mycroft_config, Configuration +from ovos_utils.messagebus import FakeBus +from ovos_utils.log import LOG +from ovos_utils import flatten_list +from ovos_plugin_manager.skills import find_skill_plugins +from ovos_workshop.skill_launcher import SkillLoader +from ovos_workshop.skills.base import BaseSkill +from ovos_workshop.intents import IntentBuilder +from ovos_workshop.resource_files import SkillResources + + +PIPELINE = ["adapt_high", "adapt_medium", "adapt_low"] +use_padacioso = getenv("INTENT_ENGINE") == "padacioso" +if use_padacioso: + PIPELINE.extend(["padacioso_high", + "padacioso_medium", + "padacioso_low"]) +else: + PIPELINE.extend(["padatious_high", + "padatious_medium", + "padatious_low"]) +LOG.level = logging.DEBUG + +CAPTURE_INTENT_MESSAGES = [ + "register_vocab", + "register_intent", +] + +@dataclass +class Intent: + service: str + name: str + filestems: set + suffix: str = "" + + def __post_init__(self): + if self.service == "padatious": + self.suffix = ".intent" + elif self.service == "adapt": + self.suffix = ".voc" + + +class MockPadatiousMatcher(PadatiousMatcher): + include_med = True + include_low = False + + def __init__(self, *args, **kwargs): + PadatiousMatcher.__init__(self, *args, **kwargs) + LOG.debug("Creating test Padatious Matcher") + + def match_medium(self, utterances, lang=None, __=None): + if not self.include_med: + LOG.info(f"Skipping medium confidence check for {utterances}") + return None + PadatiousMatcher.match_medium(self, utterances, lang=lang) + + def match_low(self, utterances, lang=None, __=None): + if not self.include_low: + LOG.info(f"Skipping low confidence check for {utterances}") + return None + PadatiousMatcher.match_low(self, utterances, lang=lang) + + +def get_skill_object(skill_entrypoint: str, bus: FakeBus, + skill_id: str, config_patch: Optional[dict] = None) -> BaseSkill: + """ + Get an initialized skill object by entrypoint with the requested skill_id. + @param skill_entrypoint: Skill plugin entrypoint or directory path + @param bus: FakeBus instance to bind to skill for testing + @param skill_id: skill_id to initialize skill with + @param config_patch: Configuration update to apply + @returns: Initialized skill object + """ + if config_patch: + user_config = update_mycroft_config(config_patch) + if user_config not in Configuration.xdg_configs: + Configuration.xdg_configs.append(user_config) + + if isdir(skill_entrypoint): + LOG.info(f"Loading local skill: {skill_entrypoint}") + loader = SkillLoader(bus, skill_entrypoint, skill_id) + if loader.load(): + return loader.instance + plugins = find_skill_plugins() + if skill_entrypoint not in plugins: + raise ValueError(f"Requested skill not found: {skill_entrypoint}; available skills: {list(plugins.keys())}") + plugin = plugins[skill_entrypoint] + skill = plugin(bus=bus, skill_id=skill_id) + return skill + + +def get_intents_from_skillcode(skill) -> List[Intent]: + intents = [] + for method_name in dir(skill): + method = getattr(skill, method_name) + if callable(method) and hasattr(method, 'intents'): + for intent in method.intents: + if isinstance(intent, str): + # If the intent is a string, it's the intent name + stem = Path(intent).stem + # string contains the suffix ".intent" + intents.append(Intent("padatious", intent, [stem])) + elif isinstance(intent, IntentBuilder): + vocs = list() + if intent.at_least_one: + vocs.append(intent.at_least_one[0]) + if intent.requires: + vocs.append((intent.requires[0][0],)) + if intent.optional: + vocs.append((intent.optional[0][0],)) + + intents.append(Intent("adapt", intent.name, vocs)) + return intents + + +def count_permutations(options): + permutations = [] + for sublist in options: + choices = set(flatten_list(sublist)) + permutations.append(len(choices)) + return reduce(operator.mul, permutations, 1) + + +def generate_sentences(options: List[List[List[str]]], + max: int, + max_random: int = 0) -> List[str]: + sentences = [] + while len(sentences) < min(max, count_permutations(options)): + # we can add an ai sentence generator in the future + sentence = [] + for sublist in options: + choice = random.choice(sublist) + sentence.append(random.choice(choice)) + _sentence = " ".join(sentence) + if _sentence not in sentences: + sentences.append(" ".join(sentence)) + return sentences + + +class TestSkillIntents(unittest.TestCase): + messages: List[Message]= list() + last_message: Optional[Message] = None + valid_intents = dict() + intents = set() + vocab = set() + regex = set() + + valid_intents = dict() + negative_intents = dict() + common_query = dict() + + # make the default session use the test pipeline + session = Session("default", pipeline=PIPELINE) + SessionManager.default_session = session + SessionManager.sessions = {"default": session} + + # Start the IntentService + bus = FakeBus() + bus.run_forever() + from ovos_core.intent_services import IntentService + intent_service = IntentService(bus) + + test_skill_id = 'test_skill.test' + skill = None + + @classmethod + def init_intent_file(self): + """ + check the intent test yaml for recent changes and keep it up to date + """ + + skill_intents = get_intents_from_skillcode(self.skill) + # those intents most likely are identical, but we want the one used in the code + # (opposed to being present as resource and not being used in the skill) + skill_resources = self.skill.resources.get_inventory() + + # Load the test intent file + yaml_location = getenv("INTENT_TEST_FILE") + with open(yaml_location) as f: + test_yaml = yaml.safe_load(f) + + self.valid_intents = deepcopy(test_yaml) + self.negative_intents = self.valid_intents.pop('unmatched intents', dict()) + self.common_query = self.valid_intents.pop("common query", dict()) + self.regex = set(skill_resources['regex']) + + for intent in skill_intents: + self.intents.add(intent.name) + if intent.service == "adapt": + self.vocab.update({voc.lower() for voc in set(flatten_list(intent.filestems))}) + + assert self.skill.config_core["secondary_langs"] == self.supported_languages + # update yaml file based on the present intents + for lang in self.supported_languages: + test_yaml.setdefault(lang, dict()) + resources = self.skill.load_lang(lang=lang) + for intent in skill_intents: + test_yaml[lang].setdefault(intent.name, list()) + present_intents = test_yaml[lang][intent.name] + valid_intents = [] + # prepare adapt intents + if intent.service == "adapt": + options = [ + [ flatten_list(resources.load_vocabulary_file(voc)) + for voc in vocs ] for vocs in intent.filestems + ] + # filter out intents that don't match the options + for line in present_intents: + if all(any(any(word in line for word in choice) for choice in option) for option in options): + valid_intents.append(line) + # add possible combinations of options + for option in options: + if not any(any(any(word in intent for word in choice) for choice in option) for intent in valid_intents): + valid_intents.extend(generate_sentences(options, 4)) + # prepare padatious intents + elif intent.service == "padatious": + options = resources.load_intent_file(intent.name, entities=False) + # filter out intents that don't match the options + for line in present_intents: + if line in options: + valid_intents.append(line) + random.shuffle(options) + if len(valid_intents) < 5: + for option in options: + if option not in valid_intents: + valid_intents.append(option) + + test_yaml[lang][intent.name] = valid_intents + self.valid_intents[lang] = test_yaml[lang] + + with open(yaml_location, "w", encoding='utf8') as f: + yaml.dump(test_yaml, f, allow_unicode=True) + + @classmethod + def setUpClass(cls) -> None: + + def _on_message(msg): + cls.last_message = msg + cls.messages.append(msg) + + for msg_type in CAPTURE_INTENT_MESSAGES: + cls.bus.on(msg_type, _on_message) + + skill_folder = getenv("TEST_SKILL_PKG_FOLDER") + cls.skill = get_skill_object(skill_entrypoint=skill_folder, + bus=cls.bus, + skill_id=cls.test_skill_id) + + # Ensure all tested languages are loaded + import ovos_config + cls.supported_languages = SkillResources.get_available_languages(skill_folder) + update_mycroft_config({"secondary_langs": cls.supported_languages}) + importlib.reload(ovos_config.config) + + cls.init_intent_file() + + @classmethod + def tearDownClass(cls) -> None: + cls.skill.shutdown() + + def test_00_init(self): + for lang in self.valid_intents: + if hasattr(self.skill, "_native_langs"): + # ovos-workshop < 0.0.15 + self.assertIn(lang, self.skill._native_langs, lang) + else: + self.assertIn(lang, self.skill.native_langs, lang) + self.assertIn(lang, + self.intent_service.padatious_service.containers) + + def test_ressources(self): + """ + test if all resources are present with all languages + """ + inventory = self.skill.resources.get_inventory() + self.assertEqual(inventory["languages"], self.supported_languages) + for lang in self.supported_languages: + lang_inventory = self.skill.load_lang(lang=lang).get_inventory() + self.assertEqual(inventory, lang_inventory) + + def test_intent_registration(self): + """ + Test if all intents are registered + """ + registered_intents = set(self.skill.intent_service.intent_names) + registered_vocab = dict() + registered_regex = dict() + for msg in self.messages: + if msg.msg_type == "register_vocab": + if msg.data.get('regex'): + regex = msg.data["regex"].split( + '<', 1)[1].split('>', 1)[0].replace( + self.test_skill_id.replace('.', '_'), '') + registered_regex.setdefault(regex, list()) + registered_regex[regex].append(msg["data"]["regex"]) + else: + voc_filename = msg.data.get("entity_type", "").replace( + self.test_skill_id.replace('.', '_'), '').lower() + registered_vocab.setdefault(voc_filename, list()) + registered_vocab[voc_filename].append( + msg.data.get("entity_value", "")) + self.assertEqual(registered_intents, self.intents, + registered_intents) + if self.vocab: + self.assertEqual(set(registered_vocab.keys()), + self.vocab) + if self.regex: + self.assertEqual(set(registered_regex.keys()), + self.regex, registered_regex) + for voc in self.vocab: + # Ensure every vocab file has at least one entry + self.assertGreater(len(registered_vocab[voc]), 0) + for rx in self.regex: + # Ensure every rx file has exactly one entry + self.assertTrue(all((rx in line for line in + registered_regex[rx])), self.regex) + # TODO + + def test_intents(self): + """ + Test if all intents are correctly recognized by the intent parser + """ + for lang in self.valid_intents: + self.assertIsInstance(lang.split('-')[0], str) + self.assertIsInstance(lang.split('-')[1], str) + for intent, examples in self.valid_intents[lang].items(): + intent_event = f'{self.test_skill_id}:{intent}' + self.skill.events.remove(intent_event) + intent_handler = Mock() + self.skill.events.add(intent_event, intent_handler) + for utt in examples: + if isinstance(utt, dict): + data = list(utt.values())[0] + utt = list(utt.keys())[0] + else: + data = list() + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + try: + intent_handler.assert_called_once() + except AssertionError as e: + LOG.error(f"sent:{message.serialize()}") + LOG.error(f"received:{self.last_message.serialize()}") + raise AssertionError(utt) from e + intent_message = intent_handler.call_args[0][0] + self.assertIsInstance(intent_message, Message, utt) + self.assertEqual(intent_message.msg_type, intent_event, utt) + for datum in data: + if isinstance(datum, dict): + name = list(datum.keys())[0] + value = list(datum.values())[0] + else: + name = datum + value = None + if name in intent_message.data: + # This is an entity + voc_id = name + else: + # We mocked the handler, data is munged + voc_id = f'{self.test_skill_id.replace(".", "_")}' \ + f'{name}' + self.assertIsInstance(intent_message.data.get(voc_id), + str, intent_message.data) + if value: + self.assertEqual(intent_message.data.get(voc_id), + value, utt) + intent_handler.reset_mock() + + @patch("ovos_core.intent_services.padacioso_service.PadaciosoService", + new=MockPadatiousMatcher) + def test_negative_intents(self): + test_config = self.negative_intents.pop('config', None) + if test_config: + MockPadatiousMatcher.include_med = test_config.get('include_med', + True) + MockPadatiousMatcher.include_low = test_config.get('include_low', + False) + + intent_failure = Mock() + original_failure = self.intent_service.send_complete_intent_failure + self.intent_service.send_complete_intent_failure = intent_failure + + # # Skip any fallback/converse handling + # self.intent_service.fallback = Mock() + # self.intent_service.converse = Mock() + # if not self.common_query: + # # Skip common_qa unless explicitly testing a Common QA skill + # self.intent_service.common_qa = Mock() + + for lang in self.negative_intents.keys(): + for utt in self.negative_intents[lang]: + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + try: + intent_failure.assert_called_once_with(message) + intent_failure.reset_mock() + except AssertionError as e: + LOG.error(self.last_message) + raise AssertionError(utt) from e + + self.intent_service.send_complete_intent_failure = original_failure + + def test_common_query(self): + qa_callback = Mock() + qa_response = Mock() + self.skill.events.add('question:action', qa_callback) + self.skill.events.add('question:query.response', qa_response) + for lang in self.common_query.keys(): + for utt in self.common_query[lang]: + if isinstance(utt, dict): + data = list(utt.values())[0] + utt = list(utt.keys())[0] + else: + data = dict() + message = Message('test_utterance', + {"utterances": [utt], "lang": lang}) + self.intent_service.handle_utterance(message) + response = qa_response.call_args[0][0] + callback = qa_response.call_args[0][0] + self.assertIsInstance(response, Message) + self.assertTrue(response.data["phrase"] in utt) + self.assertEqual(response.data["skill_id"], self.skill.skill_id) + self.assertIn("callback_data", response.data.keys()) + self.assertIsInstance(response.data["conf"], float) + self.assertIsInstance(response.data["answer"], str) + + self.assertIsInstance(callback, Message) + self.assertEqual(callback.data['skill_id'], self.skill.skill_id) + self.assertEqual(callback.data['phrase'], + response.data['phrase']) + if not data: + continue + if isinstance(data.get('callback'), dict): + self.assertEqual(callback.data['callback_data'], + data['callback']) + elif isinstance(data.get('callback'), list): + self.assertEqual(set(callback.data['callback_data'].keys()), + set(data.get('callback'))) + if data.get('min_confidence'): + self.assertGreaterEqual(response.data['conf'], + data['min_confidence']) + if data.get('max_confidence'): + self.assertLessEqual(response.data['conf'], + data['max_confidence']) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 89189bc092198b79d3396d905c4f9d33547a1fc9 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 6 Feb 2024 20:14:22 +0100 Subject: [PATCH 124/357] update to use v2 test resource routine --- .github/workflows/skill_test_intents.yml | 99 ---------------------- .github/workflows/skill_test_resources.yml | 62 +++++++++----- README.md | 25 +----- 3 files changed, 46 insertions(+), 140 deletions(-) delete mode 100644 .github/workflows/skill_test_intents.yml diff --git a/.github/workflows/skill_test_intents.yml b/.github/workflows/skill_test_intents.yml deleted file mode 100644 index 5f98be9b8..000000000 --- a/.github/workflows/skill_test_intents.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Skill Intents Tests -on: - workflow_call: - inputs: - runner: - type: string - default: "ubuntu-latest" - branch: - description: 'Branch to test' - type: string - default: ${{ github.ref }} - action_branch: - description: 'The shared action branch to checkout' - type: string - default: main - timeout: - type: number - default: 15 - python_matrix: - description: 'Python matrix (string) to use' - type: string - default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: - description: 'Custom pip packages (whitespace delimited) to install instead of pip_skill_tests.txt' - required: false - type: string - system_deps: - description: 'Optional system dependencies to install instead of requirements.txt' - required: false - type: string - intent_testfile: - description: 'Intent file to test' - type: string - default: test/unittests/intent_tests.yaml - test_padatious: - description: 'Whether to skip padatious tests' - type: boolean - default: False - test_padacioso: - description: 'Whether to skip padacioso tests' - type: boolean - default: True - -jobs: - test_intents_ovos: - runs-on: ${{inputs.runner}} - timeout-minutes: ${{inputs.timeout}} - strategy: - matrix: - python-version: ${{ fromJSON(inputs.python_matrix) }} - steps: - - name: Checkout Skill Repo - uses: actions/checkout@v4 - with: - ref: ${{inputs.branch}} - path: action/skill/ - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{inputs.action_branch}} - path: action/github/ - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install System Dependencies - run: | - sudo apt update - if [ -z "${{inputs.system_deps}}" ]; then - xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt install -y ${{inputs.system_deps}} - fi - - name: Install Python Skill and Requirements - run: | - pip install --upgrade pip - pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 - if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - else - pip install ${{inputs.pip_packages}} - fi - cd ${{ github.workspace }}/action/skill - pip install . - PKG_NAME=$(python setup.py --name) - export TEST_SKILL_PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) - - name: Test Skill Intents Padacioso - if: ${{ inputs.test_padacioso }} - run: | - export INTENT_ENGINE="padacioso" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" - pytest action/github/test/test_skill_intents.py - - name: Test Skill Intents Padatious - if: ${{ inputs.test_padatious }} - run: | - export INTENT_ENGINE="padatious" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" - pytest action/github/test/test_skill_intents.py diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 33c0ce2c9..ca8e38871 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -14,32 +14,41 @@ on: description: 'The shared action branch to checkout' type: string default: main + timeout: + type: number + default: 15 system_deps: description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' required: false type: string - python_version: - description: 'The python version to use' + python_matrix: + description: 'Python matrix (string) to use' type: string - default: "3.8" + default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' required: false type: string - skill_id: - description: 'Skill ID to test' - required: true + intent_testfile: + description: 'Intent file to test' type: string - resource_testfile: - type: string - default: test/unittests/test_resources.yaml + default: test/unittests/intent_tests.yaml + test_padatious: + description: 'Whether to skip padatious tests' + type: boolean + default: False + test_padacioso: + description: 'Whether to skip padacioso tests' + type: boolean + default: True jobs: test_resources: - env: - TEST_SKILL_ENTRYPOINT_NAME: ${{inputs.skill_id}} - runs-on: ${{inputs.runner}} - timeout-minutes: 5 + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout }} + strategy: + matrix: + python-version: ${{ fromJSON(inputs.python_matrix) }} steps: - name: Checkout Skill Repo uses: actions/checkout@v4 @@ -52,10 +61,10 @@ jobs: repository: OpenVoiceOS/.github ref: ${{inputs.action_branch}} path: action/github/ - - name: Set up python + - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: ${{inputs.python_version}} + python-version: ${{ matrix.python-version }} - name: Install System Dependencies run: | sudo apt update @@ -74,8 +83,23 @@ jobs: pip install ${{inputs.pip_packages}} fi cd ${{ github.workspace }}/action/skill - pip install . - - name: Test Skill Resources + pip install -e . + PKG_NAME=$(python setup.py --name) + export TEST_SKILL_PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) + - name: Test Skill Intents Padacioso + if: ${{ inputs.test_padacioso }} + run: | + export INTENT_ENGINE="padacioso" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" + pytest action/github/test/test_skill_resourcesV2.py + - name: Test Skill Intents Padatious + if: ${{ inputs.test_padatious }} run: | - export RESOURCE_TEST_FILE="${{ github.workspace }}/action/skill/${{inputs.resource_testfile}}" - pytest ${{ github.workspace }}/action/github/test/test_skill_resources.py \ No newline at end of file + export INTENT_ENGINE="padatious" + export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" + pytest action/github/test/test_skill_resourcesV2.py + - name: Push testyaml Change + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "update test yaml" + repository: action/skill/ diff --git a/README.md b/README.md index 0bb8f6fcc..bded45f88 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ jobs: skill_location: "skill" # Skill location relative to the root (can usually be omitted) ``` ## Skill Resource Tests -Tests the resources of a skill. This may be dialogs, vocabs, regex or intent resources +Tests the resources of a skill (e.g dialogs, vocabs, regex or intent resources) for completeness and workability. ```yaml name: Skill Ressource Tests on: @@ -237,33 +237,14 @@ jobs: uses: openvoiceos/.github/.github/workflows/skill_test_resources.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest + timeout: 15 # Timeout for the test, default: 15 branch: dev # Branch to use, default: branch that triggered the action + action_branch: custom/branch # Shared action branch to use, default: main system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt python_version: '"3.10"' # Python version to use, default: "3.8" pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - resource_file: test/test_resources.yaml # Resource test file to test against, default: test/test_resources.yaml -``` -## Skill Intent Tests -Tests if intents are successfully parsed by the skill -```yaml -name: Skill Intent Tests -on: - - -jobs: - skill_intent_tests: - uses: openvoiceos/.github/.github/workflows/skill_test_intents.yml@main - with: - runner: ubuntu-latest # Runner to use, default: ubuntu-latest - branch: dev # Branch to use, default: branch that triggered the action - timeout: 15 # Timeout for the test, default: 15 - system-deps: "libfann-dev libfann2" # Custom system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Custom python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true From 1b00df093327dc24642ea9e2838416fa3d8c5f19 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 6 Feb 2024 20:59:53 +0100 Subject: [PATCH 125/357] readme fixes --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bded45f88..c9c28a5bb 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ jobs: uses: openvoiceos/.github/.github/workflows/propose_semver_release.yml@main with: branch: dev # Branch to use, default: branch that triggered the action - python_version: '"3.10"' # Python version (quoted) to use, default: 3.8 + python_version: "3.10" # Python version (quoted) to use, default: 3.8 version_file: ovos_core/version.py # File location of the version file, default: version.py release_type: ${{inputs.release_type}} # build, minor, major changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md @@ -75,7 +75,7 @@ jobs: uses: openvoiceos/.github/.github/workflows/publish_stable_release.yml@main secrets: inherit with: - python_version: '"3.10"' + python_version: "3.10" release_type: ${{inputs.release_type}} changelog_file: ChAnGeLoG.md ``` @@ -103,7 +103,7 @@ jobs: secrets: inherit with: branch: dev # Branch to use, default: branch that triggered the action - python_version: '"3.8"' + python_version: "3.8" language: ${{ inputs.translation }} locale_folder: ovos_core/locale/ # the location of the base localisation folder, default: locale reviewers: "jarbasai,emphasize" # comma separated list of reviewers, default: emphasize @@ -123,7 +123,7 @@ jobs: with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action - python-version: '"3.8"' # Python version (quoted) to use, default: 3.8 + python-version: "3.8" # Python version (quoted) to use, default: 3.8 system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt package-extras: test # Optional extras to install the python package with @@ -242,7 +242,7 @@ jobs: action_branch: custom/branch # Shared action branch to use, default: main system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check # default packages found at requirements/sys_deb_common_deps.txt - python_version: '"3.10"' # Python version to use, default: "3.8" + python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml From 020c430c7dc453ffa1b4b27b603670970d78bb8d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 15:01:53 +0100 Subject: [PATCH 126/357] test --- .github/workflows/skill_test_resources.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index ca8e38871..5dd51d0bf 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -86,6 +86,8 @@ jobs: pip install -e . PKG_NAME=$(python setup.py --name) export TEST_SKILL_PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) + echo "Package name: $PKG_NAME" + echo "Test folder: $TEST_SKILL_PKG_FOLDER" - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | From bc3379b3808c24823a1a8bdf73a69256a93b992e Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 15:31:38 +0100 Subject: [PATCH 127/357] test --- .github/workflows/skill_test_resources.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 5dd51d0bf..3d32b9633 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -85,20 +85,24 @@ jobs: cd ${{ github.workspace }}/action/skill pip install -e . PKG_NAME=$(python setup.py --name) - export TEST_SKILL_PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) - echo "Package name: $PKG_NAME" - echo "Test folder: $TEST_SKILL_PKG_FOLDER" + BASE_FOLDER=$(echo $PKG_NAME | tr '-' '_') + PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) + echo "TEST_SKILL_PKG_FOLDER: $PKG_FOLDER/$BASE_FOLDER" >> $GITHUB_ENV - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | export INTENT_ENGINE="padacioso" export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" + echo "Intent Test File: $INTENT_TEST_FILE" + echo "Skill Folder: ${{ env.TEST_SKILL_PKG_FOLDER }}" pytest action/github/test/test_skill_resourcesV2.py - name: Test Skill Intents Padatious if: ${{ inputs.test_padatious }} run: | export INTENT_ENGINE="padatious" export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" + echo "Intent Test File: $INTENT_TEST_FILE" + echo "Skill Folder: ${{ env.TEST_SKILL_PKG_FOLDER }}" pytest action/github/test/test_skill_resourcesV2.py - name: Push testyaml Change uses: stefanzweifel/git-auto-commit-action@v4 From a4dcd454b6166fdf5220b6b11754f79a03ff1b26 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 15:38:44 +0100 Subject: [PATCH 128/357] test --- .github/workflows/skill_test_resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 3d32b9633..ad2d35f00 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -87,7 +87,7 @@ jobs: PKG_NAME=$(python setup.py --name) BASE_FOLDER=$(echo $PKG_NAME | tr '-' '_') PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) - echo "TEST_SKILL_PKG_FOLDER: $PKG_FOLDER/$BASE_FOLDER" >> $GITHUB_ENV + echo "TEST_SKILL_PKG_FOLDER=$PKG_FOLDER/$BASE_FOLDER" >> $GITHUB_ENV - name: Test Skill Intents Padacioso if: ${{ inputs.test_padacioso }} run: | From 85552e1bf6043697a4e8a8174676309565db2d7b Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 15:47:02 +0100 Subject: [PATCH 129/357] test --- requirements/pip_skill_tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/pip_skill_tests.txt b/requirements/pip_skill_tests.txt index eab8c7cdc..cd1f630a8 100644 --- a/requirements/pip_skill_tests.txt +++ b/requirements/pip_skill_tests.txt @@ -5,6 +5,7 @@ mock wheel cython ovos-core>=0.0.8a66 +ovos-workshop>=0.0.16a14 ovos-phal-plugin-connectivity-events~=0.0.1 padatious>=0.4.8, < 0.5.0 From fc2fb4266dca3bcb703d3249dec610a6a7e826e1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 18:20:02 +0100 Subject: [PATCH 130/357] test --- test/test_skill_resourcesV2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index 4c1c24c23..7a2abb8a3 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -284,8 +284,11 @@ def test_00_init(self): self.assertIn(lang, self.skill._native_langs, lang) else: self.assertIn(lang, self.skill.native_langs, lang) - self.assertIn(lang, - self.intent_service.padatious_service.containers) + if use_padacioso: + intent_containers = self.intent_service.padacioso_service.containers + else: + intent_containers = self.intent_service.padatious_service.containers + self.assertIn(lang, intent_containers) def test_ressources(self): """ From 759957d10cd7ef8aee752fc2a659be1170a114bf Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 19:12:43 +0100 Subject: [PATCH 131/357] test --- test/test_skill_resourcesV2.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index 7a2abb8a3..178209df6 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -14,7 +14,7 @@ import yaml from mock import Mock, patch -from ovos_core.intent_services import PadatiousMatcher +from ovos_core.intent_services import PadatiousMatcher, IntentService from ovos_bus_client import Message from ovos_bus_client.session import Session, SessionManager from ovos_config.config import update_mycroft_config, Configuration @@ -256,21 +256,26 @@ def setUpClass(cls) -> None: def _on_message(msg): cls.last_message = msg cls.messages.append(msg) - - for msg_type in CAPTURE_INTENT_MESSAGES: - cls.bus.on(msg_type, _on_message) - - skill_folder = getenv("TEST_SKILL_PKG_FOLDER") - cls.skill = get_skill_object(skill_entrypoint=skill_folder, - bus=cls.bus, - skill_id=cls.test_skill_id) + skill_folder = getenv("SKILL_FOLDER") # Ensure all tested languages are loaded import ovos_config cls.supported_languages = SkillResources.get_available_languages(skill_folder) update_mycroft_config({"secondary_langs": cls.supported_languages}) importlib.reload(ovos_config.config) + # Start the IntentService + cls.bus = FakeBus() + cls.bus.run_forever() + cls.intent_service = IntentService(cls.bus) + + for msg_type in CAPTURE_INTENT_MESSAGES: + cls.bus.on(msg_type, _on_message) + + cls.skill = get_skill_object(skill_entrypoint=skill_folder, + bus=cls.bus, + skill_id=cls.test_skill_id) + cls.init_intent_file() @classmethod From 4f0458d45dba40585cd8380b14382c93f50391a6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 19:13:50 +0100 Subject: [PATCH 132/357] test --- test/test_skill_resourcesV2.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index 178209df6..3d9aaaf9e 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -173,12 +173,6 @@ class TestSkillIntents(unittest.TestCase): SessionManager.default_session = session SessionManager.sessions = {"default": session} - # Start the IntentService - bus = FakeBus() - bus.run_forever() - from ovos_core.intent_services import IntentService - intent_service = IntentService(bus) - test_skill_id = 'test_skill.test' skill = None From 93778029cf65ab254098111ac547798b268ba259 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 19:19:37 +0100 Subject: [PATCH 133/357] test --- test/test_skill_resourcesV2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index 3d9aaaf9e..99d637929 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -251,7 +251,7 @@ def _on_message(msg): cls.last_message = msg cls.messages.append(msg) - skill_folder = getenv("SKILL_FOLDER") + skill_folder = getenv("TEST_SKILL_PKG_FOLDER") # Ensure all tested languages are loaded import ovos_config cls.supported_languages = SkillResources.get_available_languages(skill_folder) From bafd49db63a97e70dafa03ec32dfab8d8bdcccd1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 20:27:30 +0100 Subject: [PATCH 134/357] test --- test/test_skill_resourcesV2.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index 99d637929..ec807effc 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -6,6 +6,7 @@ from functools import reduce import operator import random +import re from copy import deepcopy from typing import Optional, List @@ -227,7 +228,10 @@ def init_intent_file(self): valid_intents.extend(generate_sentences(options, 4)) # prepare padatious intents elif intent.service == "padatious": - options = resources.load_intent_file(intent.name, entities=False) + options = resources.load_intent_file(intent.name) + # substitute entities + for i, option in enumerate(options): + options[i] = re.sub(r'\{.*?\}', "test", option) # filter out intents that don't match the options for line in present_intents: if line in options: @@ -350,6 +354,7 @@ def test_intents(self): intent_handler = Mock() self.skill.events.add(intent_event, intent_handler) for utt in examples: + LOG.info(f"Testing utterance '{utt}'") if isinstance(utt, dict): data = list(utt.values())[0] utt = list(utt.keys())[0] From 54678ad78f7c1eb4c041cf743aee88b0ddd22120 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 21:14:15 +0100 Subject: [PATCH 135/357] test --- .github/workflows/skill_test_resources.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index ad2d35f00..495825659 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -33,14 +33,6 @@ on: description: 'Intent file to test' type: string default: test/unittests/intent_tests.yaml - test_padatious: - description: 'Whether to skip padatious tests' - type: boolean - default: False - test_padacioso: - description: 'Whether to skip padacioso tests' - type: boolean - default: True jobs: test_resources: @@ -88,16 +80,7 @@ jobs: BASE_FOLDER=$(echo $PKG_NAME | tr '-' '_') PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) echo "TEST_SKILL_PKG_FOLDER=$PKG_FOLDER/$BASE_FOLDER" >> $GITHUB_ENV - - name: Test Skill Intents Padacioso - if: ${{ inputs.test_padacioso }} - run: | - export INTENT_ENGINE="padacioso" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" - echo "Intent Test File: $INTENT_TEST_FILE" - echo "Skill Folder: ${{ env.TEST_SKILL_PKG_FOLDER }}" - pytest action/github/test/test_skill_resourcesV2.py - - name: Test Skill Intents Padatious - if: ${{ inputs.test_padatious }} + - name: Test Skill Resources run: | export INTENT_ENGINE="padatious" export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" From 23b33bc678ff4d7b596f709ad8c61c817f35580b Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 7 Feb 2024 21:57:36 +0100 Subject: [PATCH 136/357] test --- .github/workflows/publish_alpha_release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index edfff1181..29ad7957a 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -38,7 +38,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} + ref: ${{ github.ref }} path: action/package/ - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -65,12 +65,13 @@ jobs: run: | python -m pip install -r action/github/requirements/pip_translation.txt python action/github/scripts/translate.py - - name: Commit to dev + - name: Commit to ${{ inputs.branch }} if: steps.filter.outputs.locales == 'true' uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: autotranslate repository: action/package/ + branch: ${{ inputs.branch }} build_and_publish: runs-on: ubuntu-latest From 35728ebcd76fe257560a1b54a66a561cca781918 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 14:56:22 +0100 Subject: [PATCH 137/357] split scripts --- .github/workflows/publish_alpha_release.yml | 34 +++- .github/workflows/skill_test_resources.yml | 15 +- README.md | 7 +- scripts/update_intent_testfile.py | 178 ++++++++++++++++++++ test/test_skill_resourcesV2.py | 153 ++--------------- 5 files changed, 229 insertions(+), 158 deletions(-) create mode 100644 scripts/update_intent_testfile.py diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/publish_alpha_release.yml index 29ad7957a..8c9acdd69 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/publish_alpha_release.yml @@ -24,6 +24,10 @@ on: description: 'The base folder to look for localization files' type: string default: locale + update_intentfile: + description: 'The file to update with the resource changes. This is atm only used for skills.' + type: string + required: false changelog_file: type: string default: CHANGELOG.md @@ -38,6 +42,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: + fetch-depth: 0 ref: ${{ github.ref }} path: action/package/ - name: Checkout Scripts Repo @@ -51,28 +56,45 @@ jobs: with: working-directory: action/package/ filters: | - locales: + us_specific: - '${{ inputs.locale_folder }}/en-us/**' - '${{ inputs.locale_folder }}/../dialog/en-us/**' - '${{ inputs.locale_folder }}/../vocab/en-us/**' + general: + - '${{ inputs.locale_folder }}/**' + - '${{ inputs.locale_folder }}/../dialog/**' + - '${{ inputs.locale_folder }}/../vocab/**' - name: Setup Python - if: steps.filter.outputs.locales == 'true' + if: steps.filter.outputs.us_specific == 'true' uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} - - name: Auto Translate - if: steps.filter.outputs.locales == 'true' + - name: Install Python Requirements + if: steps.filter.outputs.us_specific == 'true' run: | python -m pip install -r action/github/requirements/pip_translation.txt + - name: Pull latest changes + if: steps.filter.outputs.us_specific == 'true' + run: | + git pull origin ${{ inputs.branch }} + - name: Auto Translate + if: steps.filter.outputs.us_specific == 'true' + run: | python action/github/scripts/translate.py - name: Commit to ${{ inputs.branch }} - if: steps.filter.outputs.locales == 'true' + if: steps.filter.outputs.us_specific == 'true' uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: autotranslate repository: action/package/ branch: ${{ inputs.branch }} - + - name: update and commit resource_changes + if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }} + run: | + python action/github/scripts/update_intent_testfile.py + git add ${{ inputs.update_intentfile }} + git commit -m "Update resource test file" + build_and_publish: runs-on: ubuntu-latest env: diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 495825659..08c6b80f7 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -38,6 +38,8 @@ jobs: test_resources: runs-on: ${{ inputs.runner }} timeout-minutes: ${{ inputs.timeout }} + env: + INTENT_TEST_FILE: ${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }} strategy: matrix: python-version: ${{ fromJSON(inputs.python_matrix) }} @@ -75,20 +77,17 @@ jobs: pip install ${{inputs.pip_packages}} fi cd ${{ github.workspace }}/action/skill - pip install -e . + pip install . PKG_NAME=$(python setup.py --name) BASE_FOLDER=$(echo $PKG_NAME | tr '-' '_') PKG_FOLDER=$(pip show -f $PKG_NAME | grep Location | cut -d ' ' -f 2) echo "TEST_SKILL_PKG_FOLDER=$PKG_FOLDER/$BASE_FOLDER" >> $GITHUB_ENV + - name: Update Resource Testfile + run: | + python action/github/scripts/update_intent_testfile.py - name: Test Skill Resources run: | export INTENT_ENGINE="padatious" - export INTENT_TEST_FILE="${{ github.workspace }}/action/skill/${{ inputs.intent_testfile }}" - echo "Intent Test File: $INTENT_TEST_FILE" + echo "Intent Test File: ${{ env.INTENT_TEST_FILE }}" echo "Skill Folder: ${{ env.TEST_SKILL_PKG_FOLDER }}" pytest action/github/test/test_skill_resourcesV2.py - - name: Push testyaml Change - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "update test yaml" - repository: action/skill/ diff --git a/README.md b/README.md index c9c28a5bb..0b502f6d6 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ jobs: branch: dev # Branch to use, default: branch that triggered the action version_file: ovos_core/version.py # File location of the version file, default: version.py python_version: "3.8" # Python version (quoted) to use, default: 3.8 - locale_folder: ovos_core/locale # if there are localisation files the location of the base folder, default: locale - changelog_file: CHANGELOG.md # if the changlog file has a special name, default: CHANGELOG.md + locale_folder: ovos_core/locale # use if there are localisation files the location of the base folder, default: locale + update_intentfile: test/test_intents.yaml # use if there are changes to resource files, the test file to update. (ONLY USED IN SKILLS) + changelog_file: CHANGELOG.md # use if the changlog file has a special name, default: CHANGELOG.md ``` ## Propose and Publish Stable (Build,Minor,Major) Release Strategy: 2-fold @@ -154,7 +155,7 @@ jobs: ``` ## Unit Tests (file or directory) ```yaml -name: Skill Unit Tests +name: Unit Tests on: diff --git a/scripts/update_intent_testfile.py b/scripts/update_intent_testfile.py new file mode 100644 index 000000000..920c74ee7 --- /dev/null +++ b/scripts/update_intent_testfile.py @@ -0,0 +1,178 @@ +from os import getenv +from os.path import isdir, isfile +from dataclasses import dataclass +from pathlib import Path +from functools import reduce +import operator +import random +import re +from typing import List +import yaml + +from ovos_utils.log import LOG +from ovos_utils.messagebus import FakeBus +from ovos_utils import flatten_list +from ovos_workshop.skill_launcher import SkillLoader +from ovos_workshop.skills.base import BaseSkill +from ovos_workshop.intents import IntentBuilder + + +@dataclass +class Intent: + service: str + name: str + filestems: set + suffix: str = "" + + def __post_init__(self): + if self.service == "padatious": + self.suffix = ".intent" + elif self.service == "adapt": + self.suffix = ".voc" + + +def get_skill_object() -> BaseSkill: + """ + Get an initialized skill object by entrypoint with the requested skill_id. + @param skill_entrypoint: Skill plugin entrypoint or directory path + @param bus: FakeBus instance to bind to skill for testing + @param skill_id: skill_id to initialize skill with + @param config_patch: Configuration update to apply + @returns: Initialized skill object + """ + + bus = FakeBus() + bus.run_forever() + + skill_folder = getenv("TEST_SKILL_PKG_FOLDER") + if not skill_folder or not isdir(skill_folder): + raise ValueError("TEST_SKILL_PKG_FOLDER is not set or invalid") + + LOG.info(f"Loading local skill: {skill_folder}") + loader = SkillLoader(bus, skill_folder, "unknown") + if loader.load(): + return loader.instance + + return None + + +def get_intents_from_skillcode(skill) -> List[Intent]: + intents = [] + for method_name in dir(skill): + method = getattr(skill, method_name) + if callable(method) and hasattr(method, 'intents'): + for intent in method.intents: + if isinstance(intent, str): + # If the intent is a string, it's the intent name + stem = Path(intent).stem + # string contains the suffix ".intent" + intents.append(Intent("padatious", intent, [stem])) + elif isinstance(intent, IntentBuilder): + vocs = list() + if intent.at_least_one: + vocs.append(intent.at_least_one[0]) + if intent.requires: + vocs.append((intent.requires[0][0],)) + if intent.optional: + vocs.append((intent.optional[0][0],)) + + intents.append(Intent("adapt", intent.name, vocs)) + return intents + + +def count_permutations(options): + permutations = [] + for sublist in options: + choices = set(flatten_list(sublist)) + permutations.append(len(choices)) + return reduce(operator.mul, permutations, 1) + + +def generate_sentences(options: List[List[List[str]]], + max: int, + max_random: int = 0) -> List[str]: + sentences = [] + while len(sentences) < min(max, count_permutations(options)): + # we can add an ai sentence generator in the future + sentence = [] + for sublist in options: + choice = random.choice(sublist) + sentence.append(random.choice(choice)) + _sentence = " ".join(sentence) + if _sentence not in sentences: + sentences.append(" ".join(sentence)) + return sentences + + +def update_resources(skill: BaseSkill): + + if skill is None: + raise ValueError("Skill not found") + + intents = set() + supported_languages =skill.resources.get_inventory().get("languages") + + # we want the intents used in the code + # (opposed to being present as resource and not being used in the skill) + skill_intents = get_intents_from_skillcode(skill) + + # Load the test intent file + yaml_location = getenv("INTENT_TEST_FILE") + if not yaml_location or not isfile(yaml_location): + raise ValueError("INTENT_TEST_FILE is not set or invalid") + + with open(yaml_location) as f: + test_yaml = yaml.safe_load(f) + + for intent in skill_intents: + intents.add(intent.name) + + # update yaml file based on the present intents + for lang in supported_languages: + test_yaml.setdefault(lang, dict()) + resources = skill.load_lang(lang=lang) + for intent in skill_intents: + test_yaml[lang].setdefault(intent.name, list()) + present_intents = test_yaml[lang][intent.name] + valid_intents = [] + # prepare adapt intents + if intent.service == "adapt": + options = [ + [ flatten_list(resources.load_vocabulary_file(voc)) + for voc in vocs ] for vocs in intent.filestems + ] + # filter out intents that don't match the options + for line in present_intents: + if all(any(any(word in line for word in choice) for choice in option) for option in options): + valid_intents.append(line) + # add possible combinations of options + for option in options: + if not any(any(any(word in intent for word in choice) for choice in option) for intent in valid_intents): + valid_intents.extend(generate_sentences(options, 4)) + # prepare padatious intents + elif intent.service == "padatious": + options = resources.load_intent_file(intent.name) + # substitute entities + for i, option in enumerate(options): + options[i] = re.sub(r'\{.*?\}', "test", option) + # filter out intents that don't match the options + for line in present_intents: + if line in options: + valid_intents.append(line) + random.shuffle(options) + if len(valid_intents) < 5: + for option in options: + if option not in valid_intents: + valid_intents.append(option) + + test_yaml[lang][intent.name] = valid_intents + + LOG.info(f"Test yaml: {test_yaml}") + with open(yaml_location, "w", encoding='utf8') as f: + yaml.dump(test_yaml, f, allow_unicode=True) + + # shutdown skill + skill.shutdown() + + +update_resources(get_skill_object()) diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index ec807effc..e3dc050a4 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -1,13 +1,6 @@ import logging from os import getenv from os.path import isdir -from dataclasses import dataclass -from pathlib import Path -from functools import reduce -import operator -import random -import re -from copy import deepcopy from typing import Optional, List import importlib @@ -21,11 +14,9 @@ from ovos_config.config import update_mycroft_config, Configuration from ovos_utils.messagebus import FakeBus from ovos_utils.log import LOG -from ovos_utils import flatten_list from ovos_plugin_manager.skills import find_skill_plugins from ovos_workshop.skill_launcher import SkillLoader from ovos_workshop.skills.base import BaseSkill -from ovos_workshop.intents import IntentBuilder from ovos_workshop.resource_files import SkillResources @@ -46,19 +37,6 @@ "register_intent", ] -@dataclass -class Intent: - service: str - name: str - filestems: set - suffix: str = "" - - def __post_init__(self): - if self.service == "padatious": - self.suffix = ".intent" - elif self.service == "adapt": - self.suffix = ".voc" - class MockPadatiousMatcher(PadatiousMatcher): include_med = True @@ -109,54 +87,6 @@ def get_skill_object(skill_entrypoint: str, bus: FakeBus, return skill -def get_intents_from_skillcode(skill) -> List[Intent]: - intents = [] - for method_name in dir(skill): - method = getattr(skill, method_name) - if callable(method) and hasattr(method, 'intents'): - for intent in method.intents: - if isinstance(intent, str): - # If the intent is a string, it's the intent name - stem = Path(intent).stem - # string contains the suffix ".intent" - intents.append(Intent("padatious", intent, [stem])) - elif isinstance(intent, IntentBuilder): - vocs = list() - if intent.at_least_one: - vocs.append(intent.at_least_one[0]) - if intent.requires: - vocs.append((intent.requires[0][0],)) - if intent.optional: - vocs.append((intent.optional[0][0],)) - - intents.append(Intent("adapt", intent.name, vocs)) - return intents - - -def count_permutations(options): - permutations = [] - for sublist in options: - choices = set(flatten_list(sublist)) - permutations.append(len(choices)) - return reduce(operator.mul, permutations, 1) - - -def generate_sentences(options: List[List[List[str]]], - max: int, - max_random: int = 0) -> List[str]: - sentences = [] - while len(sentences) < min(max, count_permutations(options)): - # we can add an ai sentence generator in the future - sentence = [] - for sublist in options: - choice = random.choice(sublist) - sentence.append(random.choice(choice)) - _sentence = " ".join(sentence) - if _sentence not in sentences: - sentences.append(" ".join(sentence)) - return sentences - - class TestSkillIntents(unittest.TestCase): messages: List[Message]= list() last_message: Optional[Message] = None @@ -177,76 +107,6 @@ class TestSkillIntents(unittest.TestCase): test_skill_id = 'test_skill.test' skill = None - @classmethod - def init_intent_file(self): - """ - check the intent test yaml for recent changes and keep it up to date - """ - - skill_intents = get_intents_from_skillcode(self.skill) - # those intents most likely are identical, but we want the one used in the code - # (opposed to being present as resource and not being used in the skill) - skill_resources = self.skill.resources.get_inventory() - - # Load the test intent file - yaml_location = getenv("INTENT_TEST_FILE") - with open(yaml_location) as f: - test_yaml = yaml.safe_load(f) - - self.valid_intents = deepcopy(test_yaml) - self.negative_intents = self.valid_intents.pop('unmatched intents', dict()) - self.common_query = self.valid_intents.pop("common query", dict()) - self.regex = set(skill_resources['regex']) - - for intent in skill_intents: - self.intents.add(intent.name) - if intent.service == "adapt": - self.vocab.update({voc.lower() for voc in set(flatten_list(intent.filestems))}) - - assert self.skill.config_core["secondary_langs"] == self.supported_languages - # update yaml file based on the present intents - for lang in self.supported_languages: - test_yaml.setdefault(lang, dict()) - resources = self.skill.load_lang(lang=lang) - for intent in skill_intents: - test_yaml[lang].setdefault(intent.name, list()) - present_intents = test_yaml[lang][intent.name] - valid_intents = [] - # prepare adapt intents - if intent.service == "adapt": - options = [ - [ flatten_list(resources.load_vocabulary_file(voc)) - for voc in vocs ] for vocs in intent.filestems - ] - # filter out intents that don't match the options - for line in present_intents: - if all(any(any(word in line for word in choice) for choice in option) for option in options): - valid_intents.append(line) - # add possible combinations of options - for option in options: - if not any(any(any(word in intent for word in choice) for choice in option) for intent in valid_intents): - valid_intents.extend(generate_sentences(options, 4)) - # prepare padatious intents - elif intent.service == "padatious": - options = resources.load_intent_file(intent.name) - # substitute entities - for i, option in enumerate(options): - options[i] = re.sub(r'\{.*?\}', "test", option) - # filter out intents that don't match the options - for line in present_intents: - if line in options: - valid_intents.append(line) - random.shuffle(options) - if len(valid_intents) < 5: - for option in options: - if option not in valid_intents: - valid_intents.append(option) - - test_yaml[lang][intent.name] = valid_intents - self.valid_intents[lang] = test_yaml[lang] - - with open(yaml_location, "w", encoding='utf8') as f: - yaml.dump(test_yaml, f, allow_unicode=True) @classmethod def setUpClass(cls) -> None: @@ -274,7 +134,18 @@ def _on_message(msg): bus=cls.bus, skill_id=cls.test_skill_id) - cls.init_intent_file() + skill_resources = cls.skill.resources.get_inventory() + + # Load the test intent file + yaml_location = getenv("INTENT_TEST_FILE") + with open(yaml_location) as f: + valid_intents = yaml.safe_load(f) + + cls.negative_intents = valid_intents.pop('unmatched intents', dict()) + cls.common_query = valid_intents.pop("common query", dict()) + cls.regex = set(skill_resources['regex']) + + cls.valid_intents = valid_intents @classmethod def tearDownClass(cls) -> None: From 6e0af6a282434642009fe92cc75c0b670b40af8d Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 15:10:35 +0100 Subject: [PATCH 138/357] test --- .github/workflows/skill_test_resources.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 08c6b80f7..240c7038f 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -30,9 +30,9 @@ on: required: false type: string intent_testfile: - description: 'Intent file to test' + description: 'Intent file to test' + required: true type: string - default: test/unittests/intent_tests.yaml jobs: test_resources: diff --git a/README.md b/README.md index 0b502f6d6..e33152cc3 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ jobs: python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" - intent_file: test/test_intents.yaml # Intent test file to test against, default: test/test_intents.yaml + intent_testfile: test/test_intents.yaml # Intent test file to test against, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true ``` From 365e146223ea6124f1c2e77257a25f0052b74843 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 15:59:14 +0100 Subject: [PATCH 139/357] test --- README.md | 2 +- test/test_skill_resourcesV2.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e33152cc3..83399799f 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ jobs: pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required - skill_location: "skill" # Skill location relative to the root (can usually be omitted) + skill_location: "skill" # Skill location relative to the root (can usually be omitted, used if the skill is not located in the base folder) ``` ## Skill Resource Tests Tests the resources of a skill (e.g dialogs, vocabs, regex or intent resources) for completeness and workability. diff --git a/test/test_skill_resourcesV2.py b/test/test_skill_resourcesV2.py index e3dc050a4..04db2b5c8 100644 --- a/test/test_skill_resourcesV2.py +++ b/test/test_skill_resourcesV2.py @@ -146,6 +146,7 @@ def _on_message(msg): cls.regex = set(skill_resources['regex']) cls.valid_intents = valid_intents + cls.intents = set(valid_intents["en-us"].keys()) @classmethod def tearDownClass(cls) -> None: From a5bb4ee69d5754d8982fa34603af215506b5c9a0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 17:16:56 +0100 Subject: [PATCH 140/357] test --- requirements/version.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements/version.py b/requirements/version.py index 1ef7642f3..1b5a3e6c8 100644 --- a/requirements/version.py +++ b/requirements/version.py @@ -1,7 +1,8 @@ # version file used in all ovos python packages +# this is the first version in the release cycle # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 0 -VERSION_BUILD = 1 +VERSION_MINOR = 1 +VERSION_BUILD = 0 VERSION_ALPHA = 0 # END_VERSION_BLOCK \ No newline at end of file From fe760b1c9967553b344cc017bbf06e396f1ccc54 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 19:56:45 +0100 Subject: [PATCH 141/357] adjust readme --- README.md | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 83399799f..3e3c05167 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,10 @@ jobs: changelog_file: CHANGELOG.md # use if the changlog file has a special name, default: CHANGELOG.md ``` ## Propose and Publish Stable (Build,Minor,Major) Release -Strategy: 2-fold +Strategy: 2-staged *Proposal (prepares bump and pull requests to testing/stable)* +This creates a pull request to the appropriate branch. +On merge, the (shared) publish action is triggered. (no extra repo workflow needed) ```yaml name: Propose Stable Build on: @@ -58,28 +60,6 @@ jobs: release_type: ${{inputs.release_type}} # build, minor, major changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md ``` -*Publishing* -```yaml -name: Publish Stable Build -on: - workflow_dispatch: - inputs: - release_type: - type: choice - options: - - "patch" - - "minor" - - "major" - -jobs: - build_and_publish: - uses: openvoiceos/.github/.github/workflows/publish_stable_release.yml@main - secrets: inherit - with: - python_version: "3.10" - release_type: ${{inputs.release_type}} - changelog_file: ChAnGeLoG.md -``` ## Propose translatios Introduce a new language localisation by proposing a translation via pull request. (creating new branch staging/translation_xx-xx) ```yaml From aef6aecb4ef2aa6a9d2705b2eb99472a3b290013 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 21:04:46 +0100 Subject: [PATCH 142/357] test --- .github/workflows/license_tests.yml | 37 +++++++++++-------- .github/workflows/pytest_file_or_dir.yml | 2 +- .github/workflows/python_build_tests.yml | 26 +++++++++++++ .github/workflows/skill_test_installation.yml | 32 +++++++++------- .github/workflows/skill_test_resources.yml | 33 +++++++++-------- README.md | 28 ++++++++------ 6 files changed, 100 insertions(+), 58 deletions(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index b004b5707..d32c09056 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -14,15 +14,19 @@ on: description: 'The shared action branch to checkout' type: string default: main - python-version: + python_version: description: 'Python version to use' type: string default: '3.8' - system-deps: - description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + system_deps: + description: 'Additional system dependencies (whitespace delimited) to install' required: false type: string - package-extras: + pip_packages: + description: 'Additional Python packages (whitespace delimited) to install' + type: string + required: false + install_extras: description: 'Optional extras to install the python package with' required: false type: string @@ -53,25 +57,26 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} - - name: Install Build Tools + python-version: ${{ inputs.python_version }} + - name: Install Custom System Dependencies + if: ${{ inputs.system_deps != null }} run: | - python -m pip install build wheel - - name: Install System Dependencies + sudo apt update + sudo apt install -y ${{ inputs.system_deps }} + - name: upgrade pip run: | - sudo apt-get update - if [ -z ${{ inputs.system-deps }} ]; then - xargs sudo apt-get install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt-get install ${{ inputs.system-deps }} - fi + pip install --upgrade pip + - name: Install Custom Python Requirements + if: ${{ inputs.pip_packages != null }} + run: | + pip install ${{ inputs.pip_packages }} - name: Install repo run: | cd ${{ github.workspace }}/action/package - if [ -z ${{ inputs.package-extras }} ]; then + if [ -z ${{ inputs.install_extras }} ]; then pip install . else - pip install .[${{ inputs.package-extras }}] + pip install .[${{ inputs.install_extras }}] fi - name: Get explicit and transitive dependencies run: | diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 44237db11..48c39aeac 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -14,7 +14,7 @@ on: type: string default: main system_deps: - description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + description: 'Additional system dependencies (whitespace delimited) to install' required: false type: string python_matrix: diff --git a/.github/workflows/python_build_tests.yml b/.github/workflows/python_build_tests.yml index 1845e1ac2..6c26a6f55 100644 --- a/.github/workflows/python_build_tests.yml +++ b/.github/workflows/python_build_tests.yml @@ -6,6 +6,18 @@ on: description: 'Runner to use' type: string default: ubuntu-latest + branch: + description: 'Branch to check out' + type: string + default: ${{ github.ref }} + system_deps: + description: 'Additional system dependencies (whitespace delimited) to install' + required: false + type: string + pip_packages: + description: 'Additional Python packages (whitespace delimited) to install' + type: string + required: false python_matrix: description: 'Python matrix (string) to use' type: string @@ -42,14 +54,28 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install Custom System Dependencies + if: ${{ inputs.system_deps != null }} + run: | + sudo apt update + sudo apt install -y ${{ inputs.system_deps }} + - name: upgrade pip + run: | + pip install --upgrade pip - name: Install Build Tools run: | pip install --upgrade pip python -m pip install build wheel + - name: Install Custom Python Requirements + if: ${{ inputs.pip_packages != null }} + run: | + pip install ${{ inputs.pip_packages }} - name: Build Distribution Packages run: | python setup.py bdist_wheel sdist diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index 75df8886a..fb310e182 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -19,11 +19,11 @@ on: type: string default: '["3.8", "3.9", "3.10", "3.11"]' system_deps: - description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + description: 'Additional system dependencies (whitespace delimited) to install' required: false type: string pip_packages: - description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + description: 'Additional Python packages (whitespace delimited) to install' required: false type: string skill_id: @@ -58,22 +58,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install System Dependencies + - name: Install Common System Dependencies run: | sudo apt update - if [ -z "${{inputs.system_deps}}" ]; then - xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt install -y ${{inputs.system_deps}} - fi - - name: Install Python Skill and Requirements + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Install Custom System Dependencies + if: ${{ inputs.system_deps != null }} + run: | + sudo apt install -y ${{ inputs.system_deps }} + - name: upgrade pip run: | pip install --upgrade pip - if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - else - pip install ${{inputs.pip_packages}} - fi + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt + - name: Install Custom Python Requirements + if: ${{ inputs.pip_packages != null }} + run: | + pip install ${{ inputs.pip_packages }} + - name: Install Python Skill + run: | cd ${{ github.workspace }}/action/skill pip install . - name: Test Skill Installation diff --git a/.github/workflows/skill_test_resources.yml b/.github/workflows/skill_test_resources.yml index 240c7038f..e74a4639b 100644 --- a/.github/workflows/skill_test_resources.yml +++ b/.github/workflows/skill_test_resources.yml @@ -18,7 +18,7 @@ on: type: number default: 15 system_deps: - description: 'System dependencies (whitespace delimited) to install instead of sys_deb_common_deps.txt' + description: 'Additional system dependencies (whitespace delimited) to install' required: false type: string python_matrix: @@ -26,7 +26,7 @@ on: type: string default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: - description: 'Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + description: 'Additional Python packages (whitespace delimited) to install' required: false type: string intent_testfile: @@ -59,23 +59,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install System Dependencies + - name: Install Common System Dependencies run: | sudo apt update - if [ -z "${{inputs.system_deps}}" ]; then - xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - else - sudo apt install -y ${{inputs.system_deps}} - fi - - name: Install Python Skill and Requirements + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Install Custom System Dependencies + if: ${{ inputs.system_deps != null }} + run: | + sudo apt install -y ${{ inputs.system_deps }} + - name: upgrade pip run: | pip install --upgrade pip - pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 - if [ -z "${{inputs.pip_packages}}" ]; then - pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt - else - pip install ${{inputs.pip_packages}} - fi + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_skill_tests.txt + - name: Install Custom Python Requirements + if: ${{ inputs.pip_packages != null }} + run: | + pip install ${{ inputs.pip_packages }} + - name: Install Python Skill + run: | cd ${{ github.workspace }}/action/skill pip install . PKG_NAME=$(python setup.py --name) diff --git a/README.md b/README.md index 3e3c05167..5f7e6253d 100644 --- a/README.md +++ b/README.md @@ -104,10 +104,10 @@ jobs: with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action - python-version: "3.8" # Python version (quoted) to use, default: 3.8 - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt - package-extras: test # Optional extras to install the python package with + system_deps: "somepkg" # System dependencies (whitespace delimited) to install + pip_packages: "random-pkg" # Python packages (whitespace delimited) to install + python_version: "3.8" # Python version (quoted) to use, default: 3.8 + install_extras: test # Optional extras to install the python package with packages-exclude: '^(fann2|tqdm|bs4).*' # Custom regex to exclude packages from the license check # default: '^(precise-runner|fann2|tqdm|bs4|nvidia|bitstruct).*' licenses-exclude: ^(BSD-3).*$' # Custom regex to exclude licenses from the license check @@ -125,6 +125,10 @@ jobs: uses: openvoiceos/.github/.github/workflows/python_build_tests.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest + branch: dev # Branch to use, default: branch that triggered the action + system_deps: "libfann-dev libfann2" # System dependencies (whitespace delimited) to install + pip_packages: "pytest pytest-cov" # Additional python packages (whitespace delimited) to install + # commonly installed: `build wheel` python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' test_manifest: true # if to test with MANIFEST.in, default: false manifest_ignored: "test/** qt5/**" # Files to ignore in MANIFEST.in, default: "test/**" @@ -147,8 +151,8 @@ jobs: branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main timeout_minutes: 15 # Timeout in minutes for the job, default: 15 - system-deps: "libfann-dev libfann2" # Additional system dependencies to install before running the license check - # commonly installed: `gcc python3-dev swig libssl-dev libfann-dev portaudio19-dev libpulse-dev` + system_deps: "libfann-dev libfann2" # Additional system dependencies to install before running the license check + # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Additional python packages (whitespace delimited) to install # commonly installed: @@ -198,11 +202,11 @@ jobs: runner: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt + system_deps: "libfann-dev libfann2" # Additional system dependencies (whitespace delimited) to install + # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + # [common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required skill_location: "skill" # Skill location relative to the root (can usually be omitted, used if the skill is not located in the base folder) ``` @@ -221,11 +225,11 @@ jobs: timeout: 15 # Timeout for the test, default: 15 branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main - system-deps: "libfann-dev libfann2" # Optional system dependencies to install before running the license check - # default packages found at requirements/sys_deb_common_deps.txt + system_deps: "libfann-dev libfann2" # Additional system dependencies (whitespace delimited) to install + # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # default: "pytest pytest-cov mock ovos-core[skills]>=0.0.7" + # [common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) intent_testfile: test/test_intents.yaml # Intent test file to test against, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true From d7b398a8d551ab2a40392140076dfb9c5aba5926 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 21:20:04 +0100 Subject: [PATCH 143/357] test --- README.md | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5f7e6253d..4a4242400 100644 --- a/README.md +++ b/README.md @@ -151,13 +151,9 @@ jobs: branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main timeout_minutes: 15 # Timeout in minutes for the job, default: 15 - system_deps: "libfann-dev libfann2" # Additional system dependencies to install before running the license check - # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) + system_deps: "libfann-dev libfann2" # (*) Additional system dependencies to install before running the license check python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Additional python packages (whitespace delimited) to install - # commonly installed: - # Skill: see https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt - # Other: see https://github.com/OpenVoiceOS/.github/requirements/pip_tests.txt + pip_packages: "pytest pytest-cov" # (**) Additional python packages (whitespace delimited) to install pip_install_dirs: | # Additional directories to install python packages from relpath/to/package1 relpath/to/package2 @@ -188,6 +184,8 @@ jobs: upload_coverage: true # Whether to upload the coverage to codecov server ... ``` +(*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) +(**) Common python dependencies: [skill](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) / [other](https://github.com/OpenVoiceOS/.github/requirements/pip_tests.txt) ## Skills ## Skill Installation Tests ```yaml @@ -202,14 +200,14 @@ jobs: runner: ubuntu-latest branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main - system_deps: "libfann-dev libfann2" # Additional system dependencies (whitespace delimited) to install - # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) + system_deps: "libfann-dev libfann2" # (*) Additional system dependencies (whitespace delimited) to install python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # [common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) + pip_packages: "pytest pytest-cov" # (**) Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required skill_location: "skill" # Skill location relative to the root (can usually be omitted, used if the skill is not located in the base folder) ``` +(*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) +(**) [Common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) ## Skill Resource Tests Tests the resources of a skill (e.g dialogs, vocabs, regex or intent resources) for completeness and workability. ```yaml @@ -225,15 +223,15 @@ jobs: timeout: 15 # Timeout for the test, default: 15 branch: dev # Branch to use, default: branch that triggered the action action_branch: custom/branch # Shared action branch to use, default: main - system_deps: "libfann-dev libfann2" # Additional system dependencies (whitespace delimited) to install - # [common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) + system_deps: "libfann-dev libfann2" # (*) Additional system dependencies (whitespace delimited) to install python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' - # [common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) + pip_packages: "pytest pytest-cov" # (**) Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' intent_testfile: test/test_intents.yaml # Intent test file to test against, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true ``` +(*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) +(**) [Common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) ## Notify Matrix on Pull Request ```yaml name: Notify Matrix Chat From ad4598c1fe8d3f37ba7236f49cebd6b1c6ae3717 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 21:24:13 +0100 Subject: [PATCH 144/357] test --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4a4242400..ed9e97695 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ jobs: action_branch: custom/branch # Shared action branch to use, default: main system_deps: "libfann-dev libfann2" # (*) Additional system dependencies (whitespace delimited) to install python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # (**) Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + pip_packages: "pytest pytest-cov" # (**) Additional python packages (whitespace delimited) to install' skill_id: "ovos-skill-x.openvoiceos" # Skill id of the testskill, required skill_location: "skill" # Skill location relative to the root (can usually be omitted, used if the skill is not located in the base folder) ``` @@ -225,7 +225,7 @@ jobs: action_branch: custom/branch # Shared action branch to use, default: main system_deps: "libfann-dev libfann2" # (*) Additional system dependencies (whitespace delimited) to install python_matrix: '["3.8", "3.9", "3.10"]' # Python version matrix to use, default: '["3.8", "3.9", "3.10", "3.11"]' - pip_packages: "pytest pytest-cov" # (**) Python packages (whitespace delimited) to install instead of pip_skill_tests.txt' + pip_packages: "pytest pytest-cov" # (**) Additional python packages (whitespace delimited) to install intent_testfile: test/test_intents.yaml # Intent test file to test against, required test_padatious: true # if to test against padatious, default: false test_padacioso: true # if to test against padacioso, default: true From f2f6e819349a54463c237ee1d92e7eeffb2c49eb Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 22:06:58 +0100 Subject: [PATCH 145/357] license checking include pre releases --- .github/workflows/license_tests.yml | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index d32c09056..855516237 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -88,6 +88,7 @@ jobs: requirements: 'requirements-all.txt' fail: 'Copyleft,Other,Error' fails-only: true + pre: true exclude: ${{ inputs.packages-exclude }} exclude-license: ${{ inputs.licenses-exclude }} - name: Print report diff --git a/README.md b/README.md index ed9e97695..be34d2883 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ jobs: ## License testing Tests validity of licenses of all packages (explicit and transitive). +(Note:) ```yaml name: License testing on: From 402c40730d426771e83037cad1f711f4ba344281 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 8 Feb 2024 22:09:49 +0100 Subject: [PATCH 146/357] license checking add verbosity --- .github/workflows/license_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 855516237..0de81531b 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -89,6 +89,7 @@ jobs: fail: 'Copyleft,Other,Error' fails-only: true pre: true + verbose: 2 exclude: ${{ inputs.packages-exclude }} exclude-license: ${{ inputs.licenses-exclude }} - name: Print report From a0e01cd7c4e489be2e0ae00cefa12da4ed82e48d Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 16:37:01 +0100 Subject: [PATCH 147/357] conventional commits --- .github/workflows/auto_translate.yml | 7 +- .../workflows/check_conventional_commits.yml | 27 ++ .github/workflows/notify_testing_matrix.yml | 44 +++ .github/workflows/propose_semver_release.yml | 125 ------- .../propose_semver_release_merged.yml | 32 -- .github/workflows/propose_translation.yml | 4 +- .github/workflows/pull_master.yml | 44 --- .github/workflows/pytest_file_or_dir.yml | 2 +- ...sh_alpha_release.yml => release_alpha.yml} | 78 +---- ...release.yml => release_semver_publish.yml} | 39 ++- .../workflows/release_semver_pull_master.yml | 83 +++++ .github/workflows/release_semver_start.yml | 314 ++++++++++++++++++ .../workflows/semver_release_versioning.yml | 55 +++ .gitignore | 166 +++++++++ README.md | 171 ++++++++-- pyproject.toml | 57 ++++ requirements/pip_base.txt | 1 + requirements/sys_deb_common_deps.txt | 1 + scripts/parse_semver_release.py | 60 ++++ 19 files changed, 987 insertions(+), 323 deletions(-) create mode 100644 .github/workflows/check_conventional_commits.yml create mode 100644 .github/workflows/notify_testing_matrix.yml delete mode 100644 .github/workflows/propose_semver_release.yml delete mode 100644 .github/workflows/propose_semver_release_merged.yml delete mode 100644 .github/workflows/pull_master.yml rename .github/workflows/{publish_alpha_release.yml => release_alpha.yml} (50%) rename .github/workflows/{publish_semver_release.yml => release_semver_publish.yml} (71%) create mode 100644 .github/workflows/release_semver_pull_master.yml create mode 100644 .github/workflows/release_semver_start.yml create mode 100644 .github/workflows/semver_release_versioning.yml create mode 100644 .gitignore create mode 100644 pyproject.toml create mode 100644 scripts/parse_semver_release.py diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml index ecb450b8a..3ee13785a 100644 --- a/.github/workflows/auto_translate.yml +++ b/.github/workflows/auto_translate.yml @@ -26,10 +26,13 @@ jobs: API_KEY: ${{secrets.DL_API_KEY}} LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout Repository uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: ${{ inputs.branch }} path: action/package/ - name: Checkout Scripts Repo @@ -49,7 +52,7 @@ jobs: run: | python action/github/scripts/translate.py - name: Commit to dev - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: - commit_message: autotranslate + commit_message: "fix(localisation): autotranslate common languages" repository: action/package/ diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml new file mode 100644 index 000000000..59e76c4c3 --- /dev/null +++ b/.github/workflows/check_conventional_commits.yml @@ -0,0 +1,27 @@ +name: License Tests +on: + workflow_call: + inputs: + runner: + description: 'Runner to use' + type: string + default: 'ubuntu-latest' + branch: + description: 'Branch to check out' + type: string + default: ${{ github.ref }} + +jobs: + validate-pr-title: + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - name: PR Conventional Commit Validation + uses: ytanikin/PRConventionalCommits@1.1.0 + with: + token: ${{ secrets.GH_PAT }} + task_types: '["feat", "fix", "docs", "test", "ci", "refactor", "perf", "chore", "revert", "translation"]' + custom_labels: '{"feat": "feature", "fix": "fix", "docs": "documentation", "test": "test", "ci": "CI/CD", "refactor": "refactor", "perf": "performance", "chore": "housekeeping", "revert": "revert", "wip": "WIP"}' \ No newline at end of file diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml new file mode 100644 index 000000000..b30025d8a --- /dev/null +++ b/.github/workflows/notify_testing_matrix.yml @@ -0,0 +1,44 @@ +name: Notify Matrix Chat + +# only triggers on pull request closed events +on: + workflow_call: + inputs: + kickoff_pr: + type: boolean + default: true + subject: + type: string + default: 'no subject' + release_type: + type: string + required: true + +jobs: + notify_pr_matrix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Send message to Matrix bots channel + if: ${{ !inputs.kickoff_pr }} + uses: fadenb/matrix-chat-message@v0.0.6 + # this action was triggered by + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + Testing QA phase of a ${{ inputs.release_type }} release started (${{ github.repository }}). + Subject to test: ${{ inputs.subject }} + - name: Send message to Matrix bots channel + if: ${{ inputs.kickoff_pr }} + uses: fadenb/matrix-chat-message@v0.0.6 + # this action was triggered by + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + PR to kick off the testing QA phase of a ${{ inputs.release_type }} release (${{ github.repository }}). + Please merge if it the testing phase should start. + Subject to test: ${{ inputs.subject }} \ No newline at end of file diff --git a/.github/workflows/propose_semver_release.yml b/.github/workflows/propose_semver_release.yml deleted file mode 100644 index 0ddf2288e..000000000 --- a/.github/workflows/propose_semver_release.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Propose SemVer Release -on: - workflow_call: - inputs: - branch: - type: string - default: dev - action_branch: - description: 'The shared action branch to checkout' - type: string - default: main - python_version: - description: 'The python version to use' - type: string - default: "3.8" - version_file: - description: 'The file location to read the version from' - type: string - default: version.py - release_type: - type: string - description: 'The type of release to make' - required: true - changelog_file: - type: string - default: CHANGELOG.md - - -jobs: - bump_version: - runs-on: ubuntu-latest - env: - VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} - TARGET_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} - GITHUB_REPOSITORY: ${{ github.repository }} - RELEASE_TYPE: ${{ inputs.release_type }} - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - path: action/package/ - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - path: action/github/ - ref: ${{ inputs.action_branch }} - repository: OpenVoiceOS/.github - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Python Dependencies - run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: Set up Git Authentication - run: | - cd ${{ github.workspace }}/action/package/ - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - - name: Create Branch (if necessary) - run: | - cd ${{ github.workspace }}/action/package/ - git fetch --unshallow origin ${{ inputs.branch }} - INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) - TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - git checkout $INITIAL_COMMIT - git checkout -b ${{ env.TARGET_BRANCH }} - git push origin ${{ env.TARGET_BRANCH }} --force - git checkout ${{ inputs.branch }} - fi - - name: Increment Version - id: version - run: | - python action/github/scripts/bump_${{ inputs.release_type }}.py - VERSION=$(python action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Increment Version to ${{ env.VERSION }} - repository: action/package/ - - name: Check Latest Release - run: | - SINCE_TAG=$(python action/github/scripts/semver_cycle_start.py) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: Generate Changelog - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.VERSION }} - sinceTag: ${{ env.SINCE_TAG }} - - name: Push Changelog - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update Changelog - repository: action/package/ - - name: Checkout Testing Branch - uses: actions/checkout@v4 - with: - ref: ${{ env.TARGET_BRANCH }} - path: action/package/ - - name: Rebase Latest Changes - run: | - cd ${{ github.workspace }}/action/package/ - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git fetch origin ${{ inputs.branch }} - git rebase FETCH_HEAD - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - path: action/package/ - commit-message: "Propose ${{ inputs.release_type }} release" - title: "Propose ${{ inputs.release_type }} release" - body: | - This pull request proposes a ${{ env.TARGET_BRANCH }} release of the project. - Please review the changelog and merge if it looks good. - branch: staging_${{ inputs.release_type }}_release - base: ${{ env.TARGET_BRANCH }} - draft: false - assignees: ${{ github.actor }} diff --git a/.github/workflows/propose_semver_release_merged.yml b/.github/workflows/propose_semver_release_merged.yml deleted file mode 100644 index c18582797..000000000 --- a/.github/workflows/propose_semver_release_merged.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Rebase dev after SemVer bump - -# only triggers on pull request closed events -on: - workflow_call: - inputs: - release_type: - type: string - description: 'The type of release to make' - required: true - -jobs: - rebase_dev: - env: - SOURCE_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: "dev" - fetch-depth: 0 - - name: Set up Git Authentication - run: | - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} - - name: Rebase testing onto dev - run: | - git fetch origin ${{ env.SOURCE_BRANCH }} - git rebase origin/${{ env.SOURCE_BRANCH }} - - name: Push changes - run: | - git push origin dev \ No newline at end of file diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml index 2f0fcac3d..a7f54f9ee 100644 --- a/.github/workflows/propose_translation.yml +++ b/.github/workflows/propose_translation.yml @@ -60,11 +60,11 @@ jobs: - name: Create Pull Request uses: peter-evans/create-pull-request@v5 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} path: action/package/ commit-message: autotranslate author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - title: Proposed ${{ inputs.language }} Translations + title: "fix(localisation): Add ${{ inputs.language }} language" body: Translations for review labels: translation branch: staging/translation_${{ inputs.language }} diff --git a/.github/workflows/pull_master.yml b/.github/workflows/pull_master.yml deleted file mode 100644 index a0883c2d5..000000000 --- a/.github/workflows/pull_master.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Semver release pull request to master -on: - workflow_call: - inputs: - pr_reviewer: - description: Pull request reviewers, comma-separated list (no spaces) - required: false - type: string - pr_assignee: - description: Pull request assignees, comma-separated list (no spaces) - required: false - type: string - source_branch: - description: Branch name to pull from, default is triggered branch - type: string - default: "${{ github.ref }}" - pr_title: - description: Pull request title - type: string - pr_body: - description: Pull request body - type: string - pr_draft: - description: Draft pull request - type: boolean - default: true - -jobs: - pull_changes: - runs-on: "ubuntu-latest" - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - name: pull request to master - uses: repo-sync/pull-request@v2 - # "master" is the `destination_branch` default value - with: - source_branch: ${{ inputs.source_branch }} - github_token: ${{ secrets.GITHUB_TOKEN }} - pr_reviewer: ${{ inputs.pr_reviewer }} - pr_assignee: ${{ inputs.pr_assignee }} - pr_draft: ${{ inputs.pr_draft }} - pr_title: ${{ inputs.pr_title }} - pr_body: ${{ inputs.pr_body }} \ No newline at end of file diff --git a/.github/workflows/pytest_file_or_dir.yml b/.github/workflows/pytest_file_or_dir.yml index 48c39aeac..405e92699 100644 --- a/.github/workflows/pytest_file_or_dir.yml +++ b/.github/workflows/pytest_file_or_dir.yml @@ -141,7 +141,7 @@ jobs: with: name: coverage_${{ github.repository_id }} path: ${{ github.workspace }}/action/package/ - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GH_PAT }} - name: Test package with ${{ matrix.python-version }} run: | cd ${{ github.workspace }}/action/package/ diff --git a/.github/workflows/publish_alpha_release.yml b/.github/workflows/release_alpha.yml similarity index 50% rename from .github/workflows/publish_alpha_release.yml rename to .github/workflows/release_alpha.yml index 8c9acdd69..744bafaa1 100644 --- a/.github/workflows/publish_alpha_release.yml +++ b/.github/workflows/release_alpha.yml @@ -20,83 +20,15 @@ on: description: 'The python version to use' type: string default: "3.8" - locale_folder: - description: 'The base folder to look for localization files' - type: string - default: locale - update_intentfile: - description: 'The file to update with the resource changes. This is atm only used for skills.' - type: string - required: false changelog_file: type: string default: CHANGELOG.md jobs: - autotranslate: - runs-on: ubuntu-latest - env: - API_KEY: ${{secrets.DL_API_KEY}} - LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.ref }} - path: action/package/ - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} - path: action/github/ - - uses: dorny/paths-filter@v2 - id: filter - with: - working-directory: action/package/ - filters: | - us_specific: - - '${{ inputs.locale_folder }}/en-us/**' - - '${{ inputs.locale_folder }}/../dialog/en-us/**' - - '${{ inputs.locale_folder }}/../vocab/en-us/**' - general: - - '${{ inputs.locale_folder }}/**' - - '${{ inputs.locale_folder }}/../dialog/**' - - '${{ inputs.locale_folder }}/../vocab/**' - - name: Setup Python - if: steps.filter.outputs.us_specific == 'true' - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Python Requirements - if: steps.filter.outputs.us_specific == 'true' - run: | - python -m pip install -r action/github/requirements/pip_translation.txt - - name: Pull latest changes - if: steps.filter.outputs.us_specific == 'true' - run: | - git pull origin ${{ inputs.branch }} - - name: Auto Translate - if: steps.filter.outputs.us_specific == 'true' - run: | - python action/github/scripts/translate.py - - name: Commit to ${{ inputs.branch }} - if: steps.filter.outputs.us_specific == 'true' - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: autotranslate - repository: action/package/ - branch: ${{ inputs.branch }} - - name: update and commit resource_changes - if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }} - run: | - python action/github/scripts/update_intent_testfile.py - git add ${{ inputs.update_intentfile }} - git commit -m "Update resource test file" - build_and_publish: runs-on: ubuntu-latest + permissions: + contents: write env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} steps: @@ -125,10 +57,10 @@ jobs: id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} output: action/package/${{ inputs.changelog_file }} - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Increment Version to ${{ env.VERSION }} repository: action/package/ @@ -138,7 +70,7 @@ jobs: id: create_release uses: ncipollo/release-action@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} tag: V${{ env.VERSION }} name: Release ${{ env.VERSION }} body: | diff --git a/.github/workflows/publish_semver_release.yml b/.github/workflows/release_semver_publish.yml similarity index 71% rename from .github/workflows/publish_semver_release.yml rename to .github/workflows/release_semver_publish.yml index 51218e38c..c18fa3598 100644 --- a/.github/workflows/publish_semver_release.yml +++ b/.github/workflows/release_semver_publish.yml @@ -9,10 +9,6 @@ on: description: 'The shared action branch to checkout' type: string default: main - release_type: - type: string - description: 'The type of release to make' - required: true python_version: description: "Python version to use for the build process." type: string @@ -21,15 +17,22 @@ on: description: "File to write the changelog to." type: string default: CHANGELOG.md + subject: + description: 'Subject title of the push/pull-request event to parse the release type.' + type: string + required: true jobs: build_and_publish: env: - SOURCE_BRANCH: ${{ inputs.release_type == 'patch' && 'testing' || 'stable' }} + SOURCE_BRANCH: 'master' GITHUB_REPOSITORY: ${{ github.repository }} - RELEASE_TYPE: ${{ inputs.release_type }} runs-on: ubuntu-latest steps: + - name: set up release type + run: | + RELEASE_TYPE=$(echo ${{ inputs.subject }} | grep -oP '(patch|minor|major)') + echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Checkout Repository uses: actions/checkout@v4 with: @@ -57,18 +60,34 @@ jobs: run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: Generate Changelog - id: changelog + - name: Bump Changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} output: action/package/${{ inputs.changelog_file }} + - name: create release changelog (cropped) + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} sinceTag: ${{ env.SINCE_TAG }} + # pull changes in the release process back to dev + - name: Create Pull Request to dev + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GH_PAT }} + path: action/package/ + commit-message: "ci: merge testing commits to dev" + title: "ci: merge testing commits (${{ env.VERSION }}) to dev" + branch: dev + base: testing + draft: false + assignees: ${{ github.actor }} - name: Create Release id: create_release uses: ncipollo/release-action@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} tag: V${{ env.VERSION }} name: Release ${{ env.VERSION }} body: | diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml new file mode 100644 index 000000000..cbe50438d --- /dev/null +++ b/.github/workflows/release_semver_pull_master.yml @@ -0,0 +1,83 @@ +name: Declare Stable Release (Proposal) +on: + workflow_call: + inputs: + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main + python_version: + description: 'The python version to use' + type: string + default: "3.8" + changelog_file: + type: string + description: 'The file to write the changelog to' + default: CHANGELOG.md + +jobs: + propose_stable_release: + runs-on: ubuntu-latest + env: + TARGET_BRANCH: 'master' + GITHUB_REPOSITORY: ${{ github.repository }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: testing + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Install Common System Dependencies + run: | + sudo apt update + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python_version}} + - name: Install Python Dependencies + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Get Version + run: | + VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Parse Release Type + run: | + RELEASE_TYPE=$(python ${{ github.workspace }}/action/github/scripts/parse_semver_release.py) + echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV + - name: Get Start of the Version Cycle + run: | + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: Bump Changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: action/package/${{ inputs.changelog_file }} + - name: create release changelog (cropped) + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + sinceTag: ${{ env.SINCE_TAG }} + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GH_PAT }} + path: action/package/ + commit-message: "ci(release): declare ${{ env.VERSION }} stable" + title: "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.VERSION }})" + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + branch: ${{ env.TARGET_BRANCH }} + base: testing + draft: false + assignees: ${{ github.actor }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml new file mode 100644 index 000000000..e2b820876 --- /dev/null +++ b/.github/workflows/release_semver_start.yml @@ -0,0 +1,314 @@ +name: Kickoff Testing Release +on: + workflow_call: + inputs: + branch: + type: string + default: dev + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main + python_version: + description: 'The python version to use' + type: string + default: "3.8" + version_file: + description: 'The file location to read the version from' + type: string + default: version.py + release_type: + type: string + description: 'The type of release to make' + subject: + type: string + description: 'Subject title of the push/pull request event to parse the release type.' + required: false + kickoff_pr: + type: boolean + description: 'Whether to create a PR to mark the start of the testing phase or not' + default: false + locale_folder: + type: string + description: 'The folder location of the locale files' + required: false + update_intentfile: + type: string + description: 'The file location of the intent test file to update' + required: false + +jobs: + parse_conventional_commits: + outputs: + release_type: ${{ steps.parse.outputs.release_type }} + runs-on: ubuntu-latest + steps: + - name: Checkout Package Repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + fetch-depth: 0 + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ inputs.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Common System Dependencies + run: | + sudo apt update + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Parse Conventional Commits + id: parse + run: | + export TITLE=${{ inputs.subject }} + cd ${{ github.workspace }}/action/github/scripts + RELEASE_TYPE=$(python scripts/parse_semver_release.py) + echo "release_type=$RELEASE_TYPE" >> $GITHUB_OUTPUT + autotranslate: + if: ${{ inputs.locale_folder != null }} + runs-on: ubuntu-latest + permissions: + contents: write + env: + API_KEY: ${{secrets.DL_API_KEY}} + LOCALE_FOLDER: ${{ github.workspace }}/action/package/${{ inputs.locale_folder }} + INTENT_TEST_FILE: ${{ github.workspace }}/action/package/${{ inputs.update_intentfile }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.ref }} + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - uses: dorny/paths-filter@v2 + id: filter + with: + working-directory: action/package/ + filters: | + us_specific: + - '${{ inputs.locale_folder }}/en-us/**' + - '${{ inputs.locale_folder }}/../dialog/en-us/**' + - '${{ inputs.locale_folder }}/../vocab/en-us/**' + general: + - '${{ inputs.locale_folder }}/**' + - '${{ inputs.locale_folder }}/../dialog/**' + - '${{ inputs.locale_folder }}/../vocab/**' + - name: Setup Python + if: steps.filter.outputs.us_specific == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Python Requirements + if: steps.filter.outputs.us_specific == 'true' + run: | + python -m pip install -r action/github/requirements/pip_translation.txt + - name: Pull latest changes + if: steps.filter.outputs.us_specific == 'true' + run: | + git pull origin ${{ inputs.branch }} + - name: Auto Translate + if: steps.filter.outputs.us_specific == 'true' + run: | + python action/github/scripts/translate.py + - name: Commit autotranslation to ${{ inputs.branch }} + if: steps.filter.outputs.us_specific == 'true' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "ci(localisation): autotranslate common languages" + repository: action/package/ + branch: ${{ inputs.branch }} + - name: update resource test file + if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }} + run: | + python action/github/scripts/update_intent_testfile.py + - name: Commit resource test file changes to ${{ inputs.branch }} + if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }}' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "ci(localisation): update resource test file" + repository: action/package/ + branch: ${{ inputs.branch }} + # alpha release + alpha_release: + needs: parse_conventional_commits + if: ${{ needs.parse_conventional_commits.outputs.release_type == 'alpha' || inputs.release_type == 'alpha' }} + uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 + secrets: inherit + with: + branch: ${{ inputs.branch }} + action_branch: ${{ inputs.action_branch }} + version_file: ${{ inputs.version_file }} + python_version: ${{ inputs.python_version }} + locale_folder: ${{ inputs.locale_folder }} + # >= patch release (auto) + kickoff_testing_auto: + needs: parse_conventional_commits + if: ${{ needs.parse_conventional_commits.outputs.release_type != 'alpha' && needs.parse_conventional_commits.outputs.release_type != 'No semver release.'}} + runs-on: ubuntu-latest + env: + TARGET_BRANCH: 'testing' + GITHUB_REPOSITORY: ${{ github.repository }} + RELEASE_TYPE: ${{ needs.parse_conventional_commits.outputs.release_type }} + steps: + - name: Checkout Package Repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + fetch-depth: 0 + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ inputs.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Check Latest Release + if: ${{ inputs.kickoff_pr }} + run: | + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: create changelog + id: changelog + if: ${{ inputs.kickoff_pr }} + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + sinceTag: ${{ env.SINCE_TAG }} + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + if: ${{ inputs.kickoff_pr }} + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GH_PAT }} + path: action/package/ + commit-message: "ci(release): kickoff testing QA phase" + title: "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" + body: | + This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. + Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. + + (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) + + Included changes: + ${{ steps.changelog.outputs.changelog }} + branch: ${{ env.TARGET_BRANCH }} + base: ${{ inputs.branch }} + draft: false + assignees: ${{ github.actor }} + - name: Push to testing branch + if: ${{ !inputs.kickoff_pr }} + run: | + cd ${{ github.workspace }}/action/package/ + git checkout ${{ env.TARGET_BRANCH }} + git pull origin ${{ env.TARGET_BRANCH }} + git rebase ${{ inputs.branch }} + git push origin ${{ env.TARGET_BRANCH }} + - name: Bump Version + if: ${{ !inputs.kickoff_pr }} + uses: ./action/github/.github/workflows/release_semver_versioning.yml + with: + release_type: ${{ env.RELEASE_TYPE }} + version_file: ${{ inputs.version_file }} + - name: Notify Matrix Room + uses: ./action/github/.github/workflows/notify_testing_matrix.yml + with: + kickoff_pr: ${{ !inputs.kickoff_pr }} + subject: ${{ inputs.subject }} + release_type: ${{ env.RELEASE_TYPE }} + # >= patch release (manual) + kickoff_testing_manual: + if: ${{ inputs.release_type != 'alpha' && inputs.release_type != null }} + runs-on: ubuntu-latest + env: + TARGET_BRANCH: 'testing' + GITHUB_REPOSITORY: ${{ github.repository }} + RELEASE_TYPE: ${{ inputs.release_type }} + steps: + - name: Checkout Package Repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + path: action/package/ + fetch-depth: 0 + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ inputs.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Check Latest Release + if: ${{ inputs.kickoff_pr }} + run: | + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: create changelog + id: changelog + if: ${{ inputs.kickoff_pr }} + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + sinceTag: ${{ env.SINCE_TAG }} + - name: Create Pull Request to ${{ env.TARGET_BRANCH }} + uses: peter-evans/create-pull-request@v5 + if: ${{ inputs.kickoff_pr }} + with: + token: ${{ secrets.GH_PAT }} + path: action/package/ + commit-message: "ci: kickoff testing QA phase" + title: "ci: kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" + body: | + This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. + Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. + + (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) + branch: ${{ env.TARGET_BRANCH }} + base: ${{ inputs.branch }} + draft: false + assignees: ${{ github.actor }} + - name: Push to testing branch + if: ${{ !inputs.kickoff_pr }} + run: | + cd ${{ github.workspace }}/action/package/ + git checkout ${{ env.TARGET_BRANCH }} + git pull origin ${{ env.TARGET_BRANCH }} + git rebase ${{ inputs.branch }} + git push origin ${{ env.TARGET_BRANCH }} + - name: Bump Version + if: ${{ !inputs.kickoff_pr }} + uses: ./action/github/.github/workflows/release_semver_versioning.yml + with: + release_type: ${{ env.RELEASE_TYPE }} + version_file: ${{ inputs.version_file }} + - name: Notify Matrix Room + uses: ./action/github/.github/workflows/notify_testing_matrix.yml + with: + kickoff_pr: ${{ !inputs.kickoff_pr }} + release_type: ${{ env.RELEASE_TYPE }} diff --git a/.github/workflows/semver_release_versioning.yml b/.github/workflows/semver_release_versioning.yml new file mode 100644 index 000000000..7b2cad80c --- /dev/null +++ b/.github/workflows/semver_release_versioning.yml @@ -0,0 +1,55 @@ +name: Bump Version (after testing QA started) + +# only triggers on pull request closed events +on: + workflow_call: + inputs: + version_file: + description: 'The file location to read the version from' + type: string + default: version.py + subject: + type: string + description: 'Subject title of the push/pull-request event to parse the release type.' + required: true + +jobs: + start_testing_phase: + env: + VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} + SOURCE_BRANCH: 'testing' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: set up release type + run: | + RELEASE_TYPE=$(echo ${{ inputs.subject }} | grep -oP '(patch|minor|major)') + echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV + - name: Checkout + uses: actions/checkout@v4 + with: + ref: dev + fetch-depth: 0 + - name: Increment Version + id: version + run: | + python action/github/scripts/bump_${{ env.RELEASE_TYPE }}.py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Increment Version to ${{ env.VERSION }} + repository: action/package/ + # fetch the latest changes from the dev branch + - name: Rebase testing onto dev + run: | + git checkout ${{ env.SOURCE_BRANCH }} + git rebase dev + git push origin ${{ env.SOURCE_BRANCH }} --force + - name: Notify Matrix Room + uses: ./action/github/.github/workflows/notify_testing_matrix.yml + with: + kickoff_pr: false + release_type: ${{ inputs.RELEASE_TYPE }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..92961755c --- /dev/null +++ b/.gitignore @@ -0,0 +1,166 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# VS Code +# This ignores the .vscode/ to be ignored in the case where it's in the root of your repository +# which is usually not the case when it's inside .config/ which should be added to your .gitignore +# as well +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index be34d2883..8ff17206a 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,168 @@ +## Conventional Commits +We _slowly_ adopt [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) to our repositories. +Goal is to have a consistent commit message format that can be used to streamline cd/ci or simply help understand commits better for every party involved. +The usage is not mandatory atm, but we encourage to use it. (maintainers will adjust the commit messages if necessary) + +The usage of `types` will automate the versioning and labelling of the pull requests. + - `fix`: patches a bug in your codebase. This correlates with _**PATCH**_ in Semantic Versioning. + - `feat`: introduces a new feature to the codebase. This correlates with _**MINOR**_ in Semantic Versioning. + - BREAKING CHANGE: A commit that has a _footer_ `BREAKING CHANGE:`, or _appends a `!`_ after the type/scope, introduces a breaking API change. This correlates with _**MAJOR**_ in Semantic Versioning. A BREAKING CHANGE can be part of commits of any type. + + Example (usage of _!_ and _footer_): + ``` + chore!: drop support for Node 6 + + BREAKING CHANGE: use JavaScript features not available in Node 6. + ``` + Other types that create a alpha release - if not breaking: + - `build`: Changes that affect the build system or external dependencies. + - `ci`: Changes to CI configuration files and scripts. + - `chore`: Changes which don’t change source code or tests e.g. changes to the build process, auxiliary tools, libraries. + - `docs`: Documentation only changes. + - `perf`: A code change that improves performance. + - `refactor`: A code change that neither fixes a bug nor adds a feature. + - `revert`: Revert something. + - `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc). + - `test`: Adding missing tests or correcting existing tests. + + Scopes: (context of the change) + - `config`: Changes the configuration + - `parser`: Changes to the intent parser code, + - ... + - `cli`: Changes/additions to the command line interfaces, + - `gui`: Changes/additions to the graphical user interface, + - `audio`: Changes to the audio handling, + - `tts`: Changes to the text-to-speech handling, + - `sst`: Changes to the speech-to-text handling, + - `nlp`: Changes to the natural language processing, + - `plugins`: Changes to the plugin system, + - `localisation`: Changes to the localisation files, + - `resources`: Changes to the resource files, + - `release`: Everything related to the release process + + +## Premises + - The `main` branch is the stable branch. + - The `dev` branch is the development branch. + - The `testing` branch is a persistent branch for testing purposes. + + - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. + - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. + + TODO (per repo): + - `setup.py`: To fully adopt semver, the version number created by the setup has to be compliant: `x.x.x-alpha` (former alpha versions can't be used anymore) + - add `main` branch protection rules + ## Workflows/Actions **You can also find the implementation of the workflows in the [`skill-template-repo`](https://github.com/OpenVoiceOS/skill-template-repo)** -### Publish Alpha Release (example: `ovos-core`) +## Release Handling (alpha/patch/minor/major versions) +_Alpha releases are directly published without going through a test phase_ + +Strategy: 3-staged + - Manually propose a testing start or automatically with setting "Conventional Commits" (optionally as PR) + - Manually conclude testing phase, propose a stable release (PR) + - Publishing of a stable release (on merge) and feed back the changes to the dev branch + +_Start release mechanism_ ```yaml -name: Publish Alpha Build ...aX +name: Start release mechanism on: + workflow_dispatch: + inputs: + release_type: + type: choice + options: + - "alpha" + - "patch" + - "minor" + - "major" + # Make SURE that sqashed PRs do have the PRs title as commit message !!!!!!!! push: branches: - dev paths-ignore: - - 'ovos_core/version.py' + - 'ovos_testpkg/version.py' - 'test/**' - 'examples/**' - '.github/**' - '.gitignore' - - 'LICENSE' - 'CHANGELOG.md' - 'MANIFEST.in' - - 'readme.md' - 'scripts/**' - workflow_dispatch: jobs: - build_and_publish: - uses: openvoiceos/.github/.github/workflows/publish_alpha_release.yml@main + start_semver_release_mechanism: + uses: openvoiceos/.github/.github/workflows/release_semver_start.yml@feat/shared_actions1 + with: + branch: dev # Branch to use, default: branch that triggered the action + action_branch: feat/shared_actions1 # Shared action branch to use, default: main + python_version: "3.10" # the python version to use + version_file: "ovos_testpkg/version.py" # the file containing the version number + locale_folder: ovos_testpkg/locale # the location of the base localisation folder + update_intentfile: test/unittests/test_intent.yaml # the intent file resources gets tested against + release_type: ${{ inputs.release_type || null }} # if manually triggered, set a release type + subject: ${{ github.event.head_commit.message || null }} # on push, the commit message is used as release subject + kickoff_pr: true # if the release process should be started with a PR to summarize and visualize, default: false +``` +_Start testing phase_ +_optional: if a kickoff PR is opened, for versioning and synchronization purposes_ +_(if no PR is used, the testing phase starts right away)_ +```yaml +name: Start Testing Phase +# only trigger on pull request closed events +on: + pull_request: + types: [ closed ] + branches: + - testing + +jobs: + start_testing_phase: + if: (github.event.pull_request.merged == true) && (contains(github.event.pull_request.title, 'patch release') || contains(github.event.pull_request.title, 'minor release') || contains(github.event.pull_request.title, 'major release')) + uses: openvoiceos/.github/.github/workflows/release_semver_versioning.yml@feat/shared_actions1 secrets: inherit with: - branch: dev # Branch to use, default: branch that triggered the action - version_file: ovos_core/version.py # File location of the version file, default: version.py - python_version: "3.8" # Python version (quoted) to use, default: 3.8 - locale_folder: ovos_core/locale # use if there are localisation files the location of the base folder, default: locale - update_intentfile: test/test_intents.yaml # use if there are changes to resource files, the test file to update. (ONLY USED IN SKILLS) - changelog_file: CHANGELOG.md # use if the changlog file has a special name, default: CHANGELOG.md + version_file: ovos_testpkg/version.py # the file containing the version number + subject: ${{ github.event.pull_request.title }} ``` -## Propose and Publish Stable (Build,Minor,Major) Release -Strategy: 2-staged -*Proposal (prepares bump and pull requests to testing/stable)* -This creates a pull request to the appropriate branch. -On merge, the (shared) publish action is triggered. (no extra repo workflow needed) +_Conclude testing phase_ +_After the testing phase, a PR is opened to propose the stable release_ ```yaml -name: Propose Stable Build +name: Conclude testing phase on: workflow_dispatch: - inputs: - release_type: - type: choice - options: - - "patch" - - "minor" - - "major" jobs: - build_and_publish: - uses: openvoiceos/.github/.github/workflows/propose_semver_release.yml@main + pull_to_master: + uses: openvoiceos/.github/.github/workflows/release_semver_pull_master.yml@feat/shared_actions1 + secrets: inherit with: - branch: dev # Branch to use, default: branch that triggered the action - python_version: "3.10" # Python version (quoted) to use, default: 3.8 - version_file: ovos_core/version.py # File location of the version file, default: version.py - release_type: ${{inputs.release_type}} # build, minor, major - changelog_file: ChAnGeLoG.md # if the changlog file has a special name, default: CHANGELOG.md + action_branch: shared_actions1 # Shared action branch to use, default: main + python_version: "3.10" # the python version to use ``` +_Publishing stable release_ +```yaml +name: Publish Stable Release + +on: + pull_request: + types: [ closed ] + branches: + - master + +jobs: + publish_stable_release: + if: (github.event.pull_request.merged == true) && (contains(github.event.pull_request.title, 'patch release stable') || contains(github.event.pull_request.title, 'minor release stable') || contains(github.event.pull_request.title, 'major release stable')) + uses: openvoiceos/.github/.github/workflows/release_semver_publish.yml@feat/shared_actions1 + secrets: inherit + with: + action_branch: feat/shared_actions1 + python_version: "3.10" + subject: ${{ github.event.pull_request.title }} +``` +----------------- + ## Propose translatios Introduce a new language localisation by proposing a translation via pull request. (creating new branch staging/translation_xx-xx) ```yaml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..6264b07bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +# configuration keys for the Python Conventional Commit Checker +[tool] + +[tool.pccc] + +header_length = 50 +body_length = 72 +wrap = true +force_wrap = true +spell_check = false +repair = false +ignore_generated_commits = true + +generated_commits = [ + '''^\(tag:\s+v\d+\.\d+\.\d\)\s+\d+\.\d+\.\d+$''', + '''^Merge branch 'master'.*$''', +] + +# not included "ci", "docs", "style", "test" as they dont get a release +# +# chore -> alpha +# fix -> patch +# feat -> minor +# in combination with "!" ("feat!:") or BREAKING CHANGE (in body/message) -> major + +types = [ + "chore", + "build", + "depends", + "feat", + "fix", + "perf", + "refactor", + "revert" +] + +scopes = [ + "config", + "parser", + "tooling", + "cli", + "gui", + "core", + "audio", + "tts", + "sst", + "nlp", + "plugins", + "localisation", + "resources", + "release" +] + +footers = [ + "github-closes", + "signed-off-by", +] \ No newline at end of file diff --git a/requirements/pip_base.txt b/requirements/pip_base.txt index 9bbbfc51b..d864019bc 100644 --- a/requirements/pip_base.txt +++ b/requirements/pip_base.txt @@ -1,2 +1,3 @@ PyGithub semver +pccc diff --git a/requirements/sys_deb_common_deps.txt b/requirements/sys_deb_common_deps.txt index 24c07e0a9..5d168bf04 100644 --- a/requirements/sys_deb_common_deps.txt +++ b/requirements/sys_deb_common_deps.txt @@ -7,3 +7,4 @@ portaudio19-dev libpulse-dev libicu-dev pkg-config +libenchant-2-2 diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py new file mode 100644 index 000000000..734db5712 --- /dev/null +++ b/scripts/parse_semver_release.py @@ -0,0 +1,60 @@ +import pccc +import semver +from os import environ + +""" +translates a conventional commit title/message into a semver version +""" + +TITLE = environ.get("TITLE") +BODY = environ.get("BODY") +VERSION = environ.get("VERSION") + +def semver_from_cc(): + ccr = pccc.ConventionalCommitRunner() + ccr.options.load() + ccr.raw = f"{TITLE}\n{BODY}" + ccr.clean() + try: + ccr.parse() + # no spec compliant format + except Exception: + print("No semver release.") + exit(0) + + if ccr.breaking.get("flag") or ccr.breaking.get("token"): + return "major" + elif ccr.header.get("type") == "feat": + return "minor" + elif ccr.header.get("type") == "fix": + return "patch" + else: + return "alpha" + +def semver_from_version(): + try: + version = semver.VersionInfo.parse(VERSION) + except ValueError: + print("No semver release.") + exit(0) + + if version.prerelease: + return "alpha" + elif version.patch != 0: + return "patch" + elif version.minor != 0: + return "minor" + elif version.major != 0: + return "major" + + + +if VERSION: + release = semver_from_version() +elif TITLE: + release = semver_from_cc() +else: + print("No semver release.") + exit(0) + +print(release) \ No newline at end of file From 8b243f89404a4c60c8f8f9eb1984c126b6310c67 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 16:45:56 +0100 Subject: [PATCH 148/357] readme adjustments --- README.md | 8 +++++--- pyproject.toml | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8ff17206a..c067d3d30 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,18 @@ The usage of `types` will automate the versioning and labelling of the pull requ BREAKING CHANGE: use JavaScript features not available in Node 6. ``` - Other types that create a alpha release - if not breaking: + Other _types_ that create a alpha release - if not breaking: - `build`: Changes that affect the build system or external dependencies. - - `ci`: Changes to CI configuration files and scripts. - `chore`: Changes which don’t change source code or tests e.g. changes to the build process, auxiliary tools, libraries. - - `docs`: Documentation only changes. - `perf`: A code change that improves performance. - `refactor`: A code change that neither fixes a bug nor adds a feature. - `revert`: Revert something. + + _Types_ that don't get a release: + - `ci`: Changes to CI configuration files and scripts. - `style`: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc). - `test`: Adding missing tests or correcting existing tests. + - `docs`: Documentation only changes. Scopes: (context of the change) - `config`: Changes the configuration diff --git a/pyproject.toml b/pyproject.toml index 6264b07bb..4560d2911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ generated_commits = [ types = [ "chore", "build", - "depends", "feat", "fix", "perf", From 4236154bcaacfc407af6277ea8c51fb80e7f65c1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 16:48:09 +0100 Subject: [PATCH 149/357] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c067d3d30..9f237a706 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The usage of `types` will automate the versioning and labelling of the pull requ - `test`: Adding missing tests or correcting existing tests. - `docs`: Documentation only changes. - Scopes: (context of the change) + _Scopes_: (context of the change; eg. `fix(parser): ...`) - `config`: Changes the configuration - `parser`: Changes to the intent parser code, - ... From d04b55449b88c6ece08f519404ff8aa47f8b8760 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 17:03:32 +0100 Subject: [PATCH 150/357] readme edits --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9f237a706..d4351103d 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,12 @@ The usage of `types` will automate the versioning and labelling of the pull requ ## Premises - - The `main` branch is the stable branch. - - The `dev` branch is the development branch. - - The `testing` branch is a persistent branch for testing purposes. - - - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. + - The `main` branch is the stable branch. + - The `dev` branch is the development branch. + - The `testing` branch is a persistent branch for testing purposes. +--------- + - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. + - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. TODO (per repo): - `setup.py`: To fully adopt semver, the version number created by the setup has to be compliant: `x.x.x-alpha` (former alpha versions can't be used anymore) @@ -66,7 +66,8 @@ Strategy: 3-staged - Manually conclude testing phase, propose a stable release (PR) - Publishing of a stable release (on merge) and feed back the changes to the dev branch -_Start release mechanism_ +_(manual/auto)_ **Start release mechanism** + ```yaml name: Start release mechanism on: @@ -107,8 +108,9 @@ jobs: subject: ${{ github.event.head_commit.message || null }} # on push, the commit message is used as release subject kickoff_pr: true # if the release process should be started with a PR to summarize and visualize, default: false ``` -_Start testing phase_ -_optional: if a kickoff PR is opened, for versioning and synchronization purposes_ +_(auto/optional)_ **Start testing phase** + +_If a kickoff PR is opened, for versioning and synchronization purposes_ _(if no PR is used, the testing phase starts right away)_ ```yaml name: Start Testing Phase @@ -128,7 +130,8 @@ jobs: version_file: ovos_testpkg/version.py # the file containing the version number subject: ${{ github.event.pull_request.title }} ``` -_Conclude testing phase_ +_(manual)_ **Conclude testing phase** + _After the testing phase, a PR is opened to propose the stable release_ ```yaml name: Conclude testing phase @@ -143,7 +146,8 @@ jobs: action_branch: shared_actions1 # Shared action branch to use, default: main python_version: "3.10" # the python version to use ``` -_Publishing stable release_ +_(auto)_ **Publishing stable release** + ```yaml name: Publish Stable Release From 3e4c2b8f45f4289aec0b869a691e45e0bbfe0228 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 17:21:39 +0100 Subject: [PATCH 151/357] readme edits --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d4351103d..671bded1c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ ## Conventional Commits We _slowly_ adopt [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) to our repositories. -Goal is to have a consistent commit message format that can be used to streamline cd/ci or simply help understand commits better for every party involved. -The usage is not mandatory atm, but we encourage to use it. (maintainers will adjust the commit messages if necessary) +Goal is to have a consistent commit message format that can be used to + - _streamline cd/ci_ or simply + - _help understand commits better_ for every party involved. -The usage of `types` will automate the versioning and labelling of the pull requests. +The usage is not mandatory atm, but we encourage to use it. (maintainers will adjust the commit messages if necessary) + +------------ +_Types_: + +Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling of the pull requests. - `fix`: patches a bug in your codebase. This correlates with _**PATCH**_ in Semantic Versioning. - `feat`: introduces a new feature to the codebase. This correlates with _**MINOR**_ in Semantic Versioning. - BREAKING CHANGE: A commit that has a _footer_ `BREAKING CHANGE:`, or _appends a `!`_ after the type/scope, introduces a breaking API change. This correlates with _**MAJOR**_ in Semantic Versioning. A BREAKING CHANGE can be part of commits of any type. @@ -27,7 +33,10 @@ The usage of `types` will automate the versioning and labelling of the pull requ - `test`: Adding missing tests or correcting existing tests. - `docs`: Documentation only changes. - _Scopes_: (context of the change; eg. `fix(parser): ...`) +--------------- + _Scopes_: + (context of the change; eg. `fix(parser): ...`) + - `config`: Changes the configuration - `parser`: Changes to the intent parser code, - ... @@ -42,7 +51,7 @@ The usage of `types` will automate the versioning and labelling of the pull requ - `resources`: Changes to the resource files, - `release`: Everything related to the release process - +-------------- ## Premises - The `main` branch is the stable branch. - The `dev` branch is the development branch. @@ -55,6 +64,7 @@ The usage of `types` will automate the versioning and labelling of the pull requ - `setup.py`: To fully adopt semver, the version number created by the setup has to be compliant: `x.x.x-alpha` (former alpha versions can't be used anymore) - add `main` branch protection rules +------------ ## Workflows/Actions **You can also find the implementation of the workflows in the [`skill-template-repo`](https://github.com/OpenVoiceOS/skill-template-repo)** From 09dc5b4569ebccc0b00055bd8302af353d30104d Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 20 Feb 2024 19:14:37 +0100 Subject: [PATCH 152/357] tests --- .github/workflows/check_conventional_commits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 59e76c4c3..e70d41563 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -23,5 +23,5 @@ jobs: uses: ytanikin/PRConventionalCommits@1.1.0 with: token: ${{ secrets.GH_PAT }} - task_types: '["feat", "fix", "docs", "test", "ci", "refactor", "perf", "chore", "revert", "translation"]' + task_types: '["feat", "fix", "docs", "test", "ci", "refactor", "perf", "build", "chore", "revert"]' custom_labels: '{"feat": "feature", "fix": "fix", "docs": "documentation", "test": "test", "ci": "CI/CD", "refactor": "refactor", "perf": "performance", "chore": "housekeeping", "revert": "revert", "wip": "WIP"}' \ No newline at end of file From 0398bb61d93b2f200219f1b370708b7ee920cbb7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 16:14:30 +0100 Subject: [PATCH 153/357] tests --- .github/workflows/release_semver_start.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index e2b820876..514777e0e 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -154,7 +154,6 @@ jobs: action_branch: ${{ inputs.action_branch }} version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} - locale_folder: ${{ inputs.locale_folder }} # >= patch release (auto) kickoff_testing_auto: needs: parse_conventional_commits From 8f8a620071bdeb787f7ec366913a134e1fc90221 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 16:33:05 +0100 Subject: [PATCH 154/357] tests --- .github/workflows/release_semver_start.yml | 2 +- .github/workflows/skill_test_installation.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 514777e0e..19641d8b7 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -69,7 +69,7 @@ jobs: - name: Parse Conventional Commits id: parse run: | - export TITLE=${{ inputs.subject }} + export TITLE="${{ inputs.subject }}" cd ${{ github.workspace }}/action/github/scripts RELEASE_TYPE=$(python scripts/parse_semver_release.py) echo "release_type=$RELEASE_TYPE" >> $GITHUB_OUTPUT diff --git a/.github/workflows/skill_test_installation.yml b/.github/workflows/skill_test_installation.yml index fb310e182..8153f1a42 100644 --- a/.github/workflows/skill_test_installation.yml +++ b/.github/workflows/skill_test_installation.yml @@ -82,6 +82,6 @@ jobs: pip install . - name: Test Skill Installation run: | - export TEST_SKILL_PATH=${{ github.workspace }}/action/skill/${{inputs.skill_location}} - export TEST_SKILL_ENTRYPOINT_NAME=${{ inputs.skill_id }} + export TEST_SKILL_PATH="${{ github.workspace }}/action/skill/${{inputs.skill_location}}" + export TEST_SKILL_ENTRYPOINT_NAME="${{ inputs.skill_id }}" pytest ${{ github.workspace }}/action/github/test/test_skill_install.py From 15c95765a3a6551c38e36412def9e46e8b7b2094 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 16:47:00 +0100 Subject: [PATCH 155/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 19641d8b7..d00444245 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -133,11 +133,11 @@ jobs: repository: action/package/ branch: ${{ inputs.branch }} - name: update resource test file - if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }} + if: steps.filter.outputs.general == 'true' && inputs.update_intentfile != '' run: | python action/github/scripts/update_intent_testfile.py - name: Commit resource test file changes to ${{ inputs.branch }} - if: steps.filter.outputs.general == 'true' && ${{ inputs.update_intentfile }}' + if: steps.filter.outputs.general == 'true' && inputs.update_intentfile != '' uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "ci(localisation): update resource test file" From 09f1ce2a53b3b27a84b5354b9be89bdc61a2798e Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 17:10:25 +0100 Subject: [PATCH 156/357] tests --- .github/workflows/release_semver_start.yml | 10 +++++----- scripts/translate.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index d00444245..6ab9d0b92 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -109,24 +109,24 @@ jobs: - '${{ inputs.locale_folder }}/../dialog/**' - '${{ inputs.locale_folder }}/../vocab/**' - name: Setup Python - if: steps.filter.outputs.us_specific == 'true' + if: steps.filter.outputs.general == 'true' uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} - name: Install Python Requirements - if: steps.filter.outputs.us_specific == 'true' + if: steps.filter.outputs.general == 'true' run: | python -m pip install -r action/github/requirements/pip_translation.txt - name: Pull latest changes - if: steps.filter.outputs.us_specific == 'true' + if: steps.filter.outputs.general == 'true' run: | git pull origin ${{ inputs.branch }} - name: Auto Translate - if: steps.filter.outputs.us_specific == 'true' + if: steps.filter.outputs.general == 'true' run: | python action/github/scripts/translate.py - name: Commit autotranslation to ${{ inputs.branch }} - if: steps.filter.outputs.us_specific == 'true' + if: steps.filter.outputs.general == 'true' uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "ci(localisation): autotranslate common languages" diff --git a/scripts/translate.py b/scripts/translate.py index 31c957302..7727cd7be 100644 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -34,6 +34,8 @@ def get_target_languages(): if len(subfolder) == 5 and subfolder[2] == "-": langs.add(subfolder) langs = langs.union(BASE_LANGS) + if "en-us" in langs: + langs.remove("en-us") return langs single_lang = os.getenv("TARGET_LANG") @@ -59,7 +61,10 @@ def file_location(f: str, base: str) -> bool: def translate(lines: list, target_lang: str) -> list: translations = [] for line in lines: + drop = False replacements = dict() + # TODO: still not failsafe as the translator of different langs might + # interpret "@1" as something to translate (seen this with french), hence the drop for num, var in enumerate(re.findall(r"(?:{{|{)[ a-zA-Z0-9_]*(?:}}|})", line)): line = line.replace(var, f'@{num}', 1) replacements[f'@{num}'] = var @@ -68,8 +73,13 @@ def translate(lines: list, target_lang: str) -> list: except Exception as e: continue for num, var in replacements.items(): + # saveguard against bad translations + if num not in translated: + drop = True + break translated = translated.replace(num, var) - translations.append(translated) + if not drop: + translations.append(translated) return translations From 503c536a4d343b3e713ddda23f5f0d48862ae492 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 17:30:16 +0100 Subject: [PATCH 157/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 6ab9d0b92..7d33a2d98 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -70,7 +70,7 @@ jobs: id: parse run: | export TITLE="${{ inputs.subject }}" - cd ${{ github.workspace }}/action/github/scripts + cd ${{ github.workspace }}/action/github RELEASE_TYPE=$(python scripts/parse_semver_release.py) echo "release_type=$RELEASE_TYPE" >> $GITHUB_OUTPUT autotranslate: From 7e7033810690c8a9aeed157bf84ef140da24d24d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 18:50:36 +0100 Subject: [PATCH 158/357] tests --- README.md | 2 +- scripts/parse_semver_release.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 671bded1c..49e65ae69 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. TODO (per repo): - - `setup.py`: To fully adopt semver, the version number created by the setup has to be compliant: `x.x.x-alpha` (former alpha versions can't be used anymore) + - `setup.py`: setuptools cant cope with semver compliance: `x.x.x-alpha...` stays `x.x.xax` for now - add `main` branch protection rules ------------ diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index 734db5712..cbc471ac0 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -1,14 +1,23 @@ +import re +from os import environ + import pccc import semver -from os import environ """ translates a conventional commit title/message into a semver version """ -TITLE = environ.get("TITLE") -BODY = environ.get("BODY") -VERSION = environ.get("VERSION") + +def get_version(): + # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" + version = environ.get("VERSION", "") + match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", ) + if match: + return f"{match.group(1)}-{match.group(2)}" + else: + return version + def semver_from_cc(): ccr = pccc.ConventionalCommitRunner() @@ -47,7 +56,9 @@ def semver_from_version(): elif version.major != 0: return "major" - +TITLE = environ.get("TITLE") +BODY = environ.get("BODY") +VERSION = get_version() if VERSION: release = semver_from_version() From 10919d9b853f5a40bd3caa6b10581ee77f149737 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 19:04:54 +0100 Subject: [PATCH 159/357] tests --- scripts/parse_semver_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index cbc471ac0..96de28185 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -12,7 +12,7 @@ def get_version(): # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" version = environ.get("VERSION", "") - match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", ) + match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", version) if match: return f"{match.group(1)}-{match.group(2)}" else: From 7af13f170c53d8c7158ed44560f3070a81f29152 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 21:13:01 +0100 Subject: [PATCH 160/357] tests --- .../actions/notify_testing_matrix/action.yml | 44 ++++++++++++++++ .../release_semver_versioning/action.yml | 0 .github/workflows/release_semver_start.yml | 50 +++++++++++++++++-- ...ning.yml => release_semver_versioning.yml} | 0 scripts/semver_cycle_start.py | 3 +- 5 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 .github/actions/notify_testing_matrix/action.yml create mode 100644 .github/actions/release_semver_versioning/action.yml rename .github/workflows/{semver_release_versioning.yml => release_semver_versioning.yml} (100%) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml new file mode 100644 index 000000000..b30025d8a --- /dev/null +++ b/.github/actions/notify_testing_matrix/action.yml @@ -0,0 +1,44 @@ +name: Notify Matrix Chat + +# only triggers on pull request closed events +on: + workflow_call: + inputs: + kickoff_pr: + type: boolean + default: true + subject: + type: string + default: 'no subject' + release_type: + type: string + required: true + +jobs: + notify_pr_matrix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Send message to Matrix bots channel + if: ${{ !inputs.kickoff_pr }} + uses: fadenb/matrix-chat-message@v0.0.6 + # this action was triggered by + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + Testing QA phase of a ${{ inputs.release_type }} release started (${{ github.repository }}). + Subject to test: ${{ inputs.subject }} + - name: Send message to Matrix bots channel + if: ${{ inputs.kickoff_pr }} + uses: fadenb/matrix-chat-message@v0.0.6 + # this action was triggered by + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + PR to kick off the testing QA phase of a ${{ inputs.release_type }} release (${{ github.repository }}). + Please merge if it the testing phase should start. + Subject to test: ${{ inputs.subject }} \ No newline at end of file diff --git a/.github/actions/release_semver_versioning/action.yml b/.github/actions/release_semver_versioning/action.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 7d33a2d98..c63996b75 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -162,6 +162,7 @@ jobs: env: TARGET_BRANCH: 'testing' GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} RELEASE_TYPE: ${{ needs.parse_conventional_commits.outputs.release_type }} steps: - name: Checkout Package Repo @@ -183,11 +184,31 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Set up Git Authentication + run: | + cd ${{ github.workspace }}/action/package/ + git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - name: Check Latest Release if: ${{ inputs.kickoff_pr }} run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + # if there is no testing branch, create one from the tag of the latest release + - name: Create Branch (if necessary) + run: | + cd ${{ github.workspace }}/action/package/ + git fetch --unshallow origin ${{ inputs.branch }} + TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) + INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + if [ "$SINCE_TAG" == "0.0.0" ]; then + git checkout -b ${{ env.TARGET_BRANCH }} $INITIAL_COMMIT + else + git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.SINCE_TAG }} + fi + git push origin ${{ env.TARGET_BRANCH }} --force + git checkout ${{ inputs.branch }} + fi - name: create changelog id: changelog if: ${{ inputs.kickoff_pr }} @@ -225,12 +246,12 @@ jobs: git push origin ${{ env.TARGET_BRANCH }} - name: Bump Version if: ${{ !inputs.kickoff_pr }} - uses: ./action/github/.github/workflows/release_semver_versioning.yml + uses: ./action/github/.github/actions/release_semver_versioning with: release_type: ${{ env.RELEASE_TYPE }} version_file: ${{ inputs.version_file }} - name: Notify Matrix Room - uses: ./action/github/.github/workflows/notify_testing_matrix.yml + uses: ./action/github/.github/actions/notify_testing_matrix with: kickoff_pr: ${{ !inputs.kickoff_pr }} subject: ${{ inputs.subject }} @@ -242,6 +263,7 @@ jobs: env: TARGET_BRANCH: 'testing' GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} RELEASE_TYPE: ${{ inputs.release_type }} steps: - name: Checkout Package Repo @@ -263,11 +285,31 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Set up Git Authentication + run: | + cd ${{ github.workspace }}/action/package/ + git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - name: Check Latest Release if: ${{ inputs.kickoff_pr }} run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + # if there is no testing branch, create one from the tag of the latest release + - name: Create Branch (if necessary) + run: | + cd ${{ github.workspace }}/action/package/ + git fetch --unshallow origin ${{ inputs.branch }} + TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) + INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + if [ "$SINCE_TAG" == "0.0.0" ]; then + git checkout -b ${{ env.TARGET_BRANCH }} $INITIAL_COMMIT + else + git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.SINCE_TAG }} + fi + git push origin ${{ env.TARGET_BRANCH }} --force + git checkout ${{ inputs.branch }} + fi - name: create changelog id: changelog if: ${{ inputs.kickoff_pr }} @@ -302,12 +344,12 @@ jobs: git push origin ${{ env.TARGET_BRANCH }} - name: Bump Version if: ${{ !inputs.kickoff_pr }} - uses: ./action/github/.github/workflows/release_semver_versioning.yml + uses: ./action/github/.github/actions/release_semver_versioning with: release_type: ${{ env.RELEASE_TYPE }} version_file: ${{ inputs.version_file }} - name: Notify Matrix Room - uses: ./action/github/.github/workflows/notify_testing_matrix.yml + uses: ./action/github/.github/actions/notify_testing_matrix with: kickoff_pr: ${{ !inputs.kickoff_pr }} release_type: ${{ env.RELEASE_TYPE }} diff --git a/.github/workflows/semver_release_versioning.yml b/.github/workflows/release_semver_versioning.yml similarity index 100% rename from .github/workflows/semver_release_versioning.yml rename to .github/workflows/release_semver_versioning.yml diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index c88c26155..9eb805a98 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -50,7 +50,8 @@ def in_cycle(v: semver.Version) -> bool: releases = repo.get_releases() if not releases: - raise Exception("No releases found") + print("0.0.0") + exit(0) for id, release in enumerate(releases): version = get_semver(release.tag_name) From 1f9b505fc70b2499c5c729189c6e5990d9a4a4b4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 21:20:46 +0100 Subject: [PATCH 161/357] tests --- .github/workflows/release_semver_start.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c63996b75..3d556eed0 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -197,7 +197,6 @@ jobs: - name: Create Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ - git fetch --unshallow origin ${{ inputs.branch }} TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then @@ -298,7 +297,6 @@ jobs: - name: Create Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ - git fetch --unshallow origin ${{ inputs.branch }} TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then From ea835a2634bc36c9ba012433a77babfea14d6183 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 21 Feb 2024 21:28:37 +0100 Subject: [PATCH 162/357] tests --- .github/workflows/release_semver_start.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 3d556eed0..059060b63 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -158,6 +158,8 @@ jobs: kickoff_testing_auto: needs: parse_conventional_commits if: ${{ needs.parse_conventional_commits.outputs.release_type != 'alpha' && needs.parse_conventional_commits.outputs.release_type != 'No semver release.'}} + permissions: + contents: write runs-on: ubuntu-latest env: TARGET_BRANCH: 'testing' @@ -258,6 +260,8 @@ jobs: # >= patch release (manual) kickoff_testing_manual: if: ${{ inputs.release_type != 'alpha' && inputs.release_type != null }} + permissions: + contents: write runs-on: ubuntu-latest env: TARGET_BRANCH: 'testing' From 554ff1b1efb60b73ab9fa46e868b67f90b784dca Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 19:25:33 +0100 Subject: [PATCH 163/357] tests --- .../actions/notify_testing_matrix/action.yml | 78 +++++----- .github/actions/release_alpha/action.yml | 82 +++++++++++ .../release_semver_versioning/action.yml | 65 +++++++++ .github/workflows/notify_testing_matrix.yml | 27 ++-- .github/workflows/release_alpha.yml | 8 +- .github/workflows/release_semver_publish.yml | 18 +-- .github/workflows/release_semver_start.yml | 138 +++++------------- .../workflows/release_semver_versioning.yml | 22 ++- README.md | 22 ++- pyproject.toml | 3 +- scripts/parse_semver_release.py | 2 + 11 files changed, 278 insertions(+), 187 deletions(-) create mode 100644 .github/actions/release_alpha/action.yml diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index b30025d8a..a7f75eab5 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -1,44 +1,34 @@ -name: Notify Matrix Chat - -# only triggers on pull request closed events -on: - workflow_call: - inputs: - kickoff_pr: - type: boolean - default: true - subject: - type: string - default: 'no subject' - release_type: - type: string - required: true - -jobs: - notify_pr_matrix: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Send message to Matrix bots channel - if: ${{ !inputs.kickoff_pr }} - uses: fadenb/matrix-chat-message@v0.0.6 - # this action was triggered by - with: - homeserver: 'matrix.org' - token: ${{ secrets.MATRIX_TOKEN }} - channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: | - Testing QA phase of a ${{ inputs.release_type }} release started (${{ github.repository }}). - Subject to test: ${{ inputs.subject }} - - name: Send message to Matrix bots channel - if: ${{ inputs.kickoff_pr }} - uses: fadenb/matrix-chat-message@v0.0.6 - # this action was triggered by - with: - homeserver: 'matrix.org' - token: ${{ secrets.MATRIX_TOKEN }} - channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: | - PR to kick off the testing QA phase of a ${{ inputs.release_type }} release (${{ github.repository }}). - Please merge if it the testing phase should start. - Subject to test: ${{ inputs.subject }} \ No newline at end of file +name: 'Notify Matrix Chat' +description: 'Send a message to a Matrix chat room' +inputs: + kickoff_pr: + description: 'Kick off pull request' + required: true + default: true + subject: + description: 'Subject of the message' + required: true + default: 'no subject' + release_type: + description: 'Type of the release' + required: true +runs: + using: 'composite' + steps: + - name: Set message + shell: bash + run: | + if ${{ inputs.kickoff_pr }}; then + MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" + else + MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" + fi + echo "message=$MESSAGE" >> $GITHUB_OUTPUT + id: message + - name: Send message to Matrix + uses: fadenb/matrix-chat-message@v0.0.6 + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: ${{ steps.message.outputs.message }} \ No newline at end of file diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml new file mode 100644 index 000000000..ef603e636 --- /dev/null +++ b/.github/actions/release_alpha/action.yml @@ -0,0 +1,82 @@ +name: 'Publish Alpha Build' +description: 'This action will generate a distribution and upload it to PyPI' +inputs: + action_branch: + description: 'The shared action branch to checkout' + required: false + default: 'main' + version_file: + description: 'The file location to read the version from' + required: false + default: 'version.py' + python_version: + description: 'The python version to use' + required: false + default: '3.8' + changelog_file: + description: 'The file location of the changelog' + required: false + default: 'CHANGELOG.md' +runs: + using: 'composite' + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: dev + path: action/package/ + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Increment Version + shell: bash + run: | + python action/github/scripts/bump_alpha.py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: action/package/${{ inputs.changelog_file }} + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Increment Version to ${{ env.VERSION }} + repository: action/package/ + - name: Change working directory to release + shell: bash + run: cd ${{ github.workspace }}/action/package/ + - name: Create Pre-release + uses: ncipollo/release-action@v1 + with: + token: ${{ secrets.GH_PAT }} + tag: V${{ env.VERSION }} + name: Release ${{ env.VERSION }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + commit: dev + prerelease: true + - name: Install Build Tools + shell: bash + run: | + pip install --upgrade pip + python -m pip install build wheel + - name: Build Distribution Packages + shell: bash + run: | + cd action/package + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} + packages-dir: action/package/dist/ \ No newline at end of file diff --git a/.github/actions/release_semver_versioning/action.yml b/.github/actions/release_semver_versioning/action.yml index e69de29bb..8d4c41d28 100644 --- a/.github/actions/release_semver_versioning/action.yml +++ b/.github/actions/release_semver_versioning/action.yml @@ -0,0 +1,65 @@ +name: 'Bump Version (after testing QA started)' +description: 'This action bumps the version after the testing QA phase has started' +inputs: + action_branch: + description: 'The shared action branch to checkout' + required: false + default: 'main' + python_version: + description: 'The python version to use' + required: false + default: '3.8' + version_file: + description: 'The file location to read the version from' + required: false + default: 'version.py' + changelog_file: + description: 'The file location of the changelog' + required: false + default: 'CHANGELOG.md' + subject: + description: 'Subject title of the push/pull-request event to parse the release type.' + required: true + +runs: + using: 'composite' + steps: + - name: set up release type + shell: bash + run: | + RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') + echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV + - name: Checkout + uses: actions/checkout@v4 + with: + ref: dev + fetch-depth: 0 + - name: Increment Version + shell: bash + run: | + python action/github/scripts/bump_${{ env.RELEASE_TYPE }}.py + VERSION=$(python action/package/setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Push Version Change + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Increment Version to ${{ env.VERSION }} + repository: action/package/ + - name: Rebase testing onto dev + shell: bash + run: | + git checkout testing + git rebase dev + git push origin testing --force + - name: Notify Matrix Room + uses: ./action/github/.github/actions/notify_testing_matrix + with: + kickoff_pr: false + release_type: ${{ env.RELEASE_TYPE }} + - name: alpha_release + uses: ./action/github/.github/actions/release_alpha + with: + action_branch: ${{ inputs.action_branch }} + python_version: ${{ inputs.python_version }} + version_file: ${{ inputs.version_file }} + changelog_file: ${{ inputs.changelog_file }} \ No newline at end of file diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index b30025d8a..dd1b78b4c 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -19,26 +19,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set message + shell: bash + run: | + if ${{ inputs.kickoff_pr }}; then + MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" + else + MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" + fi + echo "message=$MESSAGE" >> $GITHUB_OUTPUT + id: message - name: Send message to Matrix bots channel - if: ${{ !inputs.kickoff_pr }} uses: fadenb/matrix-chat-message@v0.0.6 # this action was triggered by with: homeserver: 'matrix.org' token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: | - Testing QA phase of a ${{ inputs.release_type }} release started (${{ github.repository }}). - Subject to test: ${{ inputs.subject }} - - name: Send message to Matrix bots channel - if: ${{ inputs.kickoff_pr }} - uses: fadenb/matrix-chat-message@v0.0.6 - # this action was triggered by - with: - homeserver: 'matrix.org' - token: ${{ secrets.MATRIX_TOKEN }} - channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: | - PR to kick off the testing QA phase of a ${{ inputs.release_type }} release (${{ github.repository }}). - Please merge if it the testing phase should start. - Subject to test: ${{ inputs.subject }} \ No newline at end of file + message: ${{ steps.message.outputs.message }} \ No newline at end of file diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 744bafaa1..cfe454761 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -4,10 +4,6 @@ name: Publish Alpha Build ...aX on: workflow_call: inputs: - branch: - description: 'The branch to checkout and run the action in' - type: string - default: ${{ github.ref }} action_branch: description: 'The shared action branch to checkout' type: string @@ -35,7 +31,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} + ref: dev path: action/package/ - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -76,7 +72,7 @@ jobs: body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - commit: ${{ inputs.branch }} + commit: dev prerelease: true - name: Install Build Tools run: | diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index c18fa3598..547b2299c 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -71,18 +71,6 @@ jobs: with: token: ${{ secrets.GH_PAT }} sinceTag: ${{ env.SINCE_TAG }} - # pull changes in the release process back to dev - - name: Create Pull Request to dev - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GH_PAT }} - path: action/package/ - commit-message: "ci: merge testing commits to dev" - title: "ci: merge testing commits (${{ env.VERSION }}) to dev" - branch: dev - base: testing - draft: false - assignees: ${{ github.actor }} - name: Create Release id: create_release uses: ncipollo/release-action@v1 @@ -106,3 +94,9 @@ jobs: with: password: ${{secrets.PYPI_TOKEN}} packages-dir: action/package/dist/ + - name: Rebase dev onto master + run: | + cd ${{ github.workspace }}/action/package + git checkout dev + git rebase master + git push origin dev --force diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 059060b63..a6788a31a 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -74,7 +74,8 @@ jobs: RELEASE_TYPE=$(python scripts/parse_semver_release.py) echo "release_type=$RELEASE_TYPE" >> $GITHUB_OUTPUT autotranslate: - if: ${{ inputs.locale_folder != null }} + needs: parse_conventional_commits + if: ${{ inputs.locale_folder != '' && needs.parse_conventional_commits.outputs.release_type != 'testing' }} runs-on: ubuntu-latest permissions: contents: write @@ -143,21 +144,10 @@ jobs: commit_message: "ci(localisation): update resource test file" repository: action/package/ branch: ${{ inputs.branch }} - # alpha release - alpha_release: - needs: parse_conventional_commits - if: ${{ needs.parse_conventional_commits.outputs.release_type == 'alpha' || inputs.release_type == 'alpha' }} - uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 - secrets: inherit - with: - branch: ${{ inputs.branch }} - action_branch: ${{ inputs.action_branch }} - version_file: ${{ inputs.version_file }} - python_version: ${{ inputs.python_version }} - # >= patch release (auto) - kickoff_testing_auto: + # >= patch release + kickoff_testing: needs: parse_conventional_commits - if: ${{ needs.parse_conventional_commits.outputs.release_type != 'alpha' && needs.parse_conventional_commits.outputs.release_type != 'No semver release.'}} + if: (contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) || (inputs.release_type != 'alpha' && inputs.release_type != '')) permissions: contents: write runs-on: ubuntu-latest @@ -257,101 +247,45 @@ jobs: kickoff_pr: ${{ !inputs.kickoff_pr }} subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} - # >= patch release (manual) - kickoff_testing_manual: - if: ${{ inputs.release_type != 'alpha' && inputs.release_type != null }} + # testing release + cherry_pick_testing: + if: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' }} + runs-on: ubuntu-latest permissions: contents: write - runs-on: ubuntu-latest - env: - TARGET_BRANCH: 'testing' - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - RELEASE_TYPE: ${{ inputs.release_type }} steps: - - name: Checkout Package Repo + - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} - path: action/package/ fetch-depth: 0 - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} - path: action/github/ - - name: Set up python ${{ inputs.python_version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Common Python Requirements - run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + ref: dev + path: action/package/dev/ - name: Set up Git Authentication run: | - cd ${{ github.workspace }}/action/package/ + cd action/package/dev/ git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - - name: Check Latest Release - if: ${{ inputs.kickoff_pr }} - run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - # if there is no testing branch, create one from the tag of the latest release - - name: Create Branch (if necessary) - run: | - cd ${{ github.workspace }}/action/package/ - TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - if [ "$SINCE_TAG" == "0.0.0" ]; then - git checkout -b ${{ env.TARGET_BRANCH }} $INITIAL_COMMIT - else - git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.SINCE_TAG }} - fi - git push origin ${{ env.TARGET_BRANCH }} --force - git checkout ${{ inputs.branch }} - fi - - name: create changelog - id: changelog - if: ${{ inputs.kickoff_pr }} - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - if: ${{ inputs.kickoff_pr }} + - name: Checkout testing branch + uses: actions/checkout@v4 with: - token: ${{ secrets.GH_PAT }} - path: action/package/ - commit-message: "ci: kickoff testing QA phase" - title: "ci: kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" - body: | - This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. - Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. - - (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) - branch: ${{ env.TARGET_BRANCH }} - base: ${{ inputs.branch }} - draft: false - assignees: ${{ github.actor }} - - name: Push to testing branch - if: ${{ !inputs.kickoff_pr }} + fetch-depth: 0 + ref: testing + path: action/package/testing/ + - name: Cherry pick last commit from dev run: | - cd ${{ github.workspace }}/action/package/ - git checkout ${{ env.TARGET_BRANCH }} - git pull origin ${{ env.TARGET_BRANCH }} - git rebase ${{ inputs.branch }} - git push origin ${{ env.TARGET_BRANCH }} - - name: Bump Version - if: ${{ !inputs.kickoff_pr }} - uses: ./action/github/.github/actions/release_semver_versioning - with: - release_type: ${{ env.RELEASE_TYPE }} - version_file: ${{ inputs.version_file }} - - name: Notify Matrix Room - uses: ./action/github/.github/actions/notify_testing_matrix - with: - kickoff_pr: ${{ !inputs.kickoff_pr }} - release_type: ${{ env.RELEASE_TYPE }} + cd action/package/testing/ + git cherry-pick $(git -C ../dev rev-parse HEAD) + - name: Push changes to testing branch + run: | + cd action/package/testing/ + git push origin testing + # alpha release + alpha_release: + needs: parse_conventional_commits + if: (${{ !inputs.kickoff_pr && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ !inputs.kickoff_pr && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) + uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 + secrets: inherit + with: + branch: ${{ inputs.branch }} + action_branch: ${{ inputs.action_branch }} + version_file: ${{ inputs.version_file }} + python_version: ${{ inputs.python_version }} diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 7b2cad80c..838796935 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -4,10 +4,21 @@ name: Bump Version (after testing QA started) on: workflow_call: inputs: + action_branch: + description: 'The shared action branch to checkout' + type: string + default: main + python_version: + description: 'The python version to use' + type: string + default: "3.8" version_file: description: 'The file location to read the version from' type: string default: version.py + changelog_file: + type: string + default: CHANGELOG.md subject: type: string description: 'Subject title of the push/pull-request event to parse the release type.' @@ -52,4 +63,13 @@ jobs: uses: ./action/github/.github/workflows/notify_testing_matrix.yml with: kickoff_pr: false - release_type: ${{ inputs.RELEASE_TYPE }} + release_type: ${{ env.RELEASE_TYPE }} + alpha_release: + needs: start_testing_phase + secrets: inherit + uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 + with: + action_branch: ${{ inputs.action_branch }} + python_version: ${{ inputs.python_version }} + version_file: ${{ inputs.version_file }} + changelog_file: ${{ inputs.changelog_file }} diff --git a/README.md b/README.md index 49e65ae69..595517932 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling --------- - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. + - _Testing QA_: PRs/commits go into _dev_ with the prefix/scope `fix(testing):` TODO (per repo): - `setup.py`: setuptools cant cope with semver compliance: `x.x.x-alpha...` stays `x.x.xax` for now @@ -73,6 +74,7 @@ _Alpha releases are directly published without going through a test phase_ Strategy: 3-staged - Manually propose a testing start or automatically with setting "Conventional Commits" (optionally as PR) + - Kickoff PR: If the testing phase of the release should start right away or being time-delayed (starting when PR is merged). Input `kickoff_pr` (true/false) - Manually conclude testing phase, propose a stable release (PR) - Publishing of a stable release (on merge) and feed back the changes to the dev branch @@ -106,6 +108,7 @@ on: jobs: start_semver_release_mechanism: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/release_semver_start.yml@feat/shared_actions1 with: branch: dev # Branch to use, default: branch that triggered the action @@ -133,10 +136,11 @@ on: jobs: start_testing_phase: - if: (github.event.pull_request.merged == true) && (contains(github.event.pull_request.title, 'patch release') || contains(github.event.pull_request.title, 'minor release') || contains(github.event.pull_request.title, 'major release')) + if: github.actor != 'EggmanBot' && github.event.pull_request.merged == true && contains(fromJson('["patch release", "minor release", "major release"]'), github.event.pull_request.title) uses: openvoiceos/.github/.github/workflows/release_semver_versioning.yml@feat/shared_actions1 secrets: inherit with: + action_branch: feat/shared_actions1 # Shared action branch to use, default: main version_file: ovos_testpkg/version.py # the file containing the version number subject: ${{ github.event.pull_request.title }} ``` @@ -169,7 +173,7 @@ on: jobs: publish_stable_release: - if: (github.event.pull_request.merged == true) && (contains(github.event.pull_request.title, 'patch release stable') || contains(github.event.pull_request.title, 'minor release stable') || contains(github.event.pull_request.title, 'major release stable')) + if: github.actor != 'EggmanBot' && github.event.pull_request.merged == true && contains(fromJson('["patch release stable", "minor release stable", "major release stable"]'), github.event.pull_request.title) uses: openvoiceos/.github/.github/workflows/release_semver_publish.yml@feat/shared_actions1 secrets: inherit with: @@ -220,6 +224,7 @@ on: jobs: license_tests: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/license_tests.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest @@ -242,6 +247,7 @@ on: jobs: build_tests: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/python_build_tests.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest @@ -265,6 +271,7 @@ on: jobs: unit_tests: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main with: runner: ubuntu-latest @@ -288,6 +295,7 @@ jobs: # showcase with multiple jobs that should add to the coverage test next_test_that_needs_coverage: + if: github.actor != 'EggmanBot' needs: unit_tests uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main with: @@ -297,6 +305,7 @@ jobs: ... and_another_test_that_needs_coverage: needs: [ unit_tests, next_test_that_needs_coverage ] + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/pytest_file_or_dir.yml@main with: ... @@ -307,7 +316,7 @@ jobs: (*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) (**) Common python dependencies: [skill](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) / [other](https://github.com/OpenVoiceOS/.github/requirements/pip_tests.txt) ## Skills -## Skill Installation Tests +### Skill Installation Tests ```yaml name: Skill Installation Tests on: @@ -315,6 +324,7 @@ on: jobs: skill_installation_tests: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/skill_test_installation.yml@main with: runner: ubuntu-latest @@ -328,7 +338,7 @@ jobs: ``` (*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) (**) [Common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) -## Skill Resource Tests +### Skill Resource Tests Tests the resources of a skill (e.g dialogs, vocabs, regex or intent resources) for completeness and workability. ```yaml name: Skill Ressource Tests @@ -337,6 +347,7 @@ on: jobs: skill_resource_tests: + if: github.actor != 'EggmanBot' uses: openvoiceos/.github/.github/workflows/skill_test_resources.yml@main with: runner: ubuntu-latest # Runner to use, default: ubuntu-latest @@ -352,7 +363,8 @@ jobs: ``` (*) [Common system dependencies](https://github.com/OpenVoiceOS/.github/requirements/sys_deb_common_deps.txt) (**) [Common python dependencies](https://github.com/OpenVoiceOS/.github/requirements/pip_skill_tests.txt) -## Notify Matrix on Pull Request +## Notifications +### Notify Matrix on Pull Request ```yaml name: Notify Matrix Chat diff --git a/pyproject.toml b/pyproject.toml index 4560d2911..0a3425ec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,8 @@ scopes = [ "plugins", "localisation", "resources", - "release" + "release", + "testing" ] footers = [ diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index 96de28185..a9a676984 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -35,6 +35,8 @@ def semver_from_cc(): return "major" elif ccr.header.get("type") == "feat": return "minor" + elif ccr.header.get("type") == "fix" and ccr.header.get("scope") == "testing": + return "testing" elif ccr.header.get("type") == "fix": return "patch" else: From a487e56f7e862a740b802c364c84f47f5edf593f Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 19:38:44 +0100 Subject: [PATCH 164/357] tests --- .github/workflows/release_semver_start.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index a6788a31a..ad100d41b 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -36,6 +36,10 @@ on: type: string description: 'The file location of the intent test file to update' required: false + changelog_file: + type: string + description: 'The file location of the changelog' + default: CHANGELOG.md jobs: parse_conventional_commits: @@ -285,7 +289,7 @@ jobs: uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: - branch: ${{ inputs.branch }} action_branch: ${{ inputs.action_branch }} version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} + changelog_file: ${{ inputs.changelog_file }} From 8606df7a1b3d604c163bfaa14b80883a2ca587de Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 19:55:41 +0100 Subject: [PATCH 165/357] tests --- .github/workflows/release_semver_start.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index ad100d41b..d66210fa5 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -164,6 +164,7 @@ jobs: - name: Checkout Package Repo uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: ${{ inputs.branch }} path: action/package/ fetch-depth: 0 @@ -264,13 +265,10 @@ jobs: fetch-depth: 0 ref: dev path: action/package/dev/ - - name: Set up Git Authentication - run: | - cd action/package/dev/ - git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - name: Checkout testing branch uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} fetch-depth: 0 ref: testing path: action/package/testing/ From 5214d25c282c6cb891a0265267dd082fba3c41c6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 20:36:54 +0100 Subject: [PATCH 166/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index d66210fa5..a13f62e65 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -228,8 +228,8 @@ jobs: Included changes: ${{ steps.changelog.outputs.changelog }} - branch: ${{ env.TARGET_BRANCH }} - base: ${{ inputs.branch }} + branch: ${{ inputs.branch }} + base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} - name: Push to testing branch From 0a9d56e3ec4226db3b8dcd6bbbde0cb9eb4b1f21 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 21:15:44 +0100 Subject: [PATCH 167/357] tests --- .../actions/notify_testing_matrix/action.yml | 7 +++---- .github/workflows/notify_testing_matrix.yml | 2 +- .github/workflows/release_semver_start.yml | 18 +++++++++--------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index a7f75eab5..1bf5b29c0 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -3,22 +3,21 @@ description: 'Send a message to a Matrix chat room' inputs: kickoff_pr: description: 'Kick off pull request' - required: true - default: true + default: 'false' subject: description: 'Subject of the message' - required: true default: 'no subject' release_type: description: 'Type of the release' required: true + runs: using: 'composite' steps: - name: Set message shell: bash run: | - if ${{ inputs.kickoff_pr }}; then + if ${{ inputs.kickoff_pr == 'true' }}; then MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" else MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index dd1b78b4c..d53ed7afd 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -22,7 +22,7 @@ jobs: - name: Set message shell: bash run: | - if ${{ inputs.kickoff_pr }}; then + if ${{ inputs.kickoff_pr == 'true' }}; then MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" else MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index a13f62e65..4f2d8ad2a 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -25,9 +25,9 @@ on: description: 'Subject title of the push/pull request event to parse the release type.' required: false kickoff_pr: - type: boolean description: 'Whether to create a PR to mark the start of the testing phase or not' - default: false + type: string + default: 'false' locale_folder: type: string description: 'The folder location of the locale files' @@ -186,7 +186,7 @@ jobs: cd ${{ github.workspace }}/action/package/ git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - name: Check Latest Release - if: ${{ inputs.kickoff_pr }} + if: ${{ inputs.kickoff_pr == 'true' }} run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV @@ -207,13 +207,13 @@ jobs: fi - name: create changelog id: changelog - if: ${{ inputs.kickoff_pr }} + if: ${{ inputs.kickoff_pr == 'true' }} uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} sinceTag: ${{ env.SINCE_TAG }} - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - if: ${{ inputs.kickoff_pr }} + if: ${{ inputs.kickoff_pr == 'true'}} uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GH_PAT }} @@ -233,7 +233,7 @@ jobs: draft: false assignees: ${{ github.actor }} - name: Push to testing branch - if: ${{ !inputs.kickoff_pr }} + if: ${{ inputs.kickoff_pr == 'false' }} run: | cd ${{ github.workspace }}/action/package/ git checkout ${{ env.TARGET_BRANCH }} @@ -241,7 +241,7 @@ jobs: git rebase ${{ inputs.branch }} git push origin ${{ env.TARGET_BRANCH }} - name: Bump Version - if: ${{ !inputs.kickoff_pr }} + if: ${{ inputs.kickoff_pr == 'false' }} uses: ./action/github/.github/actions/release_semver_versioning with: release_type: ${{ env.RELEASE_TYPE }} @@ -249,7 +249,7 @@ jobs: - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: - kickoff_pr: ${{ !inputs.kickoff_pr }} + kickoff_pr: ${{ inputs.kickoff_pr }} subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} # testing release @@ -283,7 +283,7 @@ jobs: # alpha release alpha_release: needs: parse_conventional_commits - if: (${{ !inputs.kickoff_pr && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ !inputs.kickoff_pr && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) + if: (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: From e4fd5e4adff0efad84fd7eb2167c1d93e47a2f0d Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 25 Feb 2024 23:41:47 +0100 Subject: [PATCH 168/357] tests --- .github/workflows/release_semver_start.yml | 69 ++++++++++++++++------ 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4f2d8ad2a..4468f1fee 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -45,6 +45,7 @@ jobs: parse_conventional_commits: outputs: release_type: ${{ steps.parse.outputs.release_type }} + was_pr: ${{ steps.parse.outputs.was_pr }} runs-on: ubuntu-latest steps: - name: Checkout Package Repo @@ -70,6 +71,16 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + # check if the last commit was due to a PR merge (needed below) + - name: PR merge? + run: | + LAST_COMMIT_SHA=$(git rev-parse HEAD) + PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '.[0].number') + if [ -n "$PR_NUMBER" ]; then + echo "was_pr=true" >> $GITHUB_OUTPUT + else + echo "was_pr=false" >> $GITHUB_OUTPUT + fi - name: Parse Conventional Commits id: parse run: | @@ -79,8 +90,9 @@ jobs: echo "release_type=$RELEASE_TYPE" >> $GITHUB_OUTPUT autotranslate: needs: parse_conventional_commits - if: ${{ inputs.locale_folder != '' && needs.parse_conventional_commits.outputs.release_type != 'testing' }} runs-on: ubuntu-latest + outputs: + translated: ${{ steps.changes.outputs.translated }} permissions: contents: write env: @@ -101,6 +113,7 @@ jobs: ref: ${{ inputs.action_branch }} path: action/github/ - uses: dorny/paths-filter@v2 + if: ${{ inputs.locale_folder != '' }} id: filter with: working-directory: action/package/ @@ -130,24 +143,41 @@ jobs: if: steps.filter.outputs.general == 'true' run: | python action/github/scripts/translate.py + # 0 if no changes were made, 1 if changes were made + - name: changes made? + id: changes + run: | + git diff --exit-code + echo "translated=$?" >> $GITHUB_OUTPUT - name: Commit autotranslation to ${{ inputs.branch }} - if: steps.filter.outputs.general == 'true' + if: steps.changes.outputs.translated == '1' uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "ci(localisation): autotranslate common languages" repository: action/package/ branch: ${{ inputs.branch }} - name: update resource test file - if: steps.filter.outputs.general == 'true' && inputs.update_intentfile != '' + if: steps.changes.outputs.translated == '1' && inputs.update_intentfile != '' run: | python action/github/scripts/update_intent_testfile.py - name: Commit resource test file changes to ${{ inputs.branch }} - if: steps.filter.outputs.general == 'true' && inputs.update_intentfile != '' + if: steps.changes.outputs.translated == '1' && inputs.update_intentfile != '' uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "ci(localisation): update resource test file" repository: action/package/ branch: ${{ inputs.branch }} + # alpha release + alpha_release: + needs: parse_conventional_commits + if: (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) + uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 + secrets: inherit + with: + action_branch: ${{ inputs.action_branch }} + version_file: ${{ inputs.version_file }} + python_version: ${{ inputs.python_version }} + changelog_file: ${{ inputs.changelog_file }} # >= patch release kickoff_testing: needs: parse_conventional_commits @@ -253,7 +283,9 @@ jobs: subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} # testing release + # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: + needs: ["parse_conventional_commits", "autotranslate"] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' }} runs-on: ubuntu-latest permissions: @@ -272,22 +304,25 @@ jobs: fetch-depth: 0 ref: testing path: action/package/testing/ - - name: Cherry pick last commit from dev + # based on the workflow run, set the number of commits to cherry pick + # testing PRs/pushes get an alpha release for sure + - name: set commits num to cherry pick + run: | + COMMITS = 1 + if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then + COMMITS = $((COMMITS + 2)) + else + COMMITS = $((COMMITS + 1)) + fi + if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then + COMMITS = $((COMMITS + 1)) + fi + echo "COMMITS=${COMMITS}" >> $GITHUB_ENV + - name: Cherry pick n last commit from dev run: | cd action/package/testing/ - git cherry-pick $(git -C ../dev rev-parse HEAD) + git cherry-pick $(git -C ../dev log -n ${{ env.COMMITS }} --pretty=format:"%H") - name: Push changes to testing branch run: | cd action/package/testing/ git push origin testing - # alpha release - alpha_release: - needs: parse_conventional_commits - if: (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) - uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 - secrets: inherit - with: - action_branch: ${{ inputs.action_branch }} - version_file: ${{ inputs.version_file }} - python_version: ${{ inputs.python_version }} - changelog_file: ${{ inputs.changelog_file }} From 77e94e1d65c97418fb400d7605f59e1cecf519e2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 17:14:00 +0100 Subject: [PATCH 169/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4468f1fee..443098e6b 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -74,6 +74,7 @@ jobs: # check if the last commit was due to a PR merge (needed below) - name: PR merge? run: | + cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse HEAD) PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '.[0].number') if [ -n "$PR_NUMBER" ]; then From fcd25937576abb05ca88ed575857249b6278571c Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 17:18:06 +0100 Subject: [PATCH 170/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 443098e6b..c0e7b0315 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -148,6 +148,7 @@ jobs: - name: changes made? id: changes run: | + cd ${{ github.workspace }}/action/package/ git diff --exit-code echo "translated=$?" >> $GITHUB_OUTPUT - name: Commit autotranslation to ${{ inputs.branch }} From 59003b1ae3827bccf9fc3b71a3eb2c93ee445f10 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 18:59:01 +0100 Subject: [PATCH 171/357] tests --- .github/workflows/release_semver_start.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c0e7b0315..68195fb54 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -151,6 +151,10 @@ jobs: cd ${{ github.workspace }}/action/package/ git diff --exit-code echo "translated=$?" >> $GITHUB_OUTPUT + - name: Check data types + run: | + echo "Type of outputs.release_type ("${{ needs.parse_conventional_commits.outputs.release_type }}"): $(echo "${{ needs.parse_conventional_commits.outputs.release_type }}" | jq -R type)" + echo "Type of inputs.release_type ("${{ inputs.release_type }}"): $(echo "${{ inputs.release_type }}" | jq -R type)" - name: Commit autotranslation to ${{ inputs.branch }} if: steps.changes.outputs.translated == '1' uses: stefanzweifel/git-auto-commit-action@v5 @@ -260,7 +264,7 @@ jobs: Included changes: ${{ steps.changelog.outputs.changelog }} - branch: ${{ inputs.branch }} + branch: staging/test base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} From e26a97a96697f859e0e38f95397c38844a39eec7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 20:46:40 +0100 Subject: [PATCH 172/357] tests --- .github/workflows/release_semver_start.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 68195fb54..2b41f44b9 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -74,10 +74,9 @@ jobs: # check if the last commit was due to a PR merge (needed below) - name: PR merge? run: | - cd ${{ github.workspace }}/action/package/ + cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse HEAD) - PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '.[0].number') - if [ -n "$PR_NUMBER" ]; then + PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '. | select(length > 0) | .[0].number') echo "was_pr=true" >> $GITHUB_OUTPUT else echo "was_pr=false" >> $GITHUB_OUTPUT From fc5beb6a2dd5598028c6842df9062a7df7f23490 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 21:05:28 +0100 Subject: [PATCH 173/357] fix: bump --- .github/workflows/release_semver_start.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2b41f44b9..98afc2bb4 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -76,7 +76,8 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse HEAD) - PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '. | select(length > 0) | .[0].number') + PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '.[0].number') + if [ -n "$PR_NUMBER" ]; then echo "was_pr=true" >> $GITHUB_OUTPUT else echo "was_pr=false" >> $GITHUB_OUTPUT From 55ccf22d48b04aa4cba33f6f0f9d768279d3c00e Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 21:19:14 +0100 Subject: [PATCH 174/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 98afc2bb4..4d3ecdccc 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -154,7 +154,7 @@ jobs: - name: Check data types run: | echo "Type of outputs.release_type ("${{ needs.parse_conventional_commits.outputs.release_type }}"): $(echo "${{ needs.parse_conventional_commits.outputs.release_type }}" | jq -R type)" - echo "Type of inputs.release_type ("${{ inputs.release_type }}"): $(echo "${{ inputs.release_type }}" | jq -R type)" + echo "Type of inputs.kickoff_pr ("${{ inputs.kickoff_pr }}"): $(echo "${{ inputs.kickoff_pr }}" | jq -R type)" - name: Commit autotranslation to ${{ inputs.branch }} if: steps.changes.outputs.translated == '1' uses: stefanzweifel/git-auto-commit-action@v5 From 48f31144e56e8768d044f4ac89042f9a0bee14e2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 22:06:39 +0100 Subject: [PATCH 175/357] tests --- .github/workflows/release_semver_start.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4d3ecdccc..404d68abe 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -226,8 +226,15 @@ jobs: run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + - name: create changelog + id: changelog + if: ${{ inputs.kickoff_pr == 'true' }} + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + sinceTag: ${{ env.SINCE_TAG }} # if there is no testing branch, create one from the tag of the latest release - - name: Create Branch (if necessary) + - name: Create testing Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) @@ -241,13 +248,6 @@ jobs: git push origin ${{ env.TARGET_BRANCH }} --force git checkout ${{ inputs.branch }} fi - - name: create changelog - id: changelog - if: ${{ inputs.kickoff_pr == 'true' }} - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} - name: Create Pull Request to ${{ env.TARGET_BRANCH }} if: ${{ inputs.kickoff_pr == 'true'}} uses: peter-evans/create-pull-request@v5 @@ -264,7 +264,6 @@ jobs: Included changes: ${{ steps.changelog.outputs.changelog }} - branch: staging/test base: ${{ env.TARGET_BRANCH }} draft: false assignees: ${{ github.actor }} From b280d4ea262a1745da65bea7bf358d8d6dd259d8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 22:48:00 +0100 Subject: [PATCH 176/357] tests --- .github/workflows/release_semver_start.yml | 36 +++++++++------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 404d68abe..d841906c3 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -176,7 +176,7 @@ jobs: # alpha release alpha_release: needs: parse_conventional_commits - if: (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) }}) || (${{ inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type) }}) || (${{ inputs.release_type == 'alpha' }}) || (${{ contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type) }}) + if: (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type)) || (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type)) || (inputs.release_type == 'alpha') || (contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: @@ -210,6 +210,12 @@ jobs: repository: OpenVoiceOS/.github ref: ${{ inputs.action_branch }} path: action/github/ + - name: Install GitHub CLI + run: | + sudo apt update + sudo apt install gh + - name: Authenticate GitHub CLI + run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token - name: Set up python ${{ inputs.python_version }} uses: actions/setup-python@v5 with: @@ -217,10 +223,6 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: Set up Git Authentication - run: | - cd ${{ github.workspace }}/action/package/ - git remote set-url origin https://x-access-token:${{ secrets.GH_PAT }}@github.com/${{ github.repository }} - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | @@ -250,23 +252,13 @@ jobs: fi - name: Create Pull Request to ${{ env.TARGET_BRANCH }} if: ${{ inputs.kickoff_pr == 'true'}} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GH_PAT }} - path: action/package/ - commit-message: "ci(release): kickoff testing QA phase" - title: "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" - body: | - This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. - Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. - - (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) - - Included changes: - ${{ steps.changelog.outputs.changelog }} - base: ${{ env.TARGET_BRANCH }} - draft: false - assignees: ${{ github.actor }} + run: | + gh pr create \ + --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ + --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release.\nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version.\n\n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.)\n\nIncluded changes:\n${{ steps.changelog.outputs.changelog }}" \ + --base ${{ env.TARGET_BRANCH }} \ + --head dev \ + --assignee ${{ github.actor }} - name: Push to testing branch if: ${{ inputs.kickoff_pr == 'false' }} run: | From 901b2e75922fba6c0d40647806e968352acfd5db Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 23:12:03 +0100 Subject: [PATCH 177/357] tests --- .github/actions/notify_testing_matrix/action.yml | 9 ++++++--- .github/workflows/notify_testing_matrix.yml | 4 ++-- .github/workflows/release_semver_start.yml | 7 ++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index 1bf5b29c0..c1ddcf9a7 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -1,6 +1,9 @@ name: 'Notify Matrix Chat' description: 'Send a message to a Matrix chat room' inputs: + matrix_token: + description: 'The token to use to authenticate with the Matrix server' + required: true kickoff_pr: description: 'Kick off pull request' default: 'false' @@ -22,12 +25,12 @@ runs: else MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" fi - echo "message=$MESSAGE" >> $GITHUB_OUTPUT + echo "MESSAGE=$MESSAGE" >> $GITHUB_ENV id: message - name: Send message to Matrix uses: fadenb/matrix-chat-message@v0.0.6 with: homeserver: 'matrix.org' - token: ${{ secrets.MATRIX_TOKEN }} + token: ${{ inputs.matrix_token }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: ${{ steps.message.outputs.message }} \ No newline at end of file + message: ${{ env.MESSAGE }} \ No newline at end of file diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index d53ed7afd..7bbfbba0e 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -27,7 +27,7 @@ jobs: else MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" fi - echo "message=$MESSAGE" >> $GITHUB_OUTPUT + echo "MESSAGE=$MESSAGE" >> $GITHUB_ENV id: message - name: Send message to Matrix bots channel uses: fadenb/matrix-chat-message@v0.0.6 @@ -36,4 +36,4 @@ jobs: homeserver: 'matrix.org' token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: ${{ steps.message.outputs.message }} \ No newline at end of file + message: ${{ env.MESSAGE }} \ No newline at end of file diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index d841906c3..d0ab7fb59 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -151,10 +151,6 @@ jobs: cd ${{ github.workspace }}/action/package/ git diff --exit-code echo "translated=$?" >> $GITHUB_OUTPUT - - name: Check data types - run: | - echo "Type of outputs.release_type ("${{ needs.parse_conventional_commits.outputs.release_type }}"): $(echo "${{ needs.parse_conventional_commits.outputs.release_type }}" | jq -R type)" - echo "Type of inputs.kickoff_pr ("${{ inputs.kickoff_pr }}"): $(echo "${{ inputs.kickoff_pr }}" | jq -R type)" - name: Commit autotranslation to ${{ inputs.branch }} if: steps.changes.outputs.translated == '1' uses: stefanzweifel/git-auto-commit-action@v5 @@ -215,7 +211,7 @@ jobs: sudo apt update sudo apt install gh - name: Authenticate GitHub CLI - run: echo "${{ secrets.GH_TOKEN }}" | gh auth login --with-token + run: echo "${{ secrets.GH_PAT }}" | gh auth login --with-token - name: Set up python ${{ inputs.python_version }} uses: actions/setup-python@v5 with: @@ -276,6 +272,7 @@ jobs: - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: + matrix_token: ${{ secrets.MATRIX_TOKEN }} kickoff_pr: ${{ inputs.kickoff_pr }} subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} From 1b54fc838c2a3483b8f4ee521ed093a1afe339bd Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 23:18:52 +0100 Subject: [PATCH 178/357] tests --- .github/workflows/release_semver_start.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index d0ab7fb59..2ba1ef1cd 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -211,7 +211,9 @@ jobs: sudo apt update sudo apt install gh - name: Authenticate GitHub CLI - run: echo "${{ secrets.GH_PAT }}" | gh auth login --with-token + run: | + unset GITHUB_TOKEN + echo "${{ secrets.GH_PAT }}" | gh auth login --with-token - name: Set up python ${{ inputs.python_version }} uses: actions/setup-python@v5 with: From 1d3e1fcfa1532948940c4bbecbc8df7fa6fdc807 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 23:27:48 +0100 Subject: [PATCH 179/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2ba1ef1cd..140318619 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -251,6 +251,7 @@ jobs: - name: Create Pull Request to ${{ env.TARGET_BRANCH }} if: ${{ inputs.kickoff_pr == 'true'}} run: | + cd ${{ github.workspace }}/action/package/ gh pr create \ --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release.\nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version.\n\n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.)\n\nIncluded changes:\n${{ steps.changelog.outputs.changelog }}" \ From b49f5eec5263503b89dbd4e038400d48614c5921 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 26 Feb 2024 23:39:05 +0100 Subject: [PATCH 180/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 140318619..89e246c64 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -254,7 +254,7 @@ jobs: cd ${{ github.workspace }}/action/package/ gh pr create \ --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ - --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release.\nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version.\n\n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.)\n\nIncluded changes:\n${{ steps.changelog.outputs.changelog }}" \ + --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. \nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. \n \n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) \n \nIncluded changes: \n${{ steps.changelog.outputs.changelog }}" \ --base ${{ env.TARGET_BRANCH }} \ --head dev \ --assignee ${{ github.actor }} From 3706bebd2d2e08ba06f61507ec8caa65bc1c3a29 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 16:25:14 +0100 Subject: [PATCH 181/357] adjusted semver cycle script --- .github/workflows/release_semver_publish.yml | 1 + .../workflows/release_semver_pull_master.yml | 1 + .github/workflows/release_semver_start.yml | 1 + scripts/semver_cycle_start.py | 21 +++++++++++-------- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 547b2299c..29ea798ef 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -27,6 +27,7 @@ jobs: env: SOURCE_BRANCH: 'master' GITHUB_REPOSITORY: ${{ github.repository }} + GH_PAT: ${{ secrets.GH_PAT }} runs-on: ubuntu-latest steps: - name: set up release type diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index cbe50438d..ff5cdb303 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -21,6 +21,7 @@ jobs: env: TARGET_BRANCH: 'master' GITHUB_REPOSITORY: ${{ github.repository }} + GH_PAT: ${{ secrets.GH_PAT }} steps: - name: Checkout Repository uses: actions/checkout@v4 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 89e246c64..4e71155fe 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -192,6 +192,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GH_PAT }} RELEASE_TYPE: ${{ needs.parse_conventional_commits.outputs.release_type }} + GH_PAT: ${{ secrets.GH_PAT }} steps: - name: Checkout Package Repo uses: actions/checkout@v4 diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 9eb805a98..1eae86b7d 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -5,21 +5,19 @@ import semver """ -Get the start of the release cycle for the current release type. (patch, minor, major) -(the release cycle starts with the end version of the last cylce) -If `LAST_RELEASE` is set, the last release will be returned. -If `FIRST_RELEASE` is set, the first release will be returned. +Get the first release of the release cycle for the current release type. (patch, minor, major) +If `LAST_RELEASE` is set, the last release of that release type will be returned. +eg. last minor version of 0.10.7-a2 -> 0.10.0 """ GITHUB_REPOSITORY = getenv("GITHUB_REPOSITORY") RELEASE_TYPE = getenv("RELEASE_TYPE") -FIRST_RELEASE = bool(getenv("FIRST_RELEASE")) LAST_RELEASE = bool(getenv("LAST_RELEASE")) if any(req is None for req in [GITHUB_REPOSITORY, RELEASE_TYPE]): raise ValueError("Missing required environment variable(s)") -repo = Github(getenv("GITHUB_TOKEN")).get_repo(GITHUB_REPOSITORY) +repo = Github(getenv("GH_PAT")).get_repo(GITHUB_REPOSITORY) latest_version = None start_cycle_id = 0 @@ -58,6 +56,12 @@ def in_cycle(v: semver.Version) -> bool: if id == 0: latest_version = version if LAST_RELEASE: + if RELEASE_TYPE == "patch" and version.patch != latest_version.patch: + break + elif RELEASE_TYPE == "minor" and version.minor != latest_version.minor: + break + elif RELEASE_TYPE == "major" and version.major != latest_version.major: + break break continue @@ -68,8 +72,7 @@ def in_cycle(v: semver.Version) -> bool: if latest_version is None: exit(0) -elif start_cycle_id < releases.totalCount - 1 and not \ - (FIRST_RELEASE or LAST_RELEASE): - start_cycle_id += 1 +elif not LAST_RELEASE: + start_cycle_id -= 1 print(releases[start_cycle_id].tag_name) From 1d0fc89d2797c0e76b2ac51d872d0cf9db0f774d Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 17:25:51 +0100 Subject: [PATCH 182/357] fix cycle script --- scripts/semver_cycle_start.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 1eae86b7d..beb50c13b 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -55,6 +55,11 @@ def in_cycle(v: semver.Version) -> bool: version = get_semver(release.tag_name) if id == 0: latest_version = version + continue + + if not version: + continue + elif in_cycle(version): if LAST_RELEASE: if RELEASE_TYPE == "patch" and version.patch != latest_version.patch: break @@ -62,12 +67,8 @@ def in_cycle(v: semver.Version) -> bool: break elif RELEASE_TYPE == "major" and version.major != latest_version.major: break - break - continue - - if not version: - continue - elif in_cycle(version): + elif RELEASE_TYPE not in ["patch", "minor", "major"]: + break start_cycle_id = id if latest_version is None: From 5f9329d97b5b58099fa41f63f9b157f969e4fee1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 17:27:14 +0100 Subject: [PATCH 183/357] tests --- scripts/semver_cycle_start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index beb50c13b..0f52bcaa0 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -72,6 +72,7 @@ def in_cycle(v: semver.Version) -> bool: start_cycle_id = id if latest_version is None: + print("0.0.0") exit(0) elif not LAST_RELEASE: start_cycle_id -= 1 From f7690ba90c320c6f64f236d4b9fb041ee75e5fce Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 17:51:46 +0100 Subject: [PATCH 184/357] tests --- scripts/semver_cycle_start.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 0f52bcaa0..3870550f0 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -60,15 +60,6 @@ def in_cycle(v: semver.Version) -> bool: if not version: continue elif in_cycle(version): - if LAST_RELEASE: - if RELEASE_TYPE == "patch" and version.patch != latest_version.patch: - break - elif RELEASE_TYPE == "minor" and version.minor != latest_version.minor: - break - elif RELEASE_TYPE == "major" and version.major != latest_version.major: - break - elif RELEASE_TYPE not in ["patch", "minor", "major"]: - break start_cycle_id = id if latest_version is None: From 293a360f37e28b254dff42a80f26bfcc01ab14bc Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 18:23:36 +0100 Subject: [PATCH 185/357] tests --- scripts/semver_cycle_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 3870550f0..7fa8cf239 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -59,13 +59,13 @@ def in_cycle(v: semver.Version) -> bool: if not version: continue - elif in_cycle(version): + elif in_cycle(version) and RELEASE_TYPE in ["patch", "minor", "major"]: start_cycle_id = id if latest_version is None: print("0.0.0") exit(0) -elif not LAST_RELEASE: +elif not LAST_RELEASE and start_cycle_id > 0: start_cycle_id -= 1 print(releases[start_cycle_id].tag_name) From 9a3de399c45d5bb6244927909d4113c3eaea2a14 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 21:12:07 +0100 Subject: [PATCH 186/357] fixes --- .../workflows/release_semver_versioning.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 838796935..3755e55e5 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -1,4 +1,4 @@ -name: Bump Version (after testing QA started) +name: Bump Version (after testing QA started) and announce testing phase # only triggers on pull request closed events on: @@ -25,7 +25,16 @@ on: required: true jobs: - start_testing_phase: + alpha_release: + secrets: inherit + uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 + with: + action_branch: ${{ inputs.action_branch }} + python_version: ${{ inputs.python_version }} + version_file: ${{ inputs.version_file }} + changelog_file: ${{ inputs.changelog_file }} + bump_version_and_announce_testing: + needs: alpha_release env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: 'testing' @@ -64,12 +73,3 @@ jobs: with: kickoff_pr: false release_type: ${{ env.RELEASE_TYPE }} - alpha_release: - needs: start_testing_phase - secrets: inherit - uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 - with: - action_branch: ${{ inputs.action_branch }} - python_version: ${{ inputs.python_version }} - version_file: ${{ inputs.version_file }} - changelog_file: ${{ inputs.changelog_file }} From 1a1832c25ca7fbd552b21ee9be1d756acefcfa23 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 22:05:06 +0100 Subject: [PATCH 187/357] tests --- .github/workflows/release_semver_start.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4e71155fe..615c0871a 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -225,6 +225,7 @@ jobs: - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | + export LAST_RELEASE=true SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: create changelog @@ -247,12 +248,12 @@ jobs: git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.SINCE_TAG }} fi git push origin ${{ env.TARGET_BRANCH }} --force - git checkout ${{ inputs.branch }} fi - name: Create Pull Request to ${{ env.TARGET_BRANCH }} if: ${{ inputs.kickoff_pr == 'true'}} run: | cd ${{ github.workspace }}/action/package/ + git checkout ${{ inputs.branch }} gh pr create \ --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. \nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. \n \n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) \n \nIncluded changes: \n${{ steps.changelog.outputs.changelog }}" \ From 06e56e5ffbc0d6841bb80141ef4878334c4ef077 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 27 Feb 2024 23:30:58 +0100 Subject: [PATCH 188/357] tests --- .github/workflows/release_semver_start.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 615c0871a..f80eb0a13 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -228,13 +228,6 @@ jobs: export LAST_RELEASE=true SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: create changelog - id: changelog - if: ${{ inputs.kickoff_pr == 'true' }} - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} # if there is no testing branch, create one from the tag of the latest release - name: Create testing Branch (if necessary) run: | @@ -249,11 +242,17 @@ jobs: fi git push origin ${{ env.TARGET_BRANCH }} --force fi + - name: create changelog + id: changelog + if: ${{ inputs.kickoff_pr == 'true' }} + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + sinceTag: ${{ env.SINCE_TAG }} - name: Create Pull Request to ${{ env.TARGET_BRANCH }} if: ${{ inputs.kickoff_pr == 'true'}} run: | cd ${{ github.workspace }}/action/package/ - git checkout ${{ inputs.branch }} gh pr create \ --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. \nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. \n \n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) \n \nIncluded changes: \n${{ steps.changelog.outputs.changelog }}" \ From 8a6e44b9993394b0ac766707a2b64982ccb7855d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 28 Feb 2024 15:01:57 +0100 Subject: [PATCH 189/357] tests --- .github/workflows/release_semver_publish.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 29ea798ef..8f268e5ea 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -5,6 +5,10 @@ on: PYPI_TOKEN: required: true inputs: + branch: + description: 'The branch to checkout' + type: string + default: master action_branch: description: 'The shared action branch to checkout' type: string @@ -25,7 +29,6 @@ on: jobs: build_and_publish: env: - SOURCE_BRANCH: 'master' GITHUB_REPOSITORY: ${{ github.repository }} GH_PAT: ${{ secrets.GH_PAT }} runs-on: ubuntu-latest @@ -37,7 +40,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ env.SOURCE_BRANCH }} + ref: ${{ inputs.branch }} fetch-depth: 0 path: action/package/ - name: Checkout Scripts Repo From 1009b980b67a48df38da2c078ef00cb3430dcbd7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 28 Feb 2024 18:33:32 +0100 Subject: [PATCH 190/357] release version script --- scripts/semver_cycle_start.py | 53 ++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 7fa8cf239..61396dbbb 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -5,17 +5,29 @@ import semver """ -Get the first release of the release cycle for the current release type. (patch, minor, major) -If `LAST_RELEASE` is set, the last release of that release type will be returned. -eg. last minor version of 0.10.7-a2 -> 0.10.0 +Get the first, last or next release versions of the release cycle +for the current release type. (patch, minor, major) + + If `LAST_RELEASE` is set, the last release of that release type will be returned. + eg. last minor version of 0.10.7-a2 -> 0.10.0 + + If `NEXT_RELEASE` is set, the next release of the upcoming release type will be returned. + eg. next minor version of 0.10.7-a2 -> 0.11.0 + next alpha version of 0.10.7 -> 0.10.8a1 + next alpha version of 0.10.7a2 -> 0.10.7a3 + """ +ALPHA_MARKER = getenv("ALPHA_MARKER", "a") + GITHUB_REPOSITORY = getenv("GITHUB_REPOSITORY") RELEASE_TYPE = getenv("RELEASE_TYPE") LAST_RELEASE = bool(getenv("LAST_RELEASE")) +NEXT_RELEASE = bool(getenv("NEXT_RELEASE")) if any(req is None for req in [GITHUB_REPOSITORY, RELEASE_TYPE]): - raise ValueError("Missing required environment variable(s)") + raise ValueError("Missing required environment variable(s) " + "`GITHUB_REPOSITORY` and `RELEASE_TYPE`)") repo = Github(getenv("GH_PAT")).get_repo(GITHUB_REPOSITORY) latest_version = None @@ -34,6 +46,23 @@ def get_semver(tag: str) -> semver.Version: return semver.Version.parse(tag) +def bump_version(v: semver.Version) -> semver.Version: + if RELEASE_TYPE == "patch": + return v.bump_patch() + elif RELEASE_TYPE == "minor": + return v.bump_minor() + elif RELEASE_TYPE == "major": + return v.bump_major() + elif RELEASE_TYPE == "alpha": + if not v.prerelease: + v = v.bump_patch() + return to_pypi_format(v.bump_prerelease(ALPHA_MARKER)) + + +def to_pypi_format(v: semver.Version) -> str: + return f"{v.major}.{v.minor}.{v.patch}{v.prerelease.replace('.', '') if v.prerelease else ''}" + + def in_cycle(v: semver.Version) -> bool: if RELEASE_TYPE == "patch": return v.patch == latest_version.patch and not \ @@ -47,14 +76,14 @@ def in_cycle(v: semver.Version) -> bool: releases = repo.get_releases() -if not releases: - print("0.0.0") - exit(0) for id, release in enumerate(releases): version = get_semver(release.tag_name) if id == 0: latest_version = version + if NEXT_RELEASE: + next_version = bump_version(latest_version) + break continue if not version: @@ -62,10 +91,14 @@ def in_cycle(v: semver.Version) -> bool: elif in_cycle(version) and RELEASE_TYPE in ["patch", "minor", "major"]: start_cycle_id = id +version = None if latest_version is None: - print("0.0.0") - exit(0) + version = "0.0.0" + if NEXT_RELEASE: + version = bump_version(semver.Version.parse(version)) +elif NEXT_RELEASE: + version = next_version elif not LAST_RELEASE and start_cycle_id > 0: start_cycle_id -= 1 -print(releases[start_cycle_id].tag_name) +print(version or releases[start_cycle_id].tag_name) From f849971269d5eb2c5dd01cd02d05c1a6e7fc35ec Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 28 Feb 2024 20:28:32 +0100 Subject: [PATCH 191/357] release cycle script --- README.md | 1 + scripts/semver_cycle_start.py | 57 +++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 595517932..4e286b961 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. - _Testing QA_: PRs/commits go into _dev_ with the prefix/scope `fix(testing):` + - Release-tags: The tag resemble the semantic versioning (eg. `1.2.3`; no prefix!) TODO (per repo): - `setup.py`: setuptools cant cope with semver compliance: `x.x.x-alpha...` stays `x.x.xax` for now diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py index 61396dbbb..eb8f9eafb 100644 --- a/scripts/semver_cycle_start.py +++ b/scripts/semver_cycle_start.py @@ -5,30 +5,52 @@ import semver """ -Get the first, last or next release versions of the release cycle +Get the first, last, latest or next release versions of the release cycle for the current release type. (patch, minor, major) - If `LAST_RELEASE` is set, the last release of that release type will be returned. + enviroment Variable `RELEASE`: + if unset, the first release of that release type in the current cycle will be returned. + + If set to `last`, the last release of that release type in the current cycle will be returned. eg. last minor version of 0.10.7-a2 -> 0.10.0 + last patch version of 0.10.1 -> 0.10.1 + last patch version of 0.10.0a2 -> 0.10.0 (!) + + If set to `latest`, returns the latest version released. - If `NEXT_RELEASE` is set, the next release of the upcoming release type will be returned. + If set to `next` the next release of the upcoming release type will be returned. eg. next minor version of 0.10.7-a2 -> 0.11.0 next alpha version of 0.10.7 -> 0.10.8a1 next alpha version of 0.10.7a2 -> 0.10.7a3 - """ ALPHA_MARKER = getenv("ALPHA_MARKER", "a") GITHUB_REPOSITORY = getenv("GITHUB_REPOSITORY") -RELEASE_TYPE = getenv("RELEASE_TYPE") -LAST_RELEASE = bool(getenv("LAST_RELEASE")) -NEXT_RELEASE = bool(getenv("NEXT_RELEASE")) - -if any(req is None for req in [GITHUB_REPOSITORY, RELEASE_TYPE]): +RELEASE_TYPE = getenv("RELEASE_TYPE", "").lower().strip() +RELEASE = getenv("RELEASE", "").lower().strip() + +# validate environment variables +if RELEASE and RELEASE not in ["last", "next", "latest"]: + raise ValueError("Invalid value for `RELEASE` environment variable. " + "Expected one of `last`, `next`, `latest`.") + +if RELEASE_TYPE and RELEASE_TYPE not in ["patch", "minor", "major", "alpha"]: + raise ValueError("Invalid value for `RELEASE_TYPE` environment variable. " + "Expected one of `patch`, `minor`, `major`, `alpha`.") + +requirements = { + "GITHUB_REPOSITORY": GITHUB_REPOSITORY +} +if RELEASE != "latest": + requirements.update({"RELEASE_TYPE", RELEASE_TYPE}) + +missing_req = [name for name, value in requirements.items() if value is None] +if missing_req: raise ValueError("Missing required environment variable(s) " - "`GITHUB_REPOSITORY` and `RELEASE_TYPE`)") + f"{str(missing_req)}") +# get the repo repo = Github(getenv("GH_PAT")).get_repo(GITHUB_REPOSITORY) latest_version = None start_cycle_id = 0 @@ -53,7 +75,7 @@ def bump_version(v: semver.Version) -> semver.Version: return v.bump_minor() elif RELEASE_TYPE == "major": return v.bump_major() - elif RELEASE_TYPE == "alpha": + elif RELEASE_TYPE in ["alpha", ""]: if not v.prerelease: v = v.bump_patch() return to_pypi_format(v.bump_prerelease(ALPHA_MARKER)) @@ -74,16 +96,19 @@ def in_cycle(v: semver.Version) -> bool: elif RELEASE_TYPE == "major": return v.major == latest_version.major - +# get the release history releases = repo.get_releases() for id, release in enumerate(releases): version = get_semver(release.tag_name) if id == 0: latest_version = version - if NEXT_RELEASE: + if RELEASE == "next": next_version = bump_version(latest_version) break + elif RELEASE == "latest": + start_cycle_id = 0 + break continue if not version: @@ -94,11 +119,11 @@ def in_cycle(v: semver.Version) -> bool: version = None if latest_version is None: version = "0.0.0" - if NEXT_RELEASE: + if RELEASE == "next": version = bump_version(semver.Version.parse(version)) -elif NEXT_RELEASE: +elif RELEASE == "next": version = next_version -elif not LAST_RELEASE and start_cycle_id > 0: +elif not RELEASE == "last" and start_cycle_id > 0: start_cycle_id -= 1 print(version or releases[start_cycle_id].tag_name) From 89899228a137a58163c6518154e03165d24aa1c8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 28 Feb 2024 20:42:46 +0100 Subject: [PATCH 192/357] tests --- .github/workflows/release_semver_publish.yml | 1 + .github/workflows/release_semver_pull_master.yml | 3 ++- .github/workflows/release_semver_start.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 8f268e5ea..c6a638586 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -62,6 +62,7 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release run: | + export RELEASE="latest" SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Bump Changelog diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index ff5cdb303..33decf021 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -53,8 +53,9 @@ jobs: run: | RELEASE_TYPE=$(python ${{ github.workspace }}/action/github/scripts/parse_semver_release.py) echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - - name: Get Start of the Version Cycle + - name: Check Latest Release run: | + export RELEASE="latest" SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Bump Changelog diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index f80eb0a13..98d31c819 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -225,7 +225,7 @@ jobs: - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | - export LAST_RELEASE=true + export RELEASE="latest" SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV # if there is no testing branch, create one from the tag of the latest release From 98dc9c2462932de45d84ed52f02f91c88ed376c2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 29 Feb 2024 00:21:36 +0100 Subject: [PATCH 193/357] release version script --- .github/workflows/release_semver_publish.yml | 3 +- .../workflows/release_semver_pull_master.yml | 3 +- .github/workflows/release_semver_start.yml | 3 +- scripts/semver_cycle_start.py | 129 ------------------ scripts/semver_release_version.py | 128 +++++++++++++++++ 5 files changed, 131 insertions(+), 135 deletions(-) delete mode 100644 scripts/semver_cycle_start.py create mode 100644 scripts/semver_release_version.py diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index c6a638586..030c7a324 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -62,8 +62,7 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release run: | - export RELEASE="latest" - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Bump Changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 33decf021..9e8e39fff 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -55,8 +55,7 @@ jobs: echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Check Latest Release run: | - export RELEASE="latest" - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: Bump Changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 98d31c819..28093d4eb 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -225,8 +225,7 @@ jobs: - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | - export RELEASE="latest" - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_cycle_start.py) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV # if there is no testing branch, create one from the tag of the latest release - name: Create testing Branch (if necessary) diff --git a/scripts/semver_cycle_start.py b/scripts/semver_cycle_start.py deleted file mode 100644 index eb8f9eafb..000000000 --- a/scripts/semver_cycle_start.py +++ /dev/null @@ -1,129 +0,0 @@ -from os import getenv -import re - -from github import Github -import semver - -""" -Get the first, last, latest or next release versions of the release cycle -for the current release type. (patch, minor, major) - - enviroment Variable `RELEASE`: - if unset, the first release of that release type in the current cycle will be returned. - - If set to `last`, the last release of that release type in the current cycle will be returned. - eg. last minor version of 0.10.7-a2 -> 0.10.0 - last patch version of 0.10.1 -> 0.10.1 - last patch version of 0.10.0a2 -> 0.10.0 (!) - - If set to `latest`, returns the latest version released. - - If set to `next` the next release of the upcoming release type will be returned. - eg. next minor version of 0.10.7-a2 -> 0.11.0 - next alpha version of 0.10.7 -> 0.10.8a1 - next alpha version of 0.10.7a2 -> 0.10.7a3 -""" - -ALPHA_MARKER = getenv("ALPHA_MARKER", "a") - -GITHUB_REPOSITORY = getenv("GITHUB_REPOSITORY") -RELEASE_TYPE = getenv("RELEASE_TYPE", "").lower().strip() -RELEASE = getenv("RELEASE", "").lower().strip() - -# validate environment variables -if RELEASE and RELEASE not in ["last", "next", "latest"]: - raise ValueError("Invalid value for `RELEASE` environment variable. " - "Expected one of `last`, `next`, `latest`.") - -if RELEASE_TYPE and RELEASE_TYPE not in ["patch", "minor", "major", "alpha"]: - raise ValueError("Invalid value for `RELEASE_TYPE` environment variable. " - "Expected one of `patch`, `minor`, `major`, `alpha`.") - -requirements = { - "GITHUB_REPOSITORY": GITHUB_REPOSITORY -} -if RELEASE != "latest": - requirements.update({"RELEASE_TYPE", RELEASE_TYPE}) - -missing_req = [name for name, value in requirements.items() if value is None] -if missing_req: - raise ValueError("Missing required environment variable(s) " - f"{str(missing_req)}") - -# get the repo -repo = Github(getenv("GH_PAT")).get_repo(GITHUB_REPOSITORY) -latest_version = None -start_cycle_id = 0 - -def get_semver(tag: str) -> semver.Version: - # hack for v prefix - tag = tag.lstrip("v").lstrip("V") - - # hack for alpha releases - if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): - tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) - - if not semver.Version.is_valid(tag): - return None - return semver.Version.parse(tag) - - -def bump_version(v: semver.Version) -> semver.Version: - if RELEASE_TYPE == "patch": - return v.bump_patch() - elif RELEASE_TYPE == "minor": - return v.bump_minor() - elif RELEASE_TYPE == "major": - return v.bump_major() - elif RELEASE_TYPE in ["alpha", ""]: - if not v.prerelease: - v = v.bump_patch() - return to_pypi_format(v.bump_prerelease(ALPHA_MARKER)) - - -def to_pypi_format(v: semver.Version) -> str: - return f"{v.major}.{v.minor}.{v.patch}{v.prerelease.replace('.', '') if v.prerelease else ''}" - - -def in_cycle(v: semver.Version) -> bool: - if RELEASE_TYPE == "patch": - return v.patch == latest_version.patch and not \ - (v.minor < latest_version.minor) and not \ - (v.major < latest_version.major) - elif RELEASE_TYPE == "minor": - return v.minor == latest_version.minor and not \ - (v.major < latest_version.major) - elif RELEASE_TYPE == "major": - return v.major == latest_version.major - -# get the release history -releases = repo.get_releases() - -for id, release in enumerate(releases): - version = get_semver(release.tag_name) - if id == 0: - latest_version = version - if RELEASE == "next": - next_version = bump_version(latest_version) - break - elif RELEASE == "latest": - start_cycle_id = 0 - break - continue - - if not version: - continue - elif in_cycle(version) and RELEASE_TYPE in ["patch", "minor", "major"]: - start_cycle_id = id - -version = None -if latest_version is None: - version = "0.0.0" - if RELEASE == "next": - version = bump_version(semver.Version.parse(version)) -elif RELEASE == "next": - version = next_version -elif not RELEASE == "last" and start_cycle_id > 0: - start_cycle_id -= 1 - -print(version or releases[start_cycle_id].tag_name) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py new file mode 100644 index 000000000..ee3486214 --- /dev/null +++ b/scripts/semver_release_version.py @@ -0,0 +1,128 @@ +from os import getenv +from os.path import isfile +import re + +from github import Github +import semver +import argparse + +""" +Get the first, last, latest or next release versions of the release cycle +for the current release type. (patch, minor, major) + +Args + --alpha_marker: str # marker for alpha releases, default is 'a' + --repo: str # the repository to get the releases from + --type: str # the release type to get the version for + --output_file: str # writes an ovos version file to the specified path + +Flags + --last # get the last release of that release type in the current cycle + --next # get the next release of the upcoming release type + --latest # get the latest version released + none of above # get the first release of that release type in the current cycle +""" + +parser = argparse.ArgumentParser() + +parser.add_argument("--alpha_marker", default="a") +parser.add_argument("--repo", required=True) +parser.add_argument("--type", choices=["patch", "minor", "major", "alpha", "prerelease"]) +parser.add_argument("--output_file") + +release_group = parser.add_mutually_exclusive_group() +release_group.add_argument("--last", action='store_true') +release_group.add_argument("--next", action='store_true') +release_group.add_argument("--latest", action='store_true') + +args = parser.parse_args() + + +RELEASE_TYPE = args.type +# old habits +if RELEASE_TYPE == "alpha" or RELEASE_TYPE is None: + RELEASE_TYPE = "prerelease" + +if not args.latest and not RELEASE_TYPE: + raise ValueError("Missing release type. Please set the `--type` argument") + +if args.output_file and not isfile(args.output_file): + raise ValueError(f"Output file does not exist: {args.output_file}") + +# get the repo +repo = Github(getenv("GH_PAT")).get_repo(args.repo) +latest_version = None +release_id = None + +def get_semver(tag: str) -> semver.Version: + # hack for v prefix + tag = tag.lstrip("v").lstrip("V") + + # hack for alpha releases + if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): + tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) + + if not semver.Version.is_valid(tag): + return None + return semver.Version.parse(tag) + + +def bump_version(v: semver.Version) -> semver.Version: + return to_pypi_format(v.next_version(RELEASE_TYPE, args.alpha_marker)) + + +def to_pypi_format(v: semver.Version) -> str: + return f"{v.major}.{v.minor}.{v.patch}{v.prerelease.replace('.', '') if v.prerelease else ''}" + + +def in_cycle(v: semver.Version) -> bool: + if RELEASE_TYPE == "patch": + return v.patch == latest_version.patch and not \ + (v.minor < latest_version.minor) and not \ + (v.major < latest_version.major) + elif RELEASE_TYPE == "minor": + return v.minor == latest_version.minor and not \ + (v.major < latest_version.major) + elif RELEASE_TYPE == "major": + return v.major == latest_version.major + +# get the release history +releases = repo.get_releases() +if not releases: + semv = semver.Version.parse("0.0.0") + if args.next: + semv = semv.next_version(RELEASE_TYPE, args.alpha_marker) +elif not args.next and not args.latest: + for id, release in enumerate(releases): + version = get_semver(release.tag_name) + if version and in_cycle(version) and RELEASE_TYPE in ["patch", "minor", "major"]: + release_id = id +elif args.next: + latest_version = get_semver(releases[0].tag_name) + semv = latest_version.next_version(RELEASE_TYPE, args.alpha_marker) +elif args.latest: + semv = get_semver(releases[0].tag_name) + release_id = 0 + +if not args.last and release_id: + release_id -= 1 + +if release_id is not None: + semv = get_semver(releases[release_id].tag_name) + +if args.output_file: + version_block = f""" +# START_VERSION_BLOCK +VERSION_MAJOR = {semv.major} +VERSION_MINOR = {semv.minor} +VERSION_BUILD = {semv.patch} +VERSION_ALPHA = {semv.prerelease.replace(args.alpha_marker, '').replace(".", "") if semv.prerelease else 0} +# END_VERSION_BLOCK +""" + with open(args.output_file, "w") as f: + f.write(version_block) + +if release_id is not None: + print(releases[release_id].tag_name) +else: + print(to_pypi_format(semv)) From fb7c188f6c64f2bac1bea291704e4c68f233e030 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 5 Mar 2024 23:17:02 +0100 Subject: [PATCH 194/357] coming close --- .github/actions/release_alpha/action.yml | 90 ++-- .../release_semver_versioning/action.yml | 47 +- .github/workflows/release_alpha.yml | 89 ++-- .github/workflows/release_semver_publish.yml | 34 +- .../workflows/release_semver_pull_master.yml | 46 +- .github/workflows/release_semver_start.yml | 27 +- .../workflows/release_semver_versioning.yml | 31 +- README.md | 5 +- requirements/pip_build_tools.txt | 2 + requirements/sys_deb_common_deps.txt | 1 + scripts/semver_release_version.py | 448 ++++++++++++++---- 11 files changed, 592 insertions(+), 228 deletions(-) create mode 100644 requirements/pip_build_tools.txt diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index ef603e636..bda5ee8d7 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -19,12 +19,11 @@ inputs: default: 'CHANGELOG.md' runs: using: 'composite' + env: + VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} + SOURCE_BRANCH: ${{ inputs.release_type == 'alpha' && 'dev' || 'testing' }} + GITHUB_REPOSITORY: ${{ github.repository }} steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: dev - path: action/package/ - name: Checkout Scripts Repo uses: actions/checkout@v4 with: @@ -35,48 +34,79 @@ runs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} + - name: Install Python Dependencies + run: | + pip install --upgrade pip + pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_build_tools.txt + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_BRANCH }} + path: action/package/ + # testing specific steps. Those releases belong to the previous release cycle + # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there + - name: Checkout At Tag And Cherry Pick + if: ${{ inputs.release_type == 'testing' }} + run: | + cd ${{ github.workspace }}/action/package/ + LAST_COMMIT_SHA=$(git rev-parse dev) + TEST_VERSION=$(python setup.py --version) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + git checkout "tag/$VERSION" + git cherry-pick "$LAST_COMMIT_SHA" + CHERRYPICK_SHA=$(git rev-parse HEAD) + echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV - name: Increment Version - shell: bash + id: version + if: ${{ inputs.release_type == 'alpha' }} run: | - python action/github/scripts/bump_alpha.py - VERSION=$(python action/package/setup.py --version) + cd ${{ github.workspace }}/action/github/scripts/ + VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: "Generate release changelog" + NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: change working directory + run: cd ${{ github.workspace }}/action/package/ + - name: Generate Release Changelog (cropped) + id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Increment Version to ${{ env.VERSION }} - repository: action/package/ - - name: Change working directory to release - shell: bash - run: cd ${{ github.workspace }}/action/package/ + sinceTag: ${{ env.VERSION }} - name: Create Pre-release + id: create_release uses: ncipollo/release-action@v1 with: token: ${{ secrets.GH_PAT }} - tag: V${{ env.VERSION }} - name: Release ${{ env.VERSION }} + tag: ${{ env.NEXT_VERSION }} + name: Release ${{ env.NEXT_VERSION }} body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - commit: dev + commit: ${{ env.CHERRYPICK_SHA || 'dev' }} prerelease: true - - name: Install Build Tools - shell: bash - run: | - pip install --upgrade pip - python -m pip install build wheel - name: Build Distribution Packages - shell: bash run: | - cd action/package - python setup.py sdist bdist_wheel + cd ${{ github.workspace }}/action + mkdir -p dist && cd dist + python ../package/setup.py sdist bdist_wheel - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{secrets.PYPI_TOKEN}} - packages-dir: action/package/dist/ \ No newline at end of file + packages-dir: action/dist/ + - name: Generate Full Changelog + id: full_changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: ${{ env.VERSION_FILE }} + - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} + repository: action/package/ \ No newline at end of file diff --git a/.github/actions/release_semver_versioning/action.yml b/.github/actions/release_semver_versioning/action.yml index 8d4c41d28..ea4b8a586 100644 --- a/.github/actions/release_semver_versioning/action.yml +++ b/.github/actions/release_semver_versioning/action.yml @@ -29,25 +29,49 @@ runs: run: | RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - - name: Checkout + - name: Checkout Package uses: actions/checkout@v4 with: ref: dev fetch-depth: 0 - - name: Increment Version - shell: bash + path: action/package + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ inputs.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Common Python Requirements run: | - python action/github/scripts/bump_${{ env.RELEASE_TYPE }}.py - VERSION=$(python action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Push Version Change + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: alpha_release + uses: ./action/github/.github/actions/release_alpha + with: + action_branch: ${{ inputs.action_branch }} + python_version: ${{ inputs.python_version }} + version_file: ${{ inputs.version_file }} + changelog_file: ${{ inputs.changelog_file }} + - name: Bump to next ${{ env.RELEASE_TYPE }} version + id: version + run: | + cd ${{ github.workspace }}/action/github/scripts/ + VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --release ${{ env.RELEASE_TYPE }} --save) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: - commit_message: Increment Version to ${{ env.VERSION }} + commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ - name: Rebase testing onto dev shell: bash run: | + cd ${{ github.workspace }}/action/package/ git checkout testing git rebase dev git push origin testing --force @@ -56,10 +80,3 @@ runs: with: kickoff_pr: false release_type: ${{ env.RELEASE_TYPE }} - - name: alpha_release - uses: ./action/github/.github/actions/release_alpha - with: - action_branch: ${{ inputs.action_branch }} - python_version: ${{ inputs.python_version }} - version_file: ${{ inputs.version_file }} - changelog_file: ${{ inputs.changelog_file }} \ No newline at end of file diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index cfe454761..e00e84170 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -17,22 +17,23 @@ on: type: string default: "3.8" changelog_file: + description: 'The file location to write the changelog to' type: string default: CHANGELOG.md + # internal workflow variable + release_type: + description: 'The type of release to make' + type: string + default: alpha jobs: build_and_publish: runs-on: ubuntu-latest - permissions: - contents: write env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} + SOURCE_BRANCH: ${{ inputs.release_type == 'alpha' && 'dev' || 'testing' }} + GITHUB_REPOSITORY: ${{ github.repository }} steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: dev - path: action/package/ - name: Checkout Scripts Repo uses: actions/checkout@v4 with: @@ -43,47 +44,79 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} + - name: Install Python Dependencies + run: | + pip install --upgrade pip + pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_build_tools.txt + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_BRANCH }} + path: action/package/ + # testing specific steps. Those releases belong to the previous release cycle + # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there + - name: Checkout At Tag And Cherry Pick + if: ${{ inputs.release_type == 'testing' }} + run: | + cd ${{ github.workspace }}/action/package/ + LAST_COMMIT_SHA=$(git rev-parse dev) + TEST_VERSION=$(python setup.py --version) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + git checkout "tag/$VERSION" + git cherry-pick "$LAST_COMMIT_SHA" + CHERRYPICK_SHA=$(git rev-parse HEAD) + echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV - name: Increment Version id: version + if: ${{ inputs.release_type == 'alpha' }} run: | - python action/github/scripts/bump_alpha.py - VERSION=$(python action/package/setup.py --version) + cd ${{ github.workspace }}/action/github/scripts/ + VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: "Generate release changelog" + NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: change working directory + run: cd ${{ github.workspace }}/action/package/ + - name: Generate Release Changelog (cropped) id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Increment Version to ${{ env.VERSION }} - repository: action/package/ - - name: Change working directory to release - run: cd ${{ github.workspace }}/action/package/ + sinceTag: ${{ env.VERSION }} - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 with: token: ${{ secrets.GH_PAT }} - tag: V${{ env.VERSION }} - name: Release ${{ env.VERSION }} + tag: ${{ env.NEXT_VERSION }} + name: Release ${{ env.NEXT_VERSION }} body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - commit: dev + commit: ${{ env.CHERRYPICK_SHA || 'dev' }} prerelease: true - - name: Install Build Tools - run: | - pip install --upgrade pip - python -m pip install build wheel - name: Build Distribution Packages run: | - cd action/package - python setup.py sdist bdist_wheel + cd ${{ github.workspace }}/action + mkdir -p dist && cd dist + python ../package/setup.py sdist bdist_wheel - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{secrets.PYPI_TOKEN}} - packages-dir: action/package/dist/ + packages-dir: action/dist/ + - name: Generate Full Changelog + id: full_changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: ${{ env.VERSION_FILE }} + - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} + repository: action/package/ diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 030c7a324..969610e0c 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -56,19 +56,15 @@ jobs: - name: Install Python Dependencies run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt - name: Get Version run: | VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --repo ${{ env.GITHUB_REPOSITORY }}) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: Bump Changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - name: create release changelog (cropped) id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 @@ -80,27 +76,25 @@ jobs: uses: ncipollo/release-action@v1 with: token: ${{ secrets.GH_PAT }} - tag: V${{ env.VERSION }} + tag: ${{ env.VERSION }} name: Release ${{ env.VERSION }} body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - - name: Install Build Tools - run: | - pip install --upgrade pip - python -m pip install build wheel - name: Build Distribution Packages run: | - cd ${{ github.workspace }}/action/package - python setup.py sdist bdist_wheel + cd ${{ github.workspace }}/action + mkdir -p dist && cd dist + python ../package/setup.py sdist bdist_wheel - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{secrets.PYPI_TOKEN}} - packages-dir: action/package/dist/ - - name: Rebase dev onto master - run: | - cd ${{ github.workspace }}/action/package - git checkout dev - git rebase master - git push origin dev --force + packages-dir: action/dist/ + # NO REBASE APPROACH!! + # - name: Rebase dev onto master + # run: | + # cd ${{ github.workspace }}/action/package + # git checkout dev + # git rebase master + # git push origin dev --force diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 9e8e39fff..640d404fb 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -26,6 +26,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: testing path: action/package/ - name: Checkout Scripts Repo @@ -38,6 +39,10 @@ jobs: run: | sudo apt update xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt + - name: Authenticate GitHub CLI + run: | + unset GITHUB_TOKEN + echo "${{ secrets.GH_PAT }}" | gh auth login --with-token - name: Setup Python uses: actions/setup-python@v5 with: @@ -55,8 +60,8 @@ jobs: echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Check Latest Release run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV + LATEST_RELEASE=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type ${{ env.RELEASE_TYPE }}) + echo "LATEST_RELEASE=${LATEST_RELEASE}" >> $GITHUB_ENV - name: Bump Changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: @@ -67,18 +72,29 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} + sinceTag: ${{ env.LATEST_RELEASE }} + # if there is no master branch, create one from the tag of the latest release or first commit + - name: Create Master Branch (if necessary) + run: | + cd ${{ github.workspace }}/action/package/ + TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) + INITIAL_COMMIT=$(git rev-list --max-parents=0 dev) + if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then + if [ ${{ env.LATEST_RELEASE }} == "0.0.0" ]; then + git checkout -b ${{ env.TARGET_BRANCH }} $INITIAL_COMMIT + else + git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.LATEST_RELEASE }} + fi + git push origin ${{ env.TARGET_BRANCH }} --force + fi - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GH_PAT }} - path: action/package/ - commit-message: "ci(release): declare ${{ env.VERSION }} stable" - title: "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.VERSION }})" - body: | - Changes in this Release + run: | + cd ${{ github.workspace }}/action/package/ + gh pr create \ + --title "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.VERSION }})" \ + --body | + Included changes: ${{ steps.changelog.outputs.changelog }} - branch: ${{ env.TARGET_BRANCH }} - base: testing - draft: false - assignees: ${{ github.actor }} + --base ${{ env.TARGET_BRANCH }} \ + --head testing \ + --assignee ${{ github.actor }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 28093d4eb..b92d97f55 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -169,7 +169,7 @@ jobs: commit_message: "ci(localisation): update resource test file" repository: action/package/ branch: ${{ inputs.branch }} - # alpha release + # alpha release (+ testing) alpha_release: needs: parse_conventional_commits if: (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type)) || (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type)) || (inputs.release_type == 'alpha') || (contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) @@ -180,12 +180,11 @@ jobs: version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} changelog_file: ${{ inputs.changelog_file }} + release_type: ${{ needs.parse_conventional_commits.outputs.release_type }} # >= patch release kickoff_testing: needs: parse_conventional_commits if: (contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) || (inputs.release_type != 'alpha' && inputs.release_type != '')) - permissions: - contents: write runs-on: ubuntu-latest env: TARGET_BRANCH: 'testing' @@ -207,10 +206,10 @@ jobs: repository: OpenVoiceOS/.github ref: ${{ inputs.action_branch }} path: action/github/ - - name: Install GitHub CLI + - name: Install Common System Dependencies run: | sudo apt update - sudo apt install gh + xargs sudo apt install -y < ${{ github.workspace }}/action/github/requirements/sys_deb_common_deps.txt - name: Authenticate GitHub CLI run: | unset GITHUB_TOKEN @@ -227,7 +226,9 @@ jobs: run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - # if there is no testing branch, create one from the tag of the latest release + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --repo ${{ env.GITHUB_REPOSITORY }} --release ${{ env.RELEASE_TYPE }}) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + # if there is no testing branch, create one from the tag of the latest release or first commit - name: Create testing Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ @@ -253,8 +254,16 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ gh pr create \ - --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release)" \ - --body "This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release. \nPlease merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. \n \n(If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) \n \nIncluded changes: \n${{ steps.changelog.outputs.changelog }}" \ + --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release est. ${{ env.NEXT_VERSION}})" \ + --body | + This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release (est. ${{ env.NEXT_VERSION}}). + + Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. + + (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) + + Included changes: + ${{ steps.changelog.outputs.changelog }} --base ${{ env.TARGET_BRANCH }} \ --head dev \ --assignee ${{ github.actor }} @@ -285,8 +294,6 @@ jobs: needs: ["parse_conventional_commits", "autotranslate"] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' }} runs-on: ubuntu-latest - permissions: - contents: write steps: - name: Checkout Repository uses: actions/checkout@v4 diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 3755e55e5..bb955536b 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -46,30 +46,47 @@ jobs: run: | RELEASE_TYPE=$(echo ${{ inputs.subject }} | grep -oP '(patch|minor|major)') echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - - name: Checkout + - name: Checkout Package uses: actions/checkout@v4 with: ref: dev fetch-depth: 0 - - name: Increment Version + path: action/package + - name: Checkout Scripts Repo + uses: actions/checkout@v4 + with: + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ + - name: Set up python ${{ inputs.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Common Python Requirements + run: | + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name: Bump to next ${{ env.RELEASE_TYPE }} version id: version run: | - python action/github/scripts/bump_${{ env.RELEASE_TYPE }}.py - VERSION=$(python action/package/setup.py --version) + cd ${{ github.workspace }}/action/github/scripts/ + VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Push Version Change + NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --release ${{ env.RELEASE_TYPE }} --save) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: - commit_message: Increment Version to ${{ env.VERSION }} + commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ # fetch the latest changes from the dev branch - name: Rebase testing onto dev run: | + cd ${{ github.workspace }}/action/package/ git checkout ${{ env.SOURCE_BRANCH }} git rebase dev git push origin ${{ env.SOURCE_BRANCH }} --force - name: Notify Matrix Room - uses: ./action/github/.github/workflows/notify_testing_matrix.yml + uses: ./action/github/.github/actions/notify_testing_matrix with: kickoff_pr: false release_type: ${{ env.RELEASE_TYPE }} diff --git a/README.md b/README.md index 4e286b961..58e3762a4 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,9 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling --------- - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. - - _Testing QA_: PRs/commits go into _dev_ with the prefix/scope `fix(testing):` - - Release-tags: The tag resemble the semantic versioning (eg. `1.2.3`; no prefix!) + - General rule: PRs/commits are dev only, other branches will be protected. + - _Testing QA_: PRs/commits that directly address issues with that release need a prefix/scope `fix(testing):`. _Those_ will be part of the patch/minor/major release. The version of the `testing` branch is to be held steady. A fix for later releases has to be commited as usual with `fix: ...`. + - Release-tags: The tag resembles the semantic versioning (eg. `1.2.3`; no prefix!, alphas will be formatted appropriately) TODO (per repo): - `setup.py`: setuptools cant cope with semver compliance: `x.x.x-alpha...` stays `x.x.xax` for now diff --git a/requirements/pip_build_tools.txt b/requirements/pip_build_tools.txt new file mode 100644 index 000000000..97123c753 --- /dev/null +++ b/requirements/pip_build_tools.txt @@ -0,0 +1,2 @@ +build +wheel diff --git a/requirements/sys_deb_common_deps.txt b/requirements/sys_deb_common_deps.txt index 5d168bf04..5ee2b78c8 100644 --- a/requirements/sys_deb_common_deps.txt +++ b/requirements/sys_deb_common_deps.txt @@ -8,3 +8,4 @@ libpulse-dev libicu-dev pkg-config libenchant-2-2 +gh diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index ee3486214..e9ec71bb2 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -1,128 +1,374 @@ from os import getenv from os.path import isfile import re +from typing import SupportsInt, Optional, Union, List from github import Github +from github.GitRelease import GitRelease import semver import argparse """ -Get the first, last, latest or next release versions of the release cycle -for the current release type. (patch, minor, major) +This script is supposed to parse a github release history and get latest (or first / last / next) +release versions for the specified release type. (patch, minor, major, prerelease) +If no release type is specified, get the latest/.. release version. + +You can make this relative to a specific version by providing a version or file to read the version from. +eg. `... --version 0.2.1 --latest --type alpha` will get the latest alpha release below version 0.2.1 +or `... --file path/to/version.py --next` will get the next release version from the version file. + +Options + cycle # restrict the release type to the current cycle + # (i.e. if the current version is 2.1.2 and `--type patch`, it will only consider 2.1.x releases) Args - --alpha_marker: str # marker for alpha releases, default is 'a' - --repo: str # the repository to get the releases from --type: str # the release type to get the version for - --output_file: str # writes an ovos version file to the specified path + --repo: str # the github repo to get the version from + --file: str # the file to read the version from + --save: str # writes an ovos version file to the specified path (if the version is above the latest) + if no path is provided and read from a file, the file will be overwritten + --fsave: str # force the ovos version file to be written even if the version is below the latest + + --alpha_marker: str # marker for alpha releases, default is 'a' Flags --last # get the last release of that release type in the current cycle --next # get the next release of the upcoming release type - --latest # get the latest version released - none of above # get the first release of that release type in the current cycle + --first # get the first release of that release type in the current cycle + none of above # get the latest version released """ -parser = argparse.ArgumentParser() -parser.add_argument("--alpha_marker", default="a") -parser.add_argument("--repo", required=True) -parser.add_argument("--type", choices=["patch", "minor", "major", "alpha", "prerelease"]) -parser.add_argument("--output_file") +def add_common_arguments(parser): + parser.add_argument("--alpha_marker", default="a") + parser.add_argument("--type", choices=["patch", "minor", "major", "alpha", "prerelease"]) + parser.add_argument("--save", nargs='?', const=True, default=False) + parser.add_argument("--fsave", nargs='?', const=True, default=False) -release_group = parser.add_mutually_exclusive_group() -release_group.add_argument("--last", action='store_true') -release_group.add_argument("--next", action='store_true') -release_group.add_argument("--latest", action='store_true') + release_group = parser.add_mutually_exclusive_group() + release_group.add_argument("--last", action='store_true') + release_group.add_argument("--next", action='store_true') + release_group.add_argument("--latest", action='store_true') + release_group.add_argument("--first", action='store_true') -args = parser.parse_args() + input_group = parser.add_mutually_exclusive_group() + #input_group.add_argument("--repo") + input_group.add_argument("--file") + input_group.add_argument("--version") + + +parser = argparse.ArgumentParser() +add_common_arguments(parser) + +subparsers = parser.add_subparsers(dest="command") +cycle_parser = subparsers.add_parser("cycle") +add_common_arguments(cycle_parser) +args = parser.parse_args() RELEASE_TYPE = args.type -# old habits -if RELEASE_TYPE == "alpha" or RELEASE_TYPE is None: - RELEASE_TYPE = "prerelease" - -if not args.latest and not RELEASE_TYPE: - raise ValueError("Missing release type. Please set the `--type` argument") - -if args.output_file and not isfile(args.output_file): - raise ValueError(f"Output file does not exist: {args.output_file}") - -# get the repo -repo = Github(getenv("GH_PAT")).get_repo(args.repo) -latest_version = None -release_id = None - -def get_semver(tag: str) -> semver.Version: - # hack for v prefix - tag = tag.lstrip("v").lstrip("V") - - # hack for alpha releases - if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): - tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) - - if not semver.Version.is_valid(tag): - return None - return semver.Version.parse(tag) - - -def bump_version(v: semver.Version) -> semver.Version: - return to_pypi_format(v.next_version(RELEASE_TYPE, args.alpha_marker)) - - -def to_pypi_format(v: semver.Version) -> str: - return f"{v.major}.{v.minor}.{v.patch}{v.prerelease.replace('.', '') if v.prerelease else ''}" - - -def in_cycle(v: semver.Version) -> bool: - if RELEASE_TYPE == "patch": - return v.patch == latest_version.patch and not \ - (v.minor < latest_version.minor) and not \ - (v.major < latest_version.major) - elif RELEASE_TYPE == "minor": - return v.minor == latest_version.minor and not \ - (v.major < latest_version.major) - elif RELEASE_TYPE == "major": - return v.major == latest_version.major - -# get the release history -releases = repo.get_releases() -if not releases: - semv = semver.Version.parse("0.0.0") - if args.next: - semv = semv.next_version(RELEASE_TYPE, args.alpha_marker) -elif not args.next and not args.latest: - for id, release in enumerate(releases): - version = get_semver(release.tag_name) - if version and in_cycle(version) and RELEASE_TYPE in ["patch", "minor", "major"]: - release_id = id -elif args.next: - latest_version = get_semver(releases[0].tag_name) - semv = latest_version.next_version(RELEASE_TYPE, args.alpha_marker) -elif args.latest: - semv = get_semver(releases[0].tag_name) - release_id = 0 - -if not args.last and release_id: - release_id -= 1 - -if release_id is not None: - semv = get_semver(releases[release_id].tag_name) - -if args.output_file: - version_block = f""" -# START_VERSION_BLOCK -VERSION_MAJOR = {semv.major} -VERSION_MINOR = {semv.minor} -VERSION_BUILD = {semv.patch} -VERSION_ALPHA = {semv.prerelease.replace(args.alpha_marker, '').replace(".", "") if semv.prerelease else 0} +ALPHA_MARKER = args.alpha_marker +REPOSITORY = getenv("GITHUB_REPOSITORY") +RESTRICT_TO_CYCLE = args.command == "cycle" + +if REPOSITORY is None and not (args.file or args.version): + parser.error("either set up an environmental variable `GITHUB_REPOSITORY` or pass --version or --file as arguments") + + +class OVOSReleases(semver.Version): + __history = [] + __github_token = None + __repo = None + __release = None + + def __init__(self, major: SupportsInt = 0, + minor: SupportsInt = 0, + patch: SupportsInt = 0, + prerelease: Optional[Union[str, int]] = None, + build: Optional[Union[str, int]] = None, + release: Optional[GitRelease] = None): + self.__release = release + if isinstance(release, GitRelease): + ver = self.parse(release.tag_name) + major = ver.major + minor = ver.minor + patch = ver.patch + prerelease = ver.prerelease + build = ver.build + + super().__init__(major, minor, patch, prerelease, build) + + def next(self, rtype: Optional[str], alpha_marker: str = ALPHA_MARKER)\ + -> Optional["OVOSReleases"]: + rtype = rtype or "prerelease" + + return self.next_version(rtype, alpha_marker) + + def latest(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + if rtype and not self.history: + raise ValueError("No release history") + + release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) + + latest_version = None + if rtype is None and (not self.history or not release_versions): + latest_version = self + elif release_versions: + latest_version = max(release_versions) + + return latest_version + + def last(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + if not self.history: + raise ValueError("No release history") + + release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) + + last_version = None + if release_versions: + last_version = release_versions[min(1, len(release_versions) - 1)] + + return last_version + + def first(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + if not self.history: + raise ValueError("No release history") + + release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) + + first_version = None + if release_versions: + first_version = min(release_versions) + + return first_version + + @property + def history(self) -> List["OVOSReleases"]: + """ + Returns the release history of the github repository + """ + return self.__history + + @property + def released(self) -> bool: + """ + Returns whether the release is already released + """ + return self.__release is not None + + @property + def tag(self) -> str: + """ + Returns the tag of the release + """ + if not self.released: + return None + return self.__release.tag_name + + @property + def etag(self) -> str: + """ + Returns the etag (MD5 hash of the content) of the release + """ + if not self.released: + return None + return self.__release.etag + + @property + def release_url(self) -> str: + """ + Returns the url of the release + """ + if not self.released: + return None + return self.__release.url + + @property + def tarball_url(self) -> str: + """ + Returns the tarball url of the release + """ + if not self.released: + return None + return self.__release.tarball_url + + @property + def title(self) -> str: + """ + Returns the release title + """ + if not self.released: + return None + return self.__release.title + + @property + def body(self) -> str: + """ + Returns the release description + """ + if not self.released: + return None + return self.__release.body + + @classmethod + def from_file(cls, path: str) -> Optional["OVOSReleases"]: + with open(path, "r") as f: + data = f.read() + data = re.search(r"# START_VERSION_BLOCK(.*?)# END_VERSION_BLOCK", data, re.DOTALL) + if data: + data = data.group(1) + data = re.findall(r"VERSION_(\w+) = (\d+)", data) + data = {k: int(v) for k, v in data} + if data["ALPHA"]: + data["ALPHA"] = f"{ALPHA_MARKER}.{data['ALPHA']}" + else: + data["ALPHA"] = None + return cls(data["MAJOR"], data["MINOR"], data["BUILD"], data["ALPHA"]) + + @classmethod + def from_repo(cls, repo: Optional[str], token: Optional[str] = None) -> "OVOSReleases": + cls.__github_token = token + cls.__repo = repo + + releases = [] + if repo: + git = Github(token).get_repo(repo) + releases = git.get_releases() + if not releases: + return OVOSReleases(0, 0, 0) + + cls.__history = [OVOSReleases(release=release) for release in releases + if OVOSReleases.parse(release.tag_name) is not None] + return cls.__history[0] + + @classmethod + def from_list(cls, history: List[str]) -> "OVOSReleases": + cls.__history = [cls.parse(tag) for tag in history] + return cls.__history[0] + + def to_file(self, path: str) -> None: + """ + Saves the version to the specified file using + the ovos version format specification + """ + with open(path, "w") as f: + f.write(f"""# START_VERSION_BLOCK +VERSION_MAJOR = {self.major} +VERSION_MINOR = {self.minor} +VERSION_BUILD = {self.patch} +VERSION_ALPHA = {self.prerelease.replace(ALPHA_MARKER, '').replace('.', '') if self.prerelease else 0} # END_VERSION_BLOCK -""" - with open(args.output_file, "w") as f: - f.write(version_block) +""") + + def to_pypi_format(self) -> str: + return f"{self.major}.{self.minor}.{self.patch}{self.prerelease.replace('.', '') if self.prerelease else ''}" + + @staticmethod + def parse(tag: str) -> semver.Version: + # hack for v prefix + tag = tag.lstrip("v").lstrip("V") + + # hack for alpha releases + if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): + tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) + + if not semver.Version.is_valid(tag): + return None + + ver = semver.Version.parse(tag) + return OVOSReleases(ver.major, ver.minor, ver.patch, ver.prerelease, ver.build) + + def get(self, version: Optional[str], file: Optional[str]) -> "OVOSReleases": + if version: + version = OVOSReleases.parse(version) + for v in self.history: + if v.compare(version) == 0: + return v + return version + elif file: + version = OVOSReleases.from_file(file) + for v in self.history: + if v.compare(version) == 0: + return v + return version + + def filter_versions(self, release_type: Optional[str] = None, cycle_only: bool = False) -> List[semver.Version]: + """ + Returns the release versions of the specified release type -if release_id is not None: - print(releases[release_id].tag_name) + If cycle only restrict versions to the given cycle + (i.e. if self is 2.1.2, it will return all 2.1.x releases if release_type is patch, + normally: 2.1.2 (latest), 2.1.1 (last/first), depending on the release history) + """ + + if release_type: + release_type = release_type.lower() + filtered_versions = [] + if release_type == 'major': + filtered_versions = [v for v in self.history + if v <= self + and v.major != 0 + and v.minor == 0 + and v.patch == 0 + and v.prerelease is None] + if cycle_only: + filtered_versions = filter(lambda v: v.major == self.major, filtered_versions) + elif release_type == 'minor': + filtered_versions = [v for v in self.history + if v <= self + and v.minor != 0 + and v.patch == 0 + and v.prerelease is None] + if cycle_only: + filtered_versions = filter(lambda v: v.major == self.major, filtered_versions) + elif release_type == 'patch': + filtered_versions = [v for v in self.history + if v <= self and v.patch != 0 and v.prerelease is None] + if cycle_only: + filtered_versions = filter(lambda v: v.major == self.major and v.minor == self.minor, filtered_versions) + elif release_type in ["prerelease", "alpha"]: + filtered_versions = [v for v in self.history + if v <= self and v.prerelease is not None] + if cycle_only: + filtered_versions = filter(lambda v: v.major == self.major and v.minor == self.minor and v.patch == self.patch, filtered_versions) + elif release_type is None: + filtered_versions = self.history[:] + if cycle_only: + filtered_versions = filter(lambda v: v.major == self.major, filtered_versions) + else: + raise ValueError('Invalid release type') + + return sorted(filtered_versions, reverse=True) + + +# instanciate the class history +releases = OVOSReleases.from_repo(REPOSITORY, getenv("GH_PAT")) +# if version or file is provided, get the version from the repository history or use the provided version +if args.version or args.file: + base = releases.get(args.version, args.file) else: - print(to_pypi_format(semv)) + base = releases.latest() + +# output handling +if (args.save is True or args.fsave is True) and not args.file: + arg = "--save" + if args.fsave: + arg = "--fsave" + raise ValueError(f"No file specified to save the version to (ie {arg} /path/to/version.py)") + +if args.first: + version = base.first(RELEASE_TYPE) +elif args.last: + version = base.last(RELEASE_TYPE) +elif args.next: + version = base.next(RELEASE_TYPE) +else: + version = base.latest(RELEASE_TYPE) + +if (args.save or args.fsave) and version is not None: + file = args.file or args.save or args.fsave + if version > base or args.fsave: + version.to_file(file) + +if version is not None: + print(version.to_pypi_format()) From 174e6851598c5bfc5d5dc75448088ecd313861cd Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 17:17:29 +0100 Subject: [PATCH 195/357] tests --- .github/workflows/release_semver_publish.yml | 2 +- .github/workflows/release_semver_start.yml | 4 ++-- scripts/semver_release_version.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 969610e0c..d1c70d366 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -63,7 +63,7 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Check Latest Release run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --repo ${{ env.GITHUB_REPOSITORY }}) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - name: create release changelog (cropped) id: changelog diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index b92d97f55..1f21acbf9 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -224,9 +224,9 @@ jobs: - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --repo ${{ env.GITHUB_REPOSITORY }}) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --repo ${{ env.GITHUB_REPOSITORY }} --release ${{ env.RELEASE_TYPE }}) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --release ${{ env.RELEASE_TYPE }}) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV # if there is no testing branch, create one from the tag of the latest release or first commit - name: Create testing Branch (if necessary) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index e9ec71bb2..9ff59fa93 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -23,8 +23,9 @@ Args --type: str # the release type to get the version for - --repo: str # the github repo to get the version from --file: str # the file to read the version from + --version: str # the version to get the release for + --save: str # writes an ovos version file to the specified path (if the version is above the latest) if no path is provided and read from a file, the file will be overwritten --fsave: str # force the ovos version file to be written even if the version is below the latest @@ -52,7 +53,6 @@ def add_common_arguments(parser): release_group.add_argument("--first", action='store_true') input_group = parser.add_mutually_exclusive_group() - #input_group.add_argument("--repo") input_group.add_argument("--file") input_group.add_argument("--version") From 40322a1c3861c597d2cc6cb90f14f970a6d18089 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 17:38:25 +0100 Subject: [PATCH 196/357] tests --- .github/actions/release_semver_versioning/action.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- .github/workflows/release_semver_versioning.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/release_semver_versioning/action.yml b/.github/actions/release_semver_versioning/action.yml index ea4b8a586..04b0e8624 100644 --- a/.github/actions/release_semver_versioning/action.yml +++ b/.github/actions/release_semver_versioning/action.yml @@ -61,7 +61,7 @@ runs: cd ${{ github.workspace }}/action/github/scripts/ VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --release ${{ env.RELEASE_TYPE }} --save) + NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 1f21acbf9..a47947845 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -226,7 +226,7 @@ jobs: run: | SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest) echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --release ${{ env.RELEASE_TYPE }}) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --type ${{ env.RELEASE_TYPE }}) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV # if there is no testing branch, create one from the tag of the latest release or first commit - name: Create testing Branch (if necessary) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index bb955536b..ba4abadc4 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -71,7 +71,7 @@ jobs: cd ${{ github.workspace }}/action/github/scripts/ VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --release ${{ env.RELEASE_TYPE }} --save) + NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 From c2931b11c255b1bc06d01d1b0dd2421545a8b34f Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 18:33:27 +0100 Subject: [PATCH 197/357] tests --- scripts/semver_release_version.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index 9ff59fa93..dca7d9e69 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -53,6 +53,7 @@ def add_common_arguments(parser): release_group.add_argument("--first", action='store_true') input_group = parser.add_mutually_exclusive_group() + #input_group.add_argument("--repo") input_group.add_argument("--file") input_group.add_argument("--version") @@ -80,6 +81,7 @@ class OVOSReleases(semver.Version): __github_token = None __repo = None __release = None + __prefix = "" def __init__(self, major: SupportsInt = 0, minor: SupportsInt = 0, @@ -89,6 +91,7 @@ def __init__(self, major: SupportsInt = 0, release: Optional[GitRelease] = None): self.__release = release if isinstance(release, GitRelease): + self.__prefix = re.match(r"([a-zA-Z]+)?", release.tag_name).group(1) ver = self.parse(release.tag_name) major = ver.major minor = ver.minor @@ -97,6 +100,9 @@ def __init__(self, major: SupportsInt = 0, build = ver.build super().__init__(major, minor, patch, prerelease, build) + + def __str__(self) -> str: + return f"{self.__prefix}{super().__str__()}" def next(self, rtype: Optional[str], alpha_marker: str = ALPHA_MARKER)\ -> Optional["OVOSReleases"]: @@ -261,12 +267,12 @@ def to_file(self, path: str) -> None: """) def to_pypi_format(self) -> str: - return f"{self.major}.{self.minor}.{self.patch}{self.prerelease.replace('.', '') if self.prerelease else ''}" + return f"{self.__prefix}{self.major}.{self.minor}.{self.patch}{self.prerelease.replace('.', '') if self.prerelease else ''}" @staticmethod def parse(tag: str) -> semver.Version: - # hack for v prefix - tag = tag.lstrip("v").lstrip("V") + # remove prefix from tag + tag = re.sub(r"([a-zA-Z]+)?", "", tag) # hack for alpha releases if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): From 1fad69850139778b62d464e7488d41fa4773550d Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 18:48:06 +0100 Subject: [PATCH 198/357] tests --- .github/workflows/release_semver_pull_master.yml | 4 +++- .github/workflows/release_semver_start.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 640d404fb..51e580965 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -92,9 +92,11 @@ jobs: cd ${{ github.workspace }}/action/package/ gh pr create \ --title "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.VERSION }})" \ - --body | + --body "$(cat << EOF Included changes: ${{ steps.changelog.outputs.changelog }} + EOF + )" \ --base ${{ env.TARGET_BRANCH }} \ --head testing \ --assignee ${{ github.actor }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index a47947845..2a79cb08d 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -255,7 +255,7 @@ jobs: cd ${{ github.workspace }}/action/package/ gh pr create \ --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release est. ${{ env.NEXT_VERSION}})" \ - --body | + --body "$(cat << EOF This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release (est. ${{ env.NEXT_VERSION}}). Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. @@ -264,6 +264,8 @@ jobs: Included changes: ${{ steps.changelog.outputs.changelog }} + EOF + )" \ --base ${{ env.TARGET_BRANCH }} \ --head dev \ --assignee ${{ github.actor }} From c8c8e2a680892d5976d58fd1e9ddc291e44168fa Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 19:34:25 +0100 Subject: [PATCH 199/357] tests --- .github/actions/release_alpha/action.yml | 4 ++-- .github/workflows/release_alpha.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index bda5ee8d7..577a211a8 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -37,8 +37,8 @@ runs: - name: Install Python Dependencies run: | pip install --upgrade pip - pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_base.txt - pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_build_tools.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt - name: Checkout Repository uses: actions/checkout@v4 with: diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index e00e84170..7f97e28b5 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -47,8 +47,8 @@ jobs: - name: Install Python Dependencies run: | pip install --upgrade pip - pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_base.txt - pip install -r ${{ github.workspace }}/action/github/scripts/requirements/pip_build_tools.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt - name: Checkout Repository uses: actions/checkout@v4 with: From 1bb2e771dc11d9892336408b5fc9069b78ce6168 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 19:49:03 +0100 Subject: [PATCH 200/357] tests --- scripts/semver_release_version.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index dca7d9e69..fe736ec09 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -75,6 +75,8 @@ def add_common_arguments(parser): if REPOSITORY is None and not (args.file or args.version): parser.error("either set up an environmental variable `GITHUB_REPOSITORY` or pass --version or --file as arguments") +if RELEASE_TYPE == "alpha": + RELEASE_TYPE = "prerelease" class OVOSReleases(semver.Version): __history = [] From 2f96d1f1c0f4980d54651e36ce77d938fd5f84b4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 20:21:30 +0100 Subject: [PATCH 201/357] tests --- .github/actions/release_alpha/action.yml | 6 +++--- .github/workflows/release_alpha.yml | 6 +++--- .github/workflows/release_semver_publish.yml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 577a211a8..805e9f29c 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -91,9 +91,9 @@ runs: prerelease: true - name: Build Distribution Packages run: | - cd ${{ github.workspace }}/action - mkdir -p dist && cd dist - python ../package/setup.py sdist bdist_wheel + mkdir -p ${{ github.workspace }}/action/dist + cd ${{ github.workspace }}/action/package/ + python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 7f97e28b5..adaebd24c 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -101,9 +101,9 @@ jobs: prerelease: true - name: Build Distribution Packages run: | - cd ${{ github.workspace }}/action - mkdir -p dist && cd dist - python ../package/setup.py sdist bdist_wheel + mkdir -p ${{ github.workspace }}/action/dist + cd ${{ github.workspace }}/action/package/ + python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index d1c70d366..4c291848b 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -83,9 +83,9 @@ jobs: ${{ steps.changelog.outputs.changelog }} - name: Build Distribution Packages run: | - cd ${{ github.workspace }}/action - mkdir -p dist && cd dist - python ../package/setup.py sdist bdist_wheel + mkdir -p ${{ github.workspace }}/action/dist + cd ${{ github.workspace }}/action/package/ + python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: From 68411e6716cd450ed6de4a8030edf39d654e6162 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 20:35:04 +0100 Subject: [PATCH 202/357] tests --- .github/workflows/release_semver_publish.yml | 2 +- .github/workflows/release_semver_versioning.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 4c291848b..0e8e5e14a 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -35,7 +35,7 @@ jobs: steps: - name: set up release type run: | - RELEASE_TYPE=$(echo ${{ inputs.subject }} | grep -oP '(patch|minor|major)') + RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Checkout Repository uses: actions/checkout@v4 diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index ba4abadc4..8128d64ea 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -44,7 +44,7 @@ jobs: steps: - name: set up release type run: | - RELEASE_TYPE=$(echo ${{ inputs.subject }} | grep -oP '(patch|minor|major)') + RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Checkout Package uses: actions/checkout@v4 From c0ffefdccfe5f07df4558519db228c6db7f6263a Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 21:54:58 +0100 Subject: [PATCH 203/357] tests --- .github/workflows/release_semver_start.yml | 9 ++++++++- scripts/semver_release_version.py | 10 ++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2a79cb08d..2b0674533 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -224,7 +224,14 @@ jobs: - name: Check Latest Release if: ${{ inputs.kickoff_pr == 'true' }} run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest) + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py cycle --latest --type ${{ env.RELEASE_TYPE }}) + if [ -z "$SINCE_TAG" ]; then + if [ "${{ env.RELEASE_TYPE }}" == "patch" ]; then + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type minor) + elif [ "${{ env.RELEASE_TYPE }}" == "minor" ]; then + SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type major) + fi + fi echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --type ${{ env.RELEASE_TYPE }}) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index fe736ec09..5caecbbdc 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -75,8 +75,6 @@ def add_common_arguments(parser): if REPOSITORY is None and not (args.file or args.version): parser.error("either set up an environmental variable `GITHUB_REPOSITORY` or pass --version or --file as arguments") -if RELEASE_TYPE == "alpha": - RELEASE_TYPE = "prerelease" class OVOSReleases(semver.Version): __history = [] @@ -93,7 +91,7 @@ def __init__(self, major: SupportsInt = 0, release: Optional[GitRelease] = None): self.__release = release if isinstance(release, GitRelease): - self.__prefix = re.match(r"([a-zA-Z]+)?", release.tag_name).group(1) + self.__prefix = re.match(r"^([a-zA-Z-\/\\]+)?", release.tag_name).group(1) or "" ver = self.parse(release.tag_name) major = ver.major minor = ver.minor @@ -274,11 +272,11 @@ def to_pypi_format(self) -> str: @staticmethod def parse(tag: str) -> semver.Version: # remove prefix from tag - tag = re.sub(r"([a-zA-Z]+)?", "", tag) + tag = re.sub(r"^([a-zA-Z-\/\\]+)?", "", tag) # hack for alpha releases - if re.match(r"[0-9]+\.[0-9]+\.[0-9]+a[0-9]+", tag): - tag = re.sub(r"([0-9]+)(a[0-9]+)", r"\1-\2", tag) + if re.match(rf"[0-9]+\.[0-9]+\.[0-9]+{ALPHA_MARKER}[0-9]+", tag): + tag = re.sub(rf"([0-9]+){ALPHA_MARKER}([0-9]+)", rf"\1-{ALPHA_MARKER}.\2", tag) if not semver.Version.is_valid(tag): return None From a4a17d7e5ce1f0ac19959cea0057f2e24cf87416 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 22:15:44 +0100 Subject: [PATCH 204/357] tests --- .github/actions/release_alpha/action.yml | 2 +- .github/workflows/release_alpha.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 805e9f29c..38dc35a41 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -104,7 +104,7 @@ runs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: ${{ env.VERSION_FILE }} + output: ${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index adaebd24c..90d9bf0e3 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -114,7 +114,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: ${{ env.VERSION_FILE }} + output: ${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: From 7f36f1bf2fd71bba9563f957cc30a24ac14fdc84 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 22:30:31 +0100 Subject: [PATCH 205/357] tests --- scripts/bump_alpha.py | 21 --------------------- scripts/bump_major.py | 30 ------------------------------ scripts/bump_minor.py | 27 --------------------------- scripts/bump_patch.py | 24 ------------------------ scripts/remove_alpha.py | 15 --------------- scripts/semver_release_version.py | 3 +++ 6 files changed, 3 insertions(+), 117 deletions(-) delete mode 100644 scripts/bump_alpha.py delete mode 100644 scripts/bump_major.py delete mode 100644 scripts/bump_minor.py delete mode 100644 scripts/bump_patch.py delete mode 100644 scripts/remove_alpha.py diff --git a/scripts/bump_alpha.py b/scripts/bump_alpha.py deleted file mode 100644 index 37e562c37..000000000 --- a/scripts/bump_alpha.py +++ /dev/null @@ -1,21 +0,0 @@ -import fileinput -import os - - -version_file = os.getenv("VERSION_FILE") -if not os.path.isfile(version_file): - raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") - -version_var_name = "VERSION_ALPHA" - -with open(version_file, "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith(version_var_name): - version = int(line.split("=")[-1]) - new_version = int(version) + 1 - -for line in fileinput.input(version_file, inplace=True): - if line.startswith(version_var_name): - print(f"{version_var_name} = {new_version}") - else: - print(line.rstrip('\n')) diff --git a/scripts/bump_major.py b/scripts/bump_major.py deleted file mode 100644 index 52f750fa8..000000000 --- a/scripts/bump_major.py +++ /dev/null @@ -1,30 +0,0 @@ -import fileinput -import os - - -version_file = os.getenv("VERSION_FILE") -if not os.path.isfile(version_file): - raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") - -version_var_name = "VERSION_MAJOR" -minor_var_name = "VERSION_MINOR" -build_var_name = "VERSION_BUILD" -alpha_var_name = "VERSION_ALPHA" - -with open(version_file, "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith(version_var_name): - version = int(line.split("=")[-1]) - new_version = int(version) + 1 - -for line in fileinput.input(version_file, inplace=True): - if line.startswith(version_var_name): - print(f"{version_var_name} = {new_version}") - elif line.startswith(minor_var_name): - print(f"{minor_var_name} = 0") - elif line.startswith(build_var_name): - print(f"{build_var_name} = 0") - elif line.startswith(alpha_var_name): - print(f"{alpha_var_name} = 0") - else: - print(line.rstrip('\n')) diff --git a/scripts/bump_minor.py b/scripts/bump_minor.py deleted file mode 100644 index fe61cc1a3..000000000 --- a/scripts/bump_minor.py +++ /dev/null @@ -1,27 +0,0 @@ -import fileinput -import os - - -version_file = os.getenv("VERSION_FILE") -if not os.path.isfile(version_file): - raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") - -version_var_name = "VERSION_MINOR" -build_var_name = "VERSION_BUILD" -alpha_var_name = "VERSION_ALPHA" - -with open(version_file, "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith(version_var_name): - version = int(line.split("=")[-1]) - new_version = int(version) + 1 - -for line in fileinput.input(version_file, inplace=True): - if line.startswith(version_var_name): - print(f"{version_var_name} = {new_version}") - elif line.startswith(build_var_name): - print(f"{build_var_name} = 0") - elif line.startswith(alpha_var_name): - print(f"{alpha_var_name} = 0") - else: - print(line.rstrip('\n')) diff --git a/scripts/bump_patch.py b/scripts/bump_patch.py deleted file mode 100644 index f7f7091a9..000000000 --- a/scripts/bump_patch.py +++ /dev/null @@ -1,24 +0,0 @@ -import fileinput -import os - - -version_file = os.getenv("VERSION_FILE") -if not os.path.isfile(version_file): - raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") - -version_var_name = "VERSION_BUILD" -alpha_var_name = "VERSION_ALPHA" - -with open(version_file, "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith(version_var_name): - version = int(line.split("=")[-1]) - new_version = int(version) + 1 - -for line in fileinput.input(version_file, inplace=True): - if line.startswith(version_var_name): - print(f"{version_var_name} = {new_version}") - elif line.startswith(alpha_var_name): - print(f"{alpha_var_name} = 0") - else: - print(line.rstrip('\n')) diff --git a/scripts/remove_alpha.py b/scripts/remove_alpha.py deleted file mode 100644 index a33968e2b..000000000 --- a/scripts/remove_alpha.py +++ /dev/null @@ -1,15 +0,0 @@ -import fileinput -import os - - -version_file = os.getenv("VERSION_FILE") -if not os.path.isfile(version_file): - raise ValueError(f"environment variable `VERSION_FILE` is not a file: {version_file}") - -alpha_var_name = "VERSION_ALPHA" - -for line in fileinput.input(version_file, inplace=True): - if line.startswith(alpha_var_name): - print(f"{alpha_var_name} = 0") - else: - print(line.rstrip('\n')) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index 5caecbbdc..675d3d2a7 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -75,7 +75,10 @@ def add_common_arguments(parser): if REPOSITORY is None and not (args.file or args.version): parser.error("either set up an environmental variable `GITHUB_REPOSITORY` or pass --version or --file as arguments") +if RELEASE_TYPE == "alpha": + RELEASE_TYPE = "prerelease" + class OVOSReleases(semver.Version): __history = [] __github_token = None From a745f15db5dd1a186e59c0a126a1debba7324b55 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 23:06:57 +0100 Subject: [PATCH 206/357] tests --- .../workflows/release_semver_versioning.yml | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 8128d64ea..e64894baf 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -25,7 +25,25 @@ on: required: true jobs: + synchronize_testing: + runs-on: ubuntu-latest + steps: + - name: Checkout Package + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Synchronize testing with dev + run: | + git checkout testing + git pull origin dev + git push origin testing + - name: Synchronize dev with testing + run: | + git checkout dev + git pull origin testing + git push origin dev alpha_release: + needs: synchronize_testing secrets: inherit uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 with: @@ -34,7 +52,7 @@ jobs: version_file: ${{ inputs.version_file }} changelog_file: ${{ inputs.changelog_file }} bump_version_and_announce_testing: - needs: alpha_release + needs: [alpha_release, synchronize_testing] env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: 'testing' @@ -78,13 +96,6 @@ jobs: with: commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ - # fetch the latest changes from the dev branch - - name: Rebase testing onto dev - run: | - cd ${{ github.workspace }}/action/package/ - git checkout ${{ env.SOURCE_BRANCH }} - git rebase dev - git push origin ${{ env.SOURCE_BRANCH }} --force - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From c4a49759c328d879a5548cdb5eef0ac7f7fd9227 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 23:17:47 +0100 Subject: [PATCH 207/357] tests --- .../workflows/release_semver_versioning.yml | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index e64894baf..57986e30c 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -25,25 +25,7 @@ on: required: true jobs: - synchronize_testing: - runs-on: ubuntu-latest - steps: - - name: Checkout Package - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Synchronize testing with dev - run: | - git checkout testing - git pull origin dev - git push origin testing - - name: Synchronize dev with testing - run: | - git checkout dev - git pull origin testing - git push origin dev alpha_release: - needs: synchronize_testing secrets: inherit uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 with: @@ -52,7 +34,7 @@ jobs: version_file: ${{ inputs.version_file }} changelog_file: ${{ inputs.changelog_file }} bump_version_and_announce_testing: - needs: [alpha_release, synchronize_testing] + needs: alpha_release env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: 'testing' @@ -96,6 +78,17 @@ jobs: with: commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ + - name: Synchronize testing with dev + run: | + cd ${{ github.workspace }}/action/package + git checkout testing + git pull origin dev + git push origin testing + - name: Synchronize dev with testing + run: | + git checkout dev + git pull origin testing + git push origin dev - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From 9e0dda9807ad5223dfde62371bcc70e3bb1c3979 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 6 Mar 2024 23:49:06 +0100 Subject: [PATCH 208/357] tests --- .github/workflows/release_semver_versioning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 57986e30c..877006600 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -82,12 +82,12 @@ jobs: run: | cd ${{ github.workspace }}/action/package git checkout testing - git pull origin dev + git pull --rebase origin dev git push origin testing - name: Synchronize dev with testing run: | git checkout dev - git pull origin testing + git pull --rebase origin testing git push origin dev - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix From 917788b63c40e2cde52696020458d7b17dae13cf Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 00:19:35 +0100 Subject: [PATCH 209/357] tests --- .../workflows/release_semver_versioning.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 877006600..08a5faaec 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -78,16 +78,29 @@ jobs: with: commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ + - name: Checkout Testing + uses: actions/checkout@v4 + with: + ref: testing + fetch-depth: 0 + path: action/package - name: Synchronize testing with dev run: | cd ${{ github.workspace }}/action/package - git checkout testing - git pull --rebase origin dev + git pull --rebase=merges dev + if [ $? -ne 0 ]; then + git rebase --abort + git rebase -Xtheirs dev + fi git push origin testing - name: Synchronize dev with testing run: | git checkout dev - git pull --rebase origin testing + git pull --rebase=merges testing + if [ $? -ne 0 ]; then + git rebase --abort + git rebase -Xours testing + fi git push origin dev - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix From cf9d9c3521334242cee73911522f8a4252619eec Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 00:30:50 +0100 Subject: [PATCH 210/357] tests --- .github/workflows/release_semver_versioning.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 08a5faaec..b416ec1fd 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -87,7 +87,7 @@ jobs: - name: Synchronize testing with dev run: | cd ${{ github.workspace }}/action/package - git pull --rebase=merges dev + git rebase dev if [ $? -ne 0 ]; then git rebase --abort git rebase -Xtheirs dev @@ -96,7 +96,8 @@ jobs: - name: Synchronize dev with testing run: | git checkout dev - git pull --rebase=merges testing + git fetch origin + git rebase testing if [ $? -ne 0 ]; then git rebase --abort git rebase -Xours testing From 0a6a87a13f53f8a13374ff689dd60b4a40bd76d8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 00:33:39 +0100 Subject: [PATCH 211/357] tests --- .github/workflows/release_semver_versioning.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index b416ec1fd..b0ddc0bb8 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -49,6 +49,7 @@ jobs: - name: Checkout Package uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: dev fetch-depth: 0 path: action/package @@ -81,6 +82,7 @@ jobs: - name: Checkout Testing uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: testing fetch-depth: 0 path: action/package From cea9f6b3579deff4683e3ca429f8148e9db8ee8c Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 00:46:19 +0100 Subject: [PATCH 212/357] tests --- .github/workflows/release_semver_versioning.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index b0ddc0bb8..6b25eee46 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -89,6 +89,10 @@ jobs: - name: Synchronize testing with dev run: | cd ${{ github.workspace }}/action/package + git checkout dev + git pull origin dev + git checkout testing + git pull origin testing git rebase dev if [ $? -ne 0 ]; then git rebase --abort @@ -98,7 +102,6 @@ jobs: - name: Synchronize dev with testing run: | git checkout dev - git fetch origin git rebase testing if [ $? -ne 0 ]; then git rebase --abort From 14952a6451021d0b6aeeee33b837d8b61b6e229b Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 00:56:39 +0100 Subject: [PATCH 213/357] tests --- .github/workflows/release_semver_versioning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index 6b25eee46..e02312fb6 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -98,7 +98,7 @@ jobs: git rebase --abort git rebase -Xtheirs dev fi - git push origin testing + git push -f origin testing - name: Synchronize dev with testing run: | git checkout dev @@ -107,7 +107,7 @@ jobs: git rebase --abort git rebase -Xours testing fi - git push origin dev + git push -f origin dev - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From ad10552346e9cd1a1f22a65c77363d06f377d4c5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 15:18:10 +0100 Subject: [PATCH 214/357] tests --- .github/workflows/release_semver_versioning.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index e02312fb6..f82cd4ccd 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -101,6 +101,7 @@ jobs: git push -f origin testing - name: Synchronize dev with testing run: | + cd ${{ github.workspace }}/action/package git checkout dev git rebase testing if [ $? -ne 0 ]; then From 7ec9b4cd7d8f6936e6ec0eb5640f0de43f784859 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 15:38:34 +0100 Subject: [PATCH 215/357] tests --- .github/actions/release_alpha/action.yml | 2 +- .github/workflows/release_alpha.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 38dc35a41..072a04ecb 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -104,7 +104,7 @@ runs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: ${{ inputs.changelog_file }} + output: action/package/${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 90d9bf0e3..764b4f1cc 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -114,7 +114,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: ${{ inputs.changelog_file }} + output: action/package/${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: From 4e5bab5689cf6d86e196cd368bd2518cd8780296 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 17:12:59 +0100 Subject: [PATCH 216/357] tests --- .../workflows/release_semver_versioning.yml | 30 ------------------- .../workflows/release_synchronize_testing.yml | 25 ++++++++++++++++ 2 files changed, 25 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/release_synchronize_testing.yml diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index f82cd4ccd..e74719a5b 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -79,36 +79,6 @@ jobs: with: commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ - - name: Checkout Testing - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_PAT }} - ref: testing - fetch-depth: 0 - path: action/package - - name: Synchronize testing with dev - run: | - cd ${{ github.workspace }}/action/package - git checkout dev - git pull origin dev - git checkout testing - git pull origin testing - git rebase dev - if [ $? -ne 0 ]; then - git rebase --abort - git rebase -Xtheirs dev - fi - git push -f origin testing - - name: Synchronize dev with testing - run: | - cd ${{ github.workspace }}/action/package - git checkout dev - git rebase testing - if [ $? -ne 0 ]; then - git rebase --abort - git rebase -Xours testing - fi - git push -f origin dev - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: diff --git a/.github/workflows/release_synchronize_testing.yml b/.github/workflows/release_synchronize_testing.yml new file mode 100644 index 000000000..2fbc9fc2c --- /dev/null +++ b/.github/workflows/release_synchronize_testing.yml @@ -0,0 +1,25 @@ +name: Sync Branch 'Testing' with 'Dev' + +on: + workflow_call: + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout Testing + uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_PAT }} + ref: testing + fetch-depth: 0 + path: action/package + - name: Synchronize testing with dev + run: | + cd ${{ github.workspace }}/action/package + git pull --rebase origin dev + if [ $? -ne 0 ]; then + git rebase --abort + git pull --rebase -Xtheirs origin dev + fi + git push -f origin testing \ No newline at end of file From 3e2a51c0a238209a7d1c26dd19cacceda470a46c Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 19:28:06 +0100 Subject: [PATCH 217/357] tests --- .../release_semver_versioning/action.yml | 82 ------------------- .github/workflows/release_semver_start.yml | 20 +++-- .../workflows/release_semver_versioning.yml | 9 -- 3 files changed, 15 insertions(+), 96 deletions(-) delete mode 100644 .github/actions/release_semver_versioning/action.yml diff --git a/.github/actions/release_semver_versioning/action.yml b/.github/actions/release_semver_versioning/action.yml deleted file mode 100644 index 04b0e8624..000000000 --- a/.github/actions/release_semver_versioning/action.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: 'Bump Version (after testing QA started)' -description: 'This action bumps the version after the testing QA phase has started' -inputs: - action_branch: - description: 'The shared action branch to checkout' - required: false - default: 'main' - python_version: - description: 'The python version to use' - required: false - default: '3.8' - version_file: - description: 'The file location to read the version from' - required: false - default: 'version.py' - changelog_file: - description: 'The file location of the changelog' - required: false - default: 'CHANGELOG.md' - subject: - description: 'Subject title of the push/pull-request event to parse the release type.' - required: true - -runs: - using: 'composite' - steps: - - name: set up release type - shell: bash - run: | - RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') - echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - - name: Checkout Package - uses: actions/checkout@v4 - with: - ref: dev - fetch-depth: 0 - path: action/package - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} - path: action/github/ - - name: Set up python ${{ inputs.python_version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Common Python Requirements - run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: alpha_release - uses: ./action/github/.github/actions/release_alpha - with: - action_branch: ${{ inputs.action_branch }} - python_version: ${{ inputs.python_version }} - version_file: ${{ inputs.version_file }} - changelog_file: ${{ inputs.changelog_file }} - - name: Bump to next ${{ env.RELEASE_TYPE }} version - id: version - run: | - cd ${{ github.workspace }}/action/github/scripts/ - VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save) - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Increment Version to ${{ env.NEXT_VERSION }} - repository: action/package/ - - name: Rebase testing onto dev - shell: bash - run: | - cd ${{ github.workspace }}/action/package/ - git checkout testing - git rebase dev - git push origin testing --force - - name: Notify Matrix Room - uses: ./action/github/.github/actions/notify_testing_matrix - with: - kickoff_pr: false - release_type: ${{ env.RELEASE_TYPE }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2b0674533..7d42b0da8 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -172,7 +172,7 @@ jobs: # alpha release (+ testing) alpha_release: needs: parse_conventional_commits - if: (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type)) || (inputs.kickoff_pr == 'false' && contains(fromJson('["patch", "minor", "major"]'), inputs.release_type)) || (inputs.release_type == 'alpha') || (contains(fromJson('["alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) + if: (contains(fromJson('["patch", "minor", "major", "alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: @@ -284,12 +284,22 @@ jobs: git pull origin ${{ env.TARGET_BRANCH }} git rebase ${{ inputs.branch }} git push origin ${{ env.TARGET_BRANCH }} - - name: Bump Version + - name: Bump to next ${{ env.RELEASE_TYPE }} version if: ${{ inputs.kickoff_pr == 'false' }} - uses: ./action/github/.github/actions/release_semver_versioning + run: | + cd ${{ github.workspace }}/action/package/ + git checkout ${{ inputs.branch }} + VERSION=$(python setup.py --version) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + cd ${{ github.workspace }}/action/github/scripts/ + NEXT_VERSION=$( python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save ) + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + if: ${{ inputs.kickoff_pr == 'false' }} + uses: stefanzweifel/git-auto-commit-action@v5 with: - release_type: ${{ env.RELEASE_TYPE }} - version_file: ${{ inputs.version_file }} + commit_message: Increment Version to ${{ env.NEXT_VERSION }} + repository: action/package/ - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index e74719a5b..f39ab151c 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -25,16 +25,7 @@ on: required: true jobs: - alpha_release: - secrets: inherit - uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 - with: - action_branch: ${{ inputs.action_branch }} - python_version: ${{ inputs.python_version }} - version_file: ${{ inputs.version_file }} - changelog_file: ${{ inputs.changelog_file }} bump_version_and_announce_testing: - needs: alpha_release env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: 'testing' From a260b126bdf6cbec171bdae913d6245666147acf Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 19:42:27 +0100 Subject: [PATCH 218/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 7d42b0da8..367ec1a9f 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -180,7 +180,7 @@ jobs: version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} changelog_file: ${{ inputs.changelog_file }} - release_type: ${{ needs.parse_conventional_commits.outputs.release_type }} + release_type: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' && needs.parse_conventional_commits.outputs.release_type || 'alpha' }} # >= patch release kickoff_testing: needs: parse_conventional_commits From 34104fee8951eb6ca77414d2bb33ab2a9e5fa2f9 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 20:28:07 +0100 Subject: [PATCH 219/357] tests --- .github/workflows/release_alpha.yml | 1 + .github/workflows/release_semver_start.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 764b4f1cc..9e24d04cc 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -86,6 +86,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} + unreleasedLabel: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} - name: Create Pre-release id: create_release diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 367ec1a9f..4530fd522 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -171,7 +171,7 @@ jobs: branch: ${{ inputs.branch }} # alpha release (+ testing) alpha_release: - needs: parse_conventional_commits + needs: [parse_conventional_commits, autotranslate] if: (contains(fromJson('["patch", "minor", "major", "alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit @@ -183,7 +183,7 @@ jobs: release_type: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' && needs.parse_conventional_commits.outputs.release_type || 'alpha' }} # >= patch release kickoff_testing: - needs: parse_conventional_commits + needs: [parse_conventional_commits, alpha_release, autotranslate] if: (contains(fromJson('["patch", "minor", "major"]'), needs.parse_conventional_commits.outputs.release_type) || (inputs.release_type != 'alpha' && inputs.release_type != '')) runs-on: ubuntu-latest env: @@ -310,7 +310,7 @@ jobs: # testing release # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: - needs: ["parse_conventional_commits", "autotranslate"] + needs: [parse_conventional_commits, alpha_release, autotranslate] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' }} runs-on: ubuntu-latest steps: From 98e778adbd7009fc870d48d11587190444a8108b Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 21:58:06 +0100 Subject: [PATCH 220/357] tests --- .../actions/notify_testing_matrix/action.yml | 21 ++--- .github/workflows/notify_testing_matrix.yml | 23 ++---- .github/workflows/release_semver_start.yml | 79 +++---------------- .../workflows/release_semver_versioning.yml | 2 +- 4 files changed, 25 insertions(+), 100 deletions(-) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index c1ddcf9a7..7e872c11b 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -4,33 +4,26 @@ inputs: matrix_token: description: 'The token to use to authenticate with the Matrix server' required: true - kickoff_pr: - description: 'Kick off pull request' - default: 'false' + # kickoff_pr: + # description: 'Kick off pull request' + # default: 'false' subject: description: 'Subject of the message' default: 'no subject' release_type: description: 'Type of the release' required: true + version: + description: 'The version of the release' + default: "version unknown" runs: using: 'composite' steps: - - name: Set message - shell: bash - run: | - if ${{ inputs.kickoff_pr == 'true' }}; then - MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" - else - MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" - fi - echo "MESSAGE=$MESSAGE" >> $GITHUB_ENV - id: message - name: Send message to Matrix uses: fadenb/matrix-chat-message@v0.0.6 with: homeserver: 'matrix.org' token: ${{ inputs.matrix_token }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: ${{ env.MESSAGE }} \ No newline at end of file + message: "Testing QA phase of a ${{ inputs.release_type }} release (${{ inputs.version }}) started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" \ No newline at end of file diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index 7bbfbba0e..32b9b6a87 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -4,36 +4,29 @@ name: Notify Matrix Chat on: workflow_call: inputs: - kickoff_pr: - type: boolean - default: true + # kickoff_pr: + # type: boolean + # default: true subject: type: string default: 'no subject' release_type: type: string required: true + version: + type: string + description: 'The version of the release' + default: 'version unknown' jobs: notify_pr_matrix: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set message - shell: bash - run: | - if ${{ inputs.kickoff_pr == 'true' }}; then - MESSAGE="PR to kick off the testing QA phase of a ${{ inputs.release_type }} release ($GITHUB_REPOSITORY). Please merge if the testing phase should start. Subject to test: ${{ inputs.subject }}" - else - MESSAGE="Testing QA phase of a ${{ inputs.release_type }} release started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" - fi - echo "MESSAGE=$MESSAGE" >> $GITHUB_ENV - id: message - name: Send message to Matrix bots channel uses: fadenb/matrix-chat-message@v0.0.6 - # this action was triggered by with: homeserver: 'matrix.org' token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: ${{ env.MESSAGE }} \ No newline at end of file + message: "Testing QA phase of a ${{ inputs.release_type }} release (${{ inputs.version }}) started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" \ No newline at end of file diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4530fd522..7ef0ba6ba 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -24,10 +24,10 @@ on: type: string description: 'Subject title of the push/pull request event to parse the release type.' required: false - kickoff_pr: - description: 'Whether to create a PR to mark the start of the testing phase or not' - type: string - default: 'false' + # kickoff_pr: + # description: 'Whether to create a PR to mark the start of the testing phase or not' + # type: string + # default: 'false' locale_folder: type: string description: 'The folder location of the locale files' @@ -221,71 +221,7 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: Check Latest Release - if: ${{ inputs.kickoff_pr == 'true' }} - run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py cycle --latest --type ${{ env.RELEASE_TYPE }}) - if [ -z "$SINCE_TAG" ]; then - if [ "${{ env.RELEASE_TYPE }}" == "patch" ]; then - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type minor) - elif [ "${{ env.RELEASE_TYPE }}" == "minor" ]; then - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type major) - fi - fi - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --next --type ${{ env.RELEASE_TYPE }}) - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - # if there is no testing branch, create one from the tag of the latest release or first commit - - name: Create testing Branch (if necessary) - run: | - cd ${{ github.workspace }}/action/package/ - TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) - INITIAL_COMMIT=$(git rev-list --max-parents=0 ${{ inputs.branch }}) - if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then - if [ "$SINCE_TAG" == "0.0.0" ]; then - git checkout -b ${{ env.TARGET_BRANCH }} $INITIAL_COMMIT - else - git checkout -b ${{ env.TARGET_BRANCH }} tags/${{ env.SINCE_TAG }} - fi - git push origin ${{ env.TARGET_BRANCH }} --force - fi - - name: create changelog - id: changelog - if: ${{ inputs.kickoff_pr == 'true' }} - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} - - name: Create Pull Request to ${{ env.TARGET_BRANCH }} - if: ${{ inputs.kickoff_pr == 'true'}} - run: | - cd ${{ github.workspace }}/action/package/ - gh pr create \ - --title "ci(release): kickoff testing QA phase (${{ env.RELEASE_TYPE }} release est. ${{ env.NEXT_VERSION}})" \ - --body "$(cat << EOF - This kicks off the testing QA phase of a ${{ env.RELEASE_TYPE }} release (est. ${{ env.NEXT_VERSION}}). - - Please merge if it the testing phase should start. Version gets bumped to the next ${{ env.RELEASE_TYPE }} version. - - (If there's the need to delay, after merging this PR recent changes gets automatically pulled to start the testing phase.) - - Included changes: - ${{ steps.changelog.outputs.changelog }} - EOF - )" \ - --base ${{ env.TARGET_BRANCH }} \ - --head dev \ - --assignee ${{ github.actor }} - - name: Push to testing branch - if: ${{ inputs.kickoff_pr == 'false' }} - run: | - cd ${{ github.workspace }}/action/package/ - git checkout ${{ env.TARGET_BRANCH }} - git pull origin ${{ env.TARGET_BRANCH }} - git rebase ${{ inputs.branch }} - git push origin ${{ env.TARGET_BRANCH }} - name: Bump to next ${{ env.RELEASE_TYPE }} version - if: ${{ inputs.kickoff_pr == 'false' }} run: | cd ${{ github.workspace }}/action/package/ git checkout ${{ inputs.branch }} @@ -295,18 +231,21 @@ jobs: NEXT_VERSION=$( python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save ) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - if: ${{ inputs.kickoff_pr == 'false' }} uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Increment Version to ${{ env.NEXT_VERSION }} repository: action/package/ + - name: Force Push dev to testing branch + run: | + cd ${{ github.workspace }}/action/package/ + git push origin dev:testing --force - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: matrix_token: ${{ secrets.MATRIX_TOKEN }} - kickoff_pr: ${{ inputs.kickoff_pr }} subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} + version: ${{ env.NEXT_VERSION }} # testing release # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml index f39ab151c..d83806501 100644 --- a/.github/workflows/release_semver_versioning.yml +++ b/.github/workflows/release_semver_versioning.yml @@ -73,5 +73,5 @@ jobs: - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: - kickoff_pr: false release_type: ${{ env.RELEASE_TYPE }} + version: ${{ env.NEXT_VERSION }} From 5bc95f4a0bd64b3a2e374d4b03e4a2ae1aa19d51 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 22:11:31 +0100 Subject: [PATCH 221/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 7ef0ba6ba..c2b7534f1 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -190,6 +190,7 @@ jobs: TARGET_BRANCH: 'testing' GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GH_PAT }} + VERSION_FILE: ${{ github.workspace}}/action/package/${{ inputs.version_file }} RELEASE_TYPE: ${{ needs.parse_conventional_commits.outputs.release_type }} GH_PAT: ${{ secrets.GH_PAT }} steps: From 3da2629865eb47075b30d3ee70d8733415c122dc Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 7 Mar 2024 22:23:07 +0100 Subject: [PATCH 222/357] tests --- .github/workflows/notify_testing_matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index 32b9b6a87..4ce4cccfa 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -29,4 +29,4 @@ jobs: homeserver: 'matrix.org' token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: "Testing QA phase of a ${{ inputs.release_type }} release (${{ inputs.version }}) started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" \ No newline at end of file + message: "Testing QA phase of a ${{ inputs.release_type }} release (${{ inputs.version }}) started (${{ github.repository }}). Subject to test: ${{ inputs.subject }}" \ No newline at end of file From 7077640434a28cb79b76a140cd90bd2fbc75824b Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:02:55 +0100 Subject: [PATCH 223/357] tests --- .../workflows/check_conventional_commits.yml | 34 +++++++--- pr_labels.json | 29 +++++++++ scripts/check_PRs.py | 65 +++++++++++++++++++ scripts/parse_semver_release.py | 10 ++- 4 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 pr_labels.json create mode 100644 scripts/check_PRs.py diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index e70d41563..add9bc002 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -10,18 +10,36 @@ on: description: 'Branch to check out' type: string default: ${{ github.ref }} + action_branch: + description: 'Branch to check out' + type: string + default: main + custom_labels: + description: 'Custom labels to use' + type: string + pr_number: + description: 'Optional pull request number' + type: number + missing_cc_error: + description: 'Throws an error if the title not adheres to Conventional Commit format' + type: boolean + default: true jobs: validate-pr-title: runs-on: ${{ inputs.runner }} + env: + GH_PAT: ${{ secrets.GH_PAT }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_LABELS: ${{ inputs.custom_labels }} + PR_NUMBER: ${{ inputs.pr_number }} steps: - - name: Checkout + - name: Checkout Scripts Repo uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} - - name: PR Conventional Commit Validation - uses: ytanikin/PRConventionalCommits@1.1.0 - with: - token: ${{ secrets.GH_PAT }} - task_types: '["feat", "fix", "docs", "test", "ci", "refactor", "perf", "build", "chore", "revert"]' - custom_labels: '{"feat": "feature", "fix": "fix", "docs": "documentation", "test": "test", "ci": "CI/CD", "refactor": "refactor", "perf": "performance", "chore": "housekeeping", "revert": "revert", "wip": "WIP"}' \ No newline at end of file + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + - name: Status Check + run: | + python scripts/check_PRs.py + \ No newline at end of file diff --git a/pr_labels.json b/pr_labels.json new file mode 100644 index 000000000..734a4aa49 --- /dev/null +++ b/pr_labels.json @@ -0,0 +1,29 @@ +{ + "feat": "feature", + "fix": "fix", + "docs": "documentation", + "test": "unittests", + "ci": "CI/CD", + "refactor": "refactor", + "perf": "performance", + "chore": "housekeeping", + "revert": "revert", + "wip": "WIP", + "style": "codestyle", + "config": "config", + "parser": "parser", + "tooling": "tooling", + "cli": "cli", + "gui": "gui", + "core": "core", + "audio": "audio", + "tts": "tts", + "sst": "sst", + "nlp": "nlp", + "plugins": "plugins", + "localisation": "localisation", + "resources": "resources", + "release": "release", + "breaking": "breaking change", + "need_cc": "CC missing" +} \ No newline at end of file diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py new file mode 100644 index 000000000..0f9de3111 --- /dev/null +++ b/scripts/check_PRs.py @@ -0,0 +1,65 @@ +import os +from os.path import join, dirname, isfile +import re +from github import Github, PullRequest +import json +import pccc + +TOKEN = os.getenv('GH_PAT') or os.getenv('GITHUB_TOKEN') +REPOSITORY = os.getenv('GITHUB_REPOSITORY') +PR_LABELS: dict = json.loads(os.getenv('PR_LABELS', '{}')) +PR_NUMBER = os.getenv('PR_NUMBER') +ERROR_ON_MISSING_CC = os.getenv('MISSING_CC_ERROR', 'true').lower() == 'true' +if not PR_LABELS: + PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "PR_LABELS.json")).read()) + +test_phase_file = os.getenv('TEST_PHASE_FILE', '') +if not isfile(test_phase_file): + ongoing_test = False + if test_phase_file: + print("The file specified in TEST_PHASE_FILE does not exist.") +else: + with open(test_phase_file, 'r') as f: + ongoing_test = f.read().strip() == "testing" + +def check_for_labels(pull_request: PullRequest): + global cc_missing + + ccr = pccc.ConventionalCommitRunner() + ccr.options.load() + ccr.raw = f"{pull_request.title}\n{pull_request.body}" + ccr.clean() + labels = set() + try: + ccr.parse() + # no spec compliant format + except Exception: + labels.add(PR_LABELS.get("need_cc", "cc missing")) + cc_missing = True + else: + if ccr.breaking.get("flag") or ccr.breaking.get("token"): + labels.add(PR_LABELS.get("breaking", "breaking change")) + if ccr.header.get("scope") in PR_LABELS: + labels.add(PR_LABELS.get(ccr.header["scope"])) + if ccr.header.get("type") in PR_LABELS: + labels.add(PR_LABELS.get(ccr.header["type"])) + if ongoing_test: + labels.add("do no merge") + + return labels + + +git = Github(TOKEN).get_repo(REPOSITORY) +open_pulls = git.get_pulls(state='open') +cc_missing = False + + +for pr in open_pulls: + if PR_NUMBER and pr.number != int(PR_NUMBER): + continue + labels = check_for_labels(pr) + pr.set_labels(labels) + +# nuke status check +if (cc_missing and ERROR_ON_MISSING_CC) or ongoing_test: + raise Exception(f"CC missing: {cc_missing}, ongoing test phase: {ongoing_test}") diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index a9a676984..fca47c739 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -35,12 +35,16 @@ def semver_from_cc(): return "major" elif ccr.header.get("type") == "feat": return "minor" - elif ccr.header.get("type") == "fix" and ccr.header.get("scope") == "testing": - return "testing" + # commits that gets added to a release (special case) + elif ccr.header.get("type") == "release": + return "release" elif ccr.header.get("type") == "fix": return "patch" - else: + elif ccr.header.get("type") not in ["ci", "docs", "style", "test"]: return "alpha" + else: + print("No semver release.") + exit(0) def semver_from_version(): try: From 032df84b6948260c07a751371dbf72b9c5e60108 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:13:00 +0100 Subject: [PATCH 224/357] tests --- .github/workflows/check_conventional_commits.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index add9bc002..243859583 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -38,7 +38,10 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} + - name: Install Python Requirements + run: | + python -m pip install --upgrade pip + pip install -r requirements/pip_base.txt - name: Status Check run: | python scripts/check_PRs.py From b16f922f156c7322500e041828781b34b5e7953f Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:19:39 +0100 Subject: [PATCH 225/357] tests --- .github/workflows/check_conventional_commits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 243859583..fc87fedc0 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -41,7 +41,7 @@ jobs: - name: Install Python Requirements run: | python -m pip install --upgrade pip - pip install -r requirements/pip_base.txt + pip install -r ${{ github.workspace }}/requirements/pip_base.txt - name: Status Check run: | python scripts/check_PRs.py From 376e9de797f41f447bf9f276bd244d4c8cb53cb1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:28:03 +0100 Subject: [PATCH 226/357] tests --- .github/workflows/check_conventional_commits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index fc87fedc0..87e106d4e 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -38,6 +38,7 @@ jobs: uses: actions/checkout@v4 with: repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} - name: Install Python Requirements run: | python -m pip install --upgrade pip From d8eee7cc8c7a25d1f9799e3c2a64b7c08dd7a5c9 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:31:28 +0100 Subject: [PATCH 227/357] tests --- .github/workflows/check_conventional_commits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 87e106d4e..0a7040585 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -17,6 +17,7 @@ on: custom_labels: description: 'Custom labels to use' type: string + default: '{}' pr_number: description: 'Optional pull request number' type: number From 97272d5e75522a77c74237c85a3e22ccb70c3fd0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:33:24 +0100 Subject: [PATCH 228/357] tetst --- scripts/check_PRs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 0f9de3111..2cba54faf 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -11,7 +11,7 @@ PR_NUMBER = os.getenv('PR_NUMBER') ERROR_ON_MISSING_CC = os.getenv('MISSING_CC_ERROR', 'true').lower() == 'true' if not PR_LABELS: - PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "PR_LABELS.json")).read()) + PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "pr_labels.json")).read()) test_phase_file = os.getenv('TEST_PHASE_FILE', '') if not isfile(test_phase_file): From c5610c354e29e40f16f3c952cdff2b3be6b8738f Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:36:13 +0100 Subject: [PATCH 229/357] tests --- scripts/check_PRs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 2cba54faf..7a8bfe65c 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -58,7 +58,7 @@ def check_for_labels(pull_request: PullRequest): if PR_NUMBER and pr.number != int(PR_NUMBER): continue labels = check_for_labels(pr) - pr.set_labels(labels) + pr.set_labels(list(labels)) # nuke status check if (cc_missing and ERROR_ON_MISSING_CC) or ongoing_test: From 07dc45cfac27ff6bddec93aa308a7c01023bad85 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 20:58:08 +0100 Subject: [PATCH 230/357] tests --- scripts/check_PRs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 7a8bfe65c..db8144ce1 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -1,8 +1,9 @@ import os from os.path import join, dirname, isfile -import re -from github import Github, PullRequest import json +from typing import List + +from github import Github, PullRequest import pccc TOKEN = os.getenv('GH_PAT') or os.getenv('GITHUB_TOKEN') @@ -22,7 +23,7 @@ with open(test_phase_file, 'r') as f: ongoing_test = f.read().strip() == "testing" -def check_for_labels(pull_request: PullRequest): +def check_for_labels(pull_request: PullRequest) -> List[str]: global cc_missing ccr = pccc.ConventionalCommitRunner() @@ -44,9 +45,9 @@ def check_for_labels(pull_request: PullRequest): if ccr.header.get("type") in PR_LABELS: labels.add(PR_LABELS.get(ccr.header["type"])) if ongoing_test: - labels.add("do no merge") + labels.add("do not merge") - return labels + return list(labels) git = Github(TOKEN).get_repo(REPOSITORY) @@ -58,7 +59,7 @@ def check_for_labels(pull_request: PullRequest): if PR_NUMBER and pr.number != int(PR_NUMBER): continue labels = check_for_labels(pr) - pr.set_labels(list(labels)) + pr.set_labels(*labels) # nuke status check if (cc_missing and ERROR_ON_MISSING_CC) or ongoing_test: From 26a0579d964c61277903fae0c055a36836ba1ee0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 21:04:45 +0100 Subject: [PATCH 231/357] tests --- pyproject.toml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0a3425ec3..ef4c2389f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,21 +16,24 @@ generated_commits = [ '''^Merge branch 'master'.*$''', ] -# not included "ci", "docs", "style", "test" as they dont get a release -# # chore -> alpha # fix -> patch # feat -> minor # in combination with "!" ("feat!:") or BREAKING CHANGE (in body/message) -> major types = [ + "ci", + "docs", + "style", + "test", "chore", "build", "feat", "fix", "perf", "refactor", - "revert" + "revert", + "release ] scopes = [ @@ -46,9 +49,7 @@ scopes = [ "nlp", "plugins", "localisation", - "resources", - "release", - "testing" + "resources" ] footers = [ From cc7fc25793d9f1cdc7328f9c9f1b99d7f2c60462 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 21:05:56 +0100 Subject: [PATCH 232/357] tests --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ef4c2389f..ed9199aa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ types = [ "perf", "refactor", "revert", - "release + "release" ] scopes = [ From 8588385993154910c61e2b44106d756af03771b8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 8 Mar 2024 22:17:08 +0100 Subject: [PATCH 233/357] tests --- .github/workflows/check_conventional_commits.yml | 7 +++++++ .github/workflows/release_semver_publish.yml | 11 +++++++++++ .github/workflows/release_semver_start.yml | 11 +++++++++++ scripts/check_PRs.py | 8 ++++---- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 0a7040585..ad79bbcbb 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -34,6 +34,8 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_LABELS: ${{ inputs.custom_labels }} PR_NUMBER: ${{ inputs.pr_number }} + MISSING_CC_ERROR: ${{ inputs.missing_cc_error }} + TEST_PHASE_CACHE: ${{ github.workspace }}/test-status steps: - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -44,6 +46,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r ${{ github.workspace }}/requirements/pip_base.txt + - name: Get Cached Test Status + uses: actions/cache@v2 + with: + path: test-status + key: test-status- - name: Status Check run: | python scripts/check_PRs.py diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 0e8e5e14a..f5801bc37 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -91,6 +91,17 @@ jobs: with: password: ${{secrets.PYPI_TOKEN}} packages-dir: action/dist/ + # set cached test status to complete + - name: Set test status + run: | + echo "complete" > test-status + - name: Cache test status + uses: actions/cache@v2 + with: + path: test-status + key: test-status-${{ github.run_id }} + restore-keys: | + test-status- # NO REBASE APPROACH!! # - name: Rebase dev onto master # run: | diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c2b7534f1..45b6bd5e4 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -240,6 +240,17 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git push origin dev:testing --force + # set cached test status to complete + - name: Set test status + run: | + echo "testing" > test-status + - name: Cache test status + uses: actions/cache@v2 + with: + path: test-status + key: test-status-${{ github.run_id }} + restore-keys: | + test-status- - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index db8144ce1..e15b76a67 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -14,13 +14,13 @@ if not PR_LABELS: PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "pr_labels.json")).read()) -test_phase_file = os.getenv('TEST_PHASE_FILE', '') -if not isfile(test_phase_file): +test_phase_cache = os.getenv('TEST_PHASE_CACHE', '') +if not isfile(test_phase_cache): ongoing_test = False - if test_phase_file: + if test_phase_cache: print("The file specified in TEST_PHASE_FILE does not exist.") else: - with open(test_phase_file, 'r') as f: + with open(test_phase_cache, 'r') as f: ongoing_test = f.read().strip() == "testing" def check_for_labels(pull_request: PullRequest) -> List[str]: From adfe69028047d625a0a18aaf8cc838a9d98aaf92 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 10 Mar 2024 14:17:34 +0100 Subject: [PATCH 234/357] tests --- scripts/check_PRs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index e15b76a67..30d9e4890 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -45,7 +45,7 @@ def check_for_labels(pull_request: PullRequest) -> List[str]: if ccr.header.get("type") in PR_LABELS: labels.add(PR_LABELS.get(ccr.header["type"])) if ongoing_test: - labels.add("do not merge") + labels.add("ongoing test") return list(labels) From 811c3a6a3f81616d865c7da65d703ab11d619a05 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 12 Mar 2024 13:43:10 +0100 Subject: [PATCH 235/357] tests --- .../workflows/check_conventional_commits.yml | 3 ++- .github/workflows/clear_test_phase.yml | 18 ++++++++++++++++++ .github/workflows/release_semver_start.yml | 6 +----- 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/clear_test_phase.yml diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index ad79bbcbb..b9312bdd1 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -20,7 +20,8 @@ on: default: '{}' pr_number: description: 'Optional pull request number' - type: number + type: string + default: '' missing_cc_error: description: 'Throws an error if the title not adheres to Conventional Commit format' type: boolean diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml new file mode 100644 index 000000000..ca4af33ea --- /dev/null +++ b/.github/workflows/clear_test_phase.yml @@ -0,0 +1,18 @@ +on: + workflow_call: + +jobs: + clear_test_flag: + runs-on: ubuntu-latest + # set cached test status to complete + steps: + - name: Set test status + run: | + echo "complete" > test-status + - name: Cache test status + uses: actions/cache@v2 + with: + path: test-status + key: test-status-${{ github.run_id }} + restore-keys: | + test-status- \ No newline at end of file diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 45b6bd5e4..e050586ff 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -24,10 +24,6 @@ on: type: string description: 'Subject title of the push/pull request event to parse the release type.' required: false - # kickoff_pr: - # description: 'Whether to create a PR to mark the start of the testing phase or not' - # type: string - # default: 'false' locale_folder: type: string description: 'The folder location of the locale files' @@ -240,7 +236,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git push origin dev:testing --force - # set cached test status to complete + # set cached test status to active - name: Set test status run: | echo "testing" > test-status From 80b43633f6ba5555902fd0f8ca31096fa8513795 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 12 Mar 2024 14:33:05 +0100 Subject: [PATCH 236/357] tests --- .github/actions/release_alpha/action.yml | 2 +- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 6 +++--- scripts/parse_semver_release.py | 7 ++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 072a04ecb..0c86129b5 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -47,7 +47,7 @@ runs: # testing specific steps. Those releases belong to the previous release cycle # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - name: Checkout At Tag And Cherry Pick - if: ${{ inputs.release_type == 'testing' }} + if: ${{ inputs.release_type == 'release' }} run: | cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse dev) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 9e24d04cc..6147501c9 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -57,7 +57,7 @@ jobs: # testing specific steps. Those releases belong to the previous release cycle # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - name: Checkout At Tag And Cherry Pick - if: ${{ inputs.release_type == 'testing' }} + if: ${{ inputs.release_type == 'release' }} run: | cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse dev) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index e050586ff..6fd4dcfba 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -168,7 +168,7 @@ jobs: # alpha release (+ testing) alpha_release: needs: [parse_conventional_commits, autotranslate] - if: (contains(fromJson('["patch", "minor", "major", "alpha", "testing"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) + if: (contains(fromJson('["patch", "minor", "major", "alpha", "release"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: @@ -176,7 +176,7 @@ jobs: version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} changelog_file: ${{ inputs.changelog_file }} - release_type: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' && needs.parse_conventional_commits.outputs.release_type || 'alpha' }} + release_type: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' && needs.parse_conventional_commits.outputs.release_type || 'alpha' }} # >= patch release kickoff_testing: needs: [parse_conventional_commits, alpha_release, autotranslate] @@ -258,7 +258,7 @@ jobs: # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: needs: [parse_conventional_commits, alpha_release, autotranslate] - if: ${{ needs.parse_conventional_commits.outputs.release_type == 'testing' }} + if: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' }} runs-on: ubuntu-latest steps: - name: Checkout Repository diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index fca47c739..9a33a3ce1 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -33,11 +33,12 @@ def semver_from_cc(): if ccr.breaking.get("flag") or ccr.breaking.get("token"): return "major" - elif ccr.header.get("type") == "feat": - return "minor" # commits that gets added to a release (special case) - elif ccr.header.get("type") == "release": + elif ccr.header.get("type") == "release" or \ + ccr.header.get("scope") == "release": return "release" + elif ccr.header.get("type") == "feat": + return "minor" elif ccr.header.get("type") == "fix": return "patch" elif ccr.header.get("type") not in ["ci", "docs", "style", "test"]: From 7238983b8eb34a4505b2d5a247603c22968857c6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 12 Mar 2024 16:07:08 +0100 Subject: [PATCH 237/357] tests --- .../workflows/check_conventional_commits.yml | 8 +- pr_labels.json | 10 +-- scripts/check_PRs.py | 79 +++++++++++++------ 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index b9312bdd1..5d574da74 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -22,10 +22,10 @@ on: description: 'Optional pull request number' type: string default: '' - missing_cc_error: - description: 'Throws an error if the title not adheres to Conventional Commit format' + error_on_failure: + description: 'Throws an error if the title not adheres to Conventional Commit format or testing phase is ongoing' type: boolean - default: true + default: false jobs: validate-pr-title: @@ -35,7 +35,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_LABELS: ${{ inputs.custom_labels }} PR_NUMBER: ${{ inputs.pr_number }} - MISSING_CC_ERROR: ${{ inputs.missing_cc_error }} + ERROR_ON_FAILURE: ${{ inputs.error_on_failure }} TEST_PHASE_CACHE: ${{ github.workspace }}/test-status steps: - name: Checkout Scripts Repo diff --git a/pr_labels.json b/pr_labels.json index 734a4aa49..f642ffe06 100644 --- a/pr_labels.json +++ b/pr_labels.json @@ -13,13 +13,13 @@ "config": "config", "parser": "parser", "tooling": "tooling", - "cli": "cli", - "gui": "gui", + "cli": "CLI", + "gui": "GUI", "core": "core", "audio": "audio", - "tts": "tts", - "sst": "sst", - "nlp": "nlp", + "tts": "TTS", + "sst": "STT", + "nlp": "NLP", "plugins": "plugins", "localisation": "localisation", "resources": "resources", diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 30d9e4890..9f88eb0e9 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -1,7 +1,7 @@ import os from os.path import join, dirname, isfile import json -from typing import List +from typing import List, Optional from github import Github, PullRequest import pccc @@ -9,8 +9,8 @@ TOKEN = os.getenv('GH_PAT') or os.getenv('GITHUB_TOKEN') REPOSITORY = os.getenv('GITHUB_REPOSITORY') PR_LABELS: dict = json.loads(os.getenv('PR_LABELS', '{}')) -PR_NUMBER = os.getenv('PR_NUMBER') -ERROR_ON_MISSING_CC = os.getenv('MISSING_CC_ERROR', 'true').lower() == 'true' +SINGLE_PR = os.getenv('PR_NUMBER') +ERROR_ON_FAILURE = os.getenv('ERROR_ON_FAILURE', 'false').lower() == 'true' if not PR_LABELS: PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "pr_labels.json")).read()) @@ -23,29 +23,54 @@ with open(test_phase_cache, 'r') as f: ongoing_test = f.read().strip() == "testing" -def check_for_labels(pull_request: PullRequest) -> List[str]: - global cc_missing - + +def cc_type(desc: str) -> str: + ccr = parse_cc(desc) + if ccr: + if ccr.breaking.get("flag") or ccr.breaking.get("token"): + return "breaking" + return ccr.header.get("type") + + return "unknown" + + +def cc_scope(desc: str) -> str: + ccr = parse_cc(desc) + if ccr: + return ccr.header.get("scope") + + return "unknown" + + +def parse_cc(desc: str) -> Optional[pccc.ConventionalCommitRunner]: ccr = pccc.ConventionalCommitRunner() ccr.options.load() - ccr.raw = f"{pull_request.title}\n{pull_request.body}" + ccr.raw = desc ccr.clean() - labels = set() try: ccr.parse() + return ccr # no spec compliant format except Exception: - labels.add(PR_LABELS.get("need_cc", "cc missing")) - cc_missing = True - else: - if ccr.breaking.get("flag") or ccr.breaking.get("token"): - labels.add(PR_LABELS.get("breaking", "breaking change")) - if ccr.header.get("scope") in PR_LABELS: - labels.add(PR_LABELS.get(ccr.header["scope"])) - if ccr.header.get("type") in PR_LABELS: - labels.add(PR_LABELS.get(ccr.header["type"])) - if ongoing_test: - labels.add("ongoing test") + return None + + +def check_cc_labels(desc: str) -> List[str]: + global cc_missing + + labels = set() + _type = cc_type(desc) + _scope = cc_scope(desc) + if _type == "unknown": + return [PR_LABELS.get("need_cc", "CC missing")] + if _type == "breaking": + labels.add(PR_LABELS.get("breaking", "breaking change")) + if _scope in PR_LABELS: + labels.add(PR_LABELS.get(_scope)) + if _type in PR_LABELS: + labels.add(PR_LABELS.get(_type)) + if ongoing_test and not any(t == "release" for t in [_type, _scope]): + labels.add("ongoing test") return list(labels) @@ -56,11 +81,19 @@ def check_for_labels(pull_request: PullRequest) -> List[str]: for pr in open_pulls: - if PR_NUMBER and pr.number != int(PR_NUMBER): + if SINGLE_PR and pr.number != int(SINGLE_PR): continue - labels = check_for_labels(pr) + pr_description = f"{pr.title}\n{pr.body}" + labels = check_cc_labels(pr_description) pr.set_labels(*labels) -# nuke status check -if (cc_missing and ERROR_ON_MISSING_CC) or ongoing_test: + # clear the test flag if the PR adresses a release. Ie. gets added to the test + if SINGLE_PR: + if cc_type(pr_description) == "release": + ongoing_test = False + elif cc_type(pr_description) == "unknown": + cc_missing = True + +# nuke status check (if requested) +if (cc_missing or ongoing_test) and ERROR_ON_FAILURE: raise Exception(f"CC missing: {cc_missing}, ongoing test phase: {ongoing_test}") From 582dec989c6654bbbfcebaa395cd3dbec6b64132 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 12 Mar 2024 17:23:28 +0100 Subject: [PATCH 238/357] tests --- .github/actions/release_alpha/action.yml | 1 + .github/workflows/release_alpha.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 0c86129b5..785ff7af6 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -44,6 +44,7 @@ runs: with: ref: ${{ env.SOURCE_BRANCH }} path: action/package/ + fetch-depth: 0 # testing specific steps. Those releases belong to the previous release cycle # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - name: Checkout At Tag And Cherry Pick diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 6147501c9..0b8c0a466 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -54,6 +54,7 @@ jobs: with: ref: ${{ env.SOURCE_BRANCH }} path: action/package/ + fetch-depth: 0 # testing specific steps. Those releases belong to the previous release cycle # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - name: Checkout At Tag And Cherry Pick From 04aadd92a573ba09481cd8f679533daf59278ff1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 12 Mar 2024 20:44:37 +0100 Subject: [PATCH 239/357] tests --- .github/actions/release_alpha/action.yml | 20 ++++++++---- .github/workflows/release_alpha.yml | 22 +++++++------ .github/workflows/release_semver_start.yml | 36 ++++++++++++++++------ 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml index 785ff7af6..fa7065404 100644 --- a/.github/actions/release_alpha/action.yml +++ b/.github/actions/release_alpha/action.yml @@ -17,6 +17,15 @@ inputs: description: 'The file location of the changelog' required: false default: 'CHANGELOG.md' + # internal workflow variables + release_type: + description: 'The type of release to make' + type: string + default: alpha + cherrypick_sha: + description: 'the SHA(s) to cherry pick as a multi-line string' + type: string + runs: using: 'composite' env: @@ -51,16 +60,15 @@ runs: if: ${{ inputs.release_type == 'release' }} run: | cd ${{ github.workspace }}/action/package/ - LAST_COMMIT_SHA=$(git rev-parse dev) TEST_VERSION=$(python setup.py --version) VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - git checkout "tag/$VERSION" - git cherry-pick "$LAST_COMMIT_SHA" - CHERRYPICK_SHA=$(git rev-parse HEAD) - echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV + git checkout "tags/$VERSION" + git cherry-pick "${{ inputs.cherrypick_sha }}" + CHERRYPICKED_SHA=$(git rev-parse HEAD) + echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Increment Version id: version if: ${{ inputs.release_type == 'alpha' }} @@ -88,7 +96,7 @@ runs: body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - commit: ${{ env.CHERRYPICK_SHA || 'dev' }} + commit: ${{ env.CHERRYPICKED_SHA || 'dev' }} prerelease: true - name: Build Distribution Packages run: | diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 0b8c0a466..8b0fda017 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -20,11 +20,15 @@ on: description: 'The file location to write the changelog to' type: string default: CHANGELOG.md - # internal workflow variable + # internal workflow variables release_type: - description: 'The type of release to make' + description: 'The type of release to make' + type: string + default: alpha + cherrypick_sha: + description: 'the SHA(s) to cherry pick as a whitespace separated string' type: string - default: alpha + jobs: build_and_publish: @@ -52,6 +56,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} ref: ${{ env.SOURCE_BRANCH }} path: action/package/ fetch-depth: 0 @@ -61,16 +66,15 @@ jobs: if: ${{ inputs.release_type == 'release' }} run: | cd ${{ github.workspace }}/action/package/ - LAST_COMMIT_SHA=$(git rev-parse dev) TEST_VERSION=$(python setup.py --version) VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - git checkout "tag/$VERSION" - git cherry-pick "$LAST_COMMIT_SHA" - CHERRYPICK_SHA=$(git rev-parse HEAD) - echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV + git checkout "tags/$VERSION" + git cherry-pick "${{ inputs.cherrypick_sha }}" + CHERRYPICKED_SHA=$(git rev-parse HEAD) + echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Increment Version id: version if: ${{ inputs.release_type == 'alpha' }} @@ -99,7 +103,7 @@ jobs: body: | Changes in this Release ${{ steps.changelog.outputs.changelog }} - commit: ${{ env.CHERRYPICK_SHA || 'dev' }} + commit: ${{ env.CHERRYPICKED_SHA || 'dev' }} prerelease: true - name: Build Distribution Packages run: | diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 6fd4dcfba..0b218dbba 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -168,7 +168,7 @@ jobs: # alpha release (+ testing) alpha_release: needs: [parse_conventional_commits, autotranslate] - if: (contains(fromJson('["patch", "minor", "major", "alpha", "release"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) + if: (contains(fromJson('["patch", "minor", "major", "alpha"]'), needs.parse_conventional_commits.outputs.release_type)) || (contains(fromJson('["patch", "minor", "major", "alpha"]'), inputs.release_type)) uses: openvoiceos/.github/.github/workflows/release_alpha.yml@feat/shared_actions1 secrets: inherit with: @@ -176,7 +176,6 @@ jobs: version_file: ${{ inputs.version_file }} python_version: ${{ inputs.python_version }} changelog_file: ${{ inputs.changelog_file }} - release_type: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' && needs.parse_conventional_commits.outputs.release_type || 'alpha' }} # >= patch release kickoff_testing: needs: [parse_conventional_commits, alpha_release, autotranslate] @@ -275,24 +274,41 @@ jobs: ref: testing path: action/package/testing/ # based on the workflow run, set the number of commits to cherry pick - # testing PRs/pushes get an alpha release for sure + # note: merge commits are filtered + # note: the list of commits to cherrypick will be reversed, subsequent processes can use it right away - name: set commits num to cherry pick run: | COMMITS = 1 if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then - COMMITS = $((COMMITS + 2)) - else COMMITS = $((COMMITS + 1)) - fi if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS = $((COMMITS + 1)) fi - echo "COMMITS=${COMMITS}" >> $GITHUB_ENV + CHERRYPICK_SHA="" + while read sha + do + if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then + CHERRYPICK_SHA="$CHERRYPICK_SHA $sha" + fi + done < <(git log -n 4 --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV + - name: Release + uses: openvoiceos/.github/.github/actions/release_alpha@feat/shared_actions1 + with: + action_branch: ${{ inputs.action_branch }} + version_file: ${{ inputs.version_file }} + python_version: ${{ inputs.python_version }} + changelog_file: ${{ inputs.changelog_file }} + release_type: release + cherrypick_sha: ${{ env.CHERRYPICK_SHA }} - name: Cherry pick n last commit from dev run: | - cd action/package/testing/ - git cherry-pick $(git -C ../dev log -n ${{ env.COMMITS }} --pretty=format:"%H") + cd ${{ github.workspace }}/action/package/dev/ + git fetch origin dev + LOG_BUMP_SHA=$(git log -n 1 --pretty=format:"%H") + cd ${{ github.workspace }}/action/package/testing/ + git cherry-pick ${{ env.CHERRYPICK_SHA }} "$LOG_BUMP_SHA" - name: Push changes to testing branch run: | - cd action/package/testing/ + cd ${{ github.workspace }}/action/package/testing/ git push origin testing From 4423dea6643af4c9491e339133ea1607cc6d4985 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 13 Mar 2024 16:02:44 +0100 Subject: [PATCH 240/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 0b218dbba..5d4d64668 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -256,7 +256,7 @@ jobs: # testing release # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: - needs: [parse_conventional_commits, alpha_release, autotranslate] + needs: [parse_conventional_commits, autotranslate] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' }} runs-on: ubuntu-latest steps: From b8ceb2e6eaadce2727131c8ee2e2d89495be3393 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 13 Mar 2024 19:11:41 +0100 Subject: [PATCH 241/357] tests --- .github/actions/release_alpha/action.yml | 121 --------------------- .github/workflows/release_semver_start.yml | 101 +++++++++++++---- 2 files changed, 79 insertions(+), 143 deletions(-) delete mode 100644 .github/actions/release_alpha/action.yml diff --git a/.github/actions/release_alpha/action.yml b/.github/actions/release_alpha/action.yml deleted file mode 100644 index fa7065404..000000000 --- a/.github/actions/release_alpha/action.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: 'Publish Alpha Build' -description: 'This action will generate a distribution and upload it to PyPI' -inputs: - action_branch: - description: 'The shared action branch to checkout' - required: false - default: 'main' - version_file: - description: 'The file location to read the version from' - required: false - default: 'version.py' - python_version: - description: 'The python version to use' - required: false - default: '3.8' - changelog_file: - description: 'The file location of the changelog' - required: false - default: 'CHANGELOG.md' - # internal workflow variables - release_type: - description: 'The type of release to make' - type: string - default: alpha - cherrypick_sha: - description: 'the SHA(s) to cherry pick as a multi-line string' - type: string - -runs: - using: 'composite' - env: - VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} - SOURCE_BRANCH: ${{ inputs.release_type == 'alpha' && 'dev' || 'testing' }} - GITHUB_REPOSITORY: ${{ github.repository }} - steps: - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} - path: action/github/ - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Python Dependencies - run: | - pip install --upgrade pip - pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: ${{ env.SOURCE_BRANCH }} - path: action/package/ - fetch-depth: 0 - # testing specific steps. Those releases belong to the previous release cycle - # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - - name: Checkout At Tag And Cherry Pick - if: ${{ inputs.release_type == 'release' }} - run: | - cd ${{ github.workspace }}/action/package/ - TEST_VERSION=$(python setup.py --version) - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - git checkout "tags/$VERSION" - git cherry-pick "${{ inputs.cherrypick_sha }}" - CHERRYPICKED_SHA=$(git rev-parse HEAD) - echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - - name: Increment Version - id: version - if: ${{ inputs.release_type == 'alpha' }} - run: | - cd ${{ github.workspace }}/action/github/scripts/ - VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - - name: change working directory - run: cd ${{ github.workspace }}/action/package/ - - name: Generate Release Changelog (cropped) - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.VERSION }} - - name: Create Pre-release - id: create_release - uses: ncipollo/release-action@v1 - with: - token: ${{ secrets.GH_PAT }} - tag: ${{ env.NEXT_VERSION }} - name: Release ${{ env.NEXT_VERSION }} - body: | - Changes in this Release - ${{ steps.changelog.outputs.changelog }} - commit: ${{ env.CHERRYPICKED_SHA || 'dev' }} - prerelease: true - - name: Build Distribution Packages - run: | - mkdir -p ${{ github.workspace }}/action/dist - cd ${{ github.workspace }}/action/package/ - python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{secrets.PYPI_TOKEN}} - packages-dir: action/dist/ - - name: Generate Full Changelog - id: full_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} - repository: action/package/ \ No newline at end of file diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 5d4d64668..90d0eab65 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -263,6 +263,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: + token: ${{ secrets.GH_PAT }} fetch-depth: 0 ref: dev path: action/package/dev/ @@ -273,42 +274,98 @@ jobs: fetch-depth: 0 ref: testing path: action/package/testing/ + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - name: Install Python Dependencies + run: | + pip install --upgrade pip + pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered # note: the list of commits to cherrypick will be reversed, subsequent processes can use it right away - - name: set commits num to cherry pick + - name: get commit shas to cherry pick + id: get_commit_sha run: | - COMMITS = 1 + COMMITS=1 if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then - COMMITS = $((COMMITS + 1)) + COMMITS=$((COMMITS + 2)) + else + COMMITS=$((COMMITS + 1)) + fi if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then - COMMITS = $((COMMITS + 1)) + COMMITS=$((COMMITS + 1)) fi - CHERRYPICK_SHA="" + COMMIT_SHA="" while read sha do if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then - CHERRYPICK_SHA="$CHERRYPICK_SHA $sha" + COMMIT_SHA="$COMMIT_SHA $sha" fi - done < <(git log -n 4 --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') - echo "CHERRYPICK_SHA=${CHERRYPICK_SHA}" >> $GITHUB_ENV - - name: Release - uses: openvoiceos/.github/.github/actions/release_alpha@feat/shared_actions1 - with: - action_branch: ${{ inputs.action_branch }} - version_file: ${{ inputs.version_file }} - python_version: ${{ inputs.python_version }} - changelog_file: ${{ inputs.changelog_file }} - release_type: release - cherrypick_sha: ${{ env.CHERRYPICK_SHA }} - - name: Cherry pick n last commit from dev + done < <(git -C ${{ github.workspace}}/action/package/dev log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV + - name: Versioning + run: | + cd ${{ github.workspace }}/action/package/testing/ + TEST_VERSION=$(python setup.py --version) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + - name: Change Directory run: | cd ${{ github.workspace }}/action/package/dev/ - git fetch origin dev - LOG_BUMP_SHA=$(git log -n 1 --pretty=format:"%H") + - name: Generate Full Changelog + id: full_changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: action/package/dev/${{ inputs.changelog_file }} + futureRelease: ${{ env.NEXT_VERSION}} + - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} + repository: action/package/dev/ + - name: Checkout At Tag And Cherry Pick + run: | + cd ${{ github.workspace }}/action/package/testing + git checkout -b temp-branch "tags/$VERSION" + git cherry-pick "${{ env.COMMIT_SHA }}" + CHERRYPICKED_SHA=$(git rev-parse HEAD) + echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV + - name: Generate Release Changelog (cropped) + id: changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + unreleasedOnly: true + futureRelease: ${{ env.NEXT_VERSION}} + - name: Create Pre-release + id: create_release + uses: ncipollo/release-action@v1 + with: + token: ${{ secrets.GH_PAT }} + tag: ${{ env.NEXT_VERSION }} + name: Release ${{ env.NEXT_VERSION }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + commit: ${{ env.CHERRYPICKED_SHA}} + prerelease: true + - name: Build Distribution Packages + run: | + mkdir -p ${{ github.workspace }}/action/dist cd ${{ github.workspace }}/action/package/testing/ - git cherry-pick ${{ env.CHERRYPICK_SHA }} "$LOG_BUMP_SHA" + python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} + packages-dir: action/dist/ - name: Push changes to testing branch run: | cd ${{ github.workspace }}/action/package/testing/ - git push origin testing + git push --force origin testing From 97867f5214c0393f0a34027a9e18e1495a081a56 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 13 Mar 2024 19:27:39 +0100 Subject: [PATCH 242/357] test --- .github/workflows/release_semver_start.yml | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 90d0eab65..db25ab603 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -266,14 +266,13 @@ jobs: token: ${{ secrets.GH_PAT }} fetch-depth: 0 ref: dev - path: action/package/dev/ - - name: Checkout testing branch + path: action/package/ + - name: Checkout Scripts Repo uses: actions/checkout@v4 with: - token: ${{ secrets.GH_PAT }} - fetch-depth: 0 - ref: testing - path: action/package/testing/ + repository: OpenVoiceOS/.github + ref: ${{ inputs.action_branch }} + path: action/github/ - name: Setup Python uses: actions/setup-python@v5 with: @@ -289,6 +288,7 @@ jobs: - name: get commit shas to cherry pick id: get_commit_sha run: | + cd ${{ github.workspace }}/action/package/ COMMITS=1 if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then COMMITS=$((COMMITS + 2)) @@ -304,40 +304,39 @@ jobs: if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then COMMIT_SHA="$COMMIT_SHA $sha" fi - done < <(git -C ${{ github.workspace}}/action/package/dev log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Versioning run: | - cd ${{ github.workspace }}/action/package/testing/ + cd ${{ github.workspace }}/action/package/ + git checkout testing TEST_VERSION=$(python setup.py --version) VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - - name: Change Directory - run: | - cd ${{ github.workspace }}/action/package/dev/ + git checkout dev - name: Generate Full Changelog id: full_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - output: action/package/dev/${{ inputs.changelog_file }} + output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.NEXT_VERSION}} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} - repository: action/package/dev/ + repository: action/package/ - name: Checkout At Tag And Cherry Pick run: | - cd ${{ github.workspace }}/action/package/testing + cd ${{ github.workspace }}/action/package/ git checkout -b temp-branch "tags/$VERSION" git cherry-pick "${{ env.COMMIT_SHA }}" CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Generate Release Changelog (cropped) - id: changelog + id: release_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} @@ -352,13 +351,13 @@ jobs: name: Release ${{ env.NEXT_VERSION }} body: | Changes in this Release - ${{ steps.changelog.outputs.changelog }} + ${{ steps.release_changelog.outputs.changelog }} commit: ${{ env.CHERRYPICKED_SHA}} prerelease: true - name: Build Distribution Packages run: | mkdir -p ${{ github.workspace }}/action/dist - cd ${{ github.workspace }}/action/package/testing/ + cd ${{ github.workspace }}/action/package/ python setup.py sdist bdist_wheel --dist-dir ${{ github.workspace }}/action/dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -367,5 +366,5 @@ jobs: packages-dir: action/dist/ - name: Push changes to testing branch run: | - cd ${{ github.workspace }}/action/package/testing/ + cd ${{ github.workspace }}/action/package/ git push --force origin testing From aa20eac3cec498a9e6f4b6bfa6b26b901efba3f0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 13 Mar 2024 20:08:36 +0100 Subject: [PATCH 243/357] tests --- .github/workflows/release_semver_start.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index db25ab603..f60685500 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -41,7 +41,7 @@ jobs: parse_conventional_commits: outputs: release_type: ${{ steps.parse.outputs.release_type }} - was_pr: ${{ steps.parse.outputs.was_pr }} + was_pr: ${{ steps.pr_check.outputs.was_pr }} runs-on: ubuntu-latest steps: - name: Checkout Package Repo @@ -69,6 +69,7 @@ jobs: pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt # check if the last commit was due to a PR merge (needed below) - name: PR merge? + id: pr_check run: | cd ${{ github.workspace }}/action/package/ LAST_COMMIT_SHA=$(git rev-parse HEAD) @@ -301,9 +302,13 @@ jobs: COMMIT_SHA="" while read sha do - if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then - COMMIT_SHA="$COMMIT_SHA $sha" + if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then + if [ -z "$COMMIT_SHA" ]; then + COMMIT_SHA="$sha" + else + COMMIT_SHA="$COMMIT_SHA $sha" fi + fi done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Versioning From 6f23c15880c46b0bd19dcdd39005258098f27e4f Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 18:30:26 +0100 Subject: [PATCH 244/357] tests --- .github/workflows/release_alpha.yml | 29 +++++++++++----------- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 8b0fda017..189a1ee31 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -72,7 +72,7 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV git checkout "tags/$VERSION" - git cherry-pick "${{ inputs.cherrypick_sha }}" + git cherry-pick ${{ inputs.cherrypick_sha }} CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Increment Version @@ -87,12 +87,24 @@ jobs: - name: change working directory run: cd ${{ github.workspace }}/action/package/ - name: Generate Release Changelog (cropped) - id: changelog + id: release_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} unreleasedLabel: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} + - name: Generate Full Changelog + id: full_changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + futureRelease: ${{ env.NEXT_VERSION}} + output: action/package/${{ inputs.changelog_file }} + - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} + repository: action/package/ - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 @@ -102,7 +114,7 @@ jobs: name: Release ${{ env.NEXT_VERSION }} body: | Changes in this Release - ${{ steps.changelog.outputs.changelog }} + ${{ steps.release_changelog.outputs.changelog }} commit: ${{ env.CHERRYPICKED_SHA || 'dev' }} prerelease: true - name: Build Distribution Packages @@ -115,14 +127,3 @@ jobs: with: password: ${{secrets.PYPI_TOKEN}} packages-dir: action/dist/ - - name: Generate Full Changelog - id: full_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} - repository: action/package/ diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index f60685500..508f9aa9c 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -337,7 +337,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git checkout -b temp-branch "tags/$VERSION" - git cherry-pick "${{ env.COMMIT_SHA }}" + git cherry-pick ${{ env.COMMIT_SHA }} CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Generate Release Changelog (cropped) From 27854698d722f3aa07649a65be9dd27e9d6cb290 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 19:02:12 +0100 Subject: [PATCH 245/357] tests --- .github/workflows/check_conventional_commits.yml | 2 +- .github/workflows/clear_test_phase.yml | 4 +--- .github/workflows/release_semver_publish.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 5d574da74..c56d5adb0 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -48,7 +48,7 @@ jobs: python -m pip install --upgrade pip pip install -r ${{ github.workspace }}/requirements/pip_base.txt - name: Get Cached Test Status - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: test-status key: test-status- diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml index ca4af33ea..a2707cd56 100644 --- a/.github/workflows/clear_test_phase.yml +++ b/.github/workflows/clear_test_phase.yml @@ -10,9 +10,7 @@ jobs: run: | echo "complete" > test-status - name: Cache test status - uses: actions/cache@v2 + uses: actions/cache/save@v4 with: path: test-status key: test-status-${{ github.run_id }} - restore-keys: | - test-status- \ No newline at end of file diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index f5801bc37..211351313 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -96,7 +96,7 @@ jobs: run: | echo "complete" > test-status - name: Cache test status - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: test-status key: test-status-${{ github.run_id }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 508f9aa9c..34e35a5d4 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -241,7 +241,7 @@ jobs: run: | echo "testing" > test-status - name: Cache test status - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: test-status key: test-status-${{ github.run_id }} From 8a4e90d48843c39521c1ab9f185ecfd5e986bede Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 19:41:13 +0100 Subject: [PATCH 246/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 189a1ee31..1e048aed2 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -67,7 +67,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ TEST_VERSION=$(python setup.py --version) - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 34e35a5d4..14481343f 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -316,7 +316,7 @@ jobs: cd ${{ github.workspace }}/action/package/ git checkout testing TEST_VERSION=$(python setup.py --version) - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --last) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV From f7da58409febf697d3744ee19f7c297d1219cfa2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 20:24:02 +0100 Subject: [PATCH 247/357] tests --- .github/workflows/release_semver_start.yml | 44 +++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 14481343f..45fe2eb63 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -283,6 +283,28 @@ jobs: pip install --upgrade pip pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt + - name: Versioning + run: | + cd ${{ github.workspace }}/action/package/ + git checkout testing + TEST_VERSION=$(python setup.py --version) + VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV + git checkout dev + - name: Generate Full Changelog + id: full_changelog + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GH_PAT }} + output: action/package/${{ inputs.changelog_file }} + futureRelease: ${{ env.NEXT_VERSION}} + - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} + repository: action/package/ # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered # note: the list of commits to cherrypick will be reversed, subsequent processes can use it right away @@ -311,28 +333,6 @@ jobs: fi done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - - name: Versioning - run: | - cd ${{ github.workspace }}/action/package/ - git checkout testing - TEST_VERSION=$(python setup.py --version) - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - git checkout dev - - name: Generate Full Changelog - id: full_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.NEXT_VERSION}} - - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} - repository: action/package/ - name: Checkout At Tag And Cherry Pick run: | cd ${{ github.workspace }}/action/package/ From 36ace76512046a063c478a66708a5e388fc5e7c7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 20:30:56 +0100 Subject: [PATCH 248/357] tests --- .github/workflows/release_semver_start.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 45fe2eb63..c8800d58f 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -321,6 +321,7 @@ jobs: if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi + echo "Cherry-picking $COMMITS commits" COMMIT_SHA="" while read sha do @@ -332,6 +333,7 @@ jobs: fi fi done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + echo "Cherry-picking commits: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick run: | From 391e97b4b9cb7138232961a60e5c2e7992bba06a Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 21:28:33 +0100 Subject: [PATCH 249/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c8800d58f..e8cc5d45d 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -332,7 +332,7 @@ jobs: COMMIT_SHA="$COMMIT_SHA $sha" fi fi - done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tac) echo "Cherry-picking commits: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick From 7dea1fbb889ca9278a69fed48ad111b59462d424 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 22:03:09 +0100 Subject: [PATCH 250/357] tests --- .github/workflows/release_semver_start.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index e8cc5d45d..a808a3e4b 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -307,11 +307,12 @@ jobs: repository: action/package/ # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered - # note: the list of commits to cherrypick will be reversed, subsequent processes can use it right away - name: get commit shas to cherry pick id: get_commit_sha run: | cd ${{ github.workspace }}/action/package/ + git checkout dev + git pull origin dev COMMITS=1 if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then COMMITS=$((COMMITS + 2)) @@ -331,8 +332,10 @@ jobs: else COMMIT_SHA="$COMMIT_SHA $sha" fi + else + echo "Skipping merge commit $sha" fi - done < <(git log -n "$COMMITS" --pretty=format:"%H" origin/dev | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tac) + done < <(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') echo "Cherry-picking commits: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick From f74d243ecf589ba65f9aed8a34ded85d565bbcee Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 22:46:56 +0100 Subject: [PATCH 251/357] tests --- .github/workflows/release_semver_start.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index a808a3e4b..42802e3dd 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -322,8 +322,9 @@ jobs: if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi - echo "Cherry-picking $COMMITS commits" COMMIT_SHA="" + ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + echo "Considerring $COMMITS commits: $ALL_SHA" while read sha do if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then @@ -335,8 +336,8 @@ jobs: else echo "Skipping merge commit $sha" fi - done < <(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') - echo "Cherry-picking commits: $COMMIT_SHA" + done <<< "$ALL_SHA" + echo "Filtered (non-merge) commits: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick run: | From 581437e35a0b0218872a9c728cd058b15d687251 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 14 Mar 2024 23:33:22 +0100 Subject: [PATCH 252/357] tests --- .github/workflows/release_semver_start.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 42802e3dd..6f1743483 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -323,8 +323,10 @@ jobs: COMMITS=$((COMMITS + 1)) fi COMMIT_SHA="" - ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') - echo "Considerring $COMMITS commits: $ALL_SHA" + ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H") + echo "Git SHA: $(echo $ALL_SHA | tr '\n' ' ')" + ALL_SHA=$(echo $ALL_SHA | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + echo "Considerring $COMMITS commits: $(echo $ALL_SHA | tr '\n' ' ')" while read sha do if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then From 09cd260fa1e143c1a213c555645233604d893f70 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 00:32:55 +0100 Subject: [PATCH 253/357] tests --- .github/workflows/release_semver_start.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 6f1743483..2045d8cb3 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -322,13 +322,19 @@ jobs: if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi + echo "Commit History (max. 10):" + echo "--------------" + git log --pretty=format:"%h %s" -s HEAD --max-count=10 + echo "--------------" COMMIT_SHA="" - ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H") - echo "Git SHA: $(echo $ALL_SHA | tr '\n' ' ')" - ALL_SHA=$(echo $ALL_SHA | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') + ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') echo "Considerring $COMMITS commits: $(echo $ALL_SHA | tr '\n' ' ')" + echo "HEAD: $(git rev-parse HEAD)" while read sha do + echo "--------------" + echo "SHA: $sha" + echo "Parents: $(git rev-list --parents -n 1 $sha)" if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then if [ -z "$COMMIT_SHA" ]; then COMMIT_SHA="$sha" From 2de9e9e2bbe96fe17a4f31f52d3edba15a50c1b2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 01:53:48 +0100 Subject: [PATCH 254/357] tests --- .github/workflows/release_semver_start.yml | 33 ++++------------------ 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2045d8cb3..2f2403d49 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -307,45 +307,22 @@ jobs: repository: action/package/ # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered + # note: COMMITS=2 (PR commit + version bump) - name: get commit shas to cherry pick id: get_commit_sha run: | cd ${{ github.workspace }}/action/package/ git checkout dev - git pull origin dev - COMMITS=1 - if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then - COMMITS=$((COMMITS + 2)) - else - COMMITS=$((COMMITS + 1)) - fi + COMMITS=2 if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi echo "Commit History (max. 10):" echo "--------------" git log --pretty=format:"%h %s" -s HEAD --max-count=10 - echo "--------------" - COMMIT_SHA="" - ALL_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') - echo "Considerring $COMMITS commits: $(echo $ALL_SHA | tr '\n' ' ')" - echo "HEAD: $(git rev-parse HEAD)" - while read sha - do - echo "--------------" - echo "SHA: $sha" - echo "Parents: $(git rev-list --parents -n 1 $sha)" - if [ $(git rev-list --parents -n 1 $sha | wc -w) -le 2 ]; then - if [ -z "$COMMIT_SHA" ]; then - COMMIT_SHA="$sha" - else - COMMIT_SHA="$COMMIT_SHA $sha" - fi - else - echo "Skipping merge commit $sha" - fi - done <<< "$ALL_SHA" - echo "Filtered (non-merge) commits: $COMMIT_SHA" + echo "\n--------------" + COMMIT_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" --no-merges | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tr '\n' ' ') + echo "Considerring $COMMITS (non-merge) commits to cherry pick: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick run: | From 38664f5fb2b5b6cbbc646192d6d5dc63fe4925ba Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 02:02:09 +0100 Subject: [PATCH 255/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 2f2403d49..c14c86df1 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -312,6 +312,7 @@ jobs: id: get_commit_sha run: | cd ${{ github.workspace }}/action/package/ + git fetch --unshallow git checkout dev COMMITS=2 if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then From 3f487306520a89fcfb76d3b509d0aa4a683c2d64 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 02:17:59 +0100 Subject: [PATCH 256/357] tests --- .github/workflows/release_semver_start.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c14c86df1..046d324ec 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -321,13 +321,15 @@ jobs: echo "Commit History (max. 10):" echo "--------------" git log --pretty=format:"%h %s" -s HEAD --max-count=10 - echo "\n--------------" + echo -e "\n--------------" COMMIT_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" --no-merges | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tr '\n' ' ') echo "Considerring $COMMITS (non-merge) commits to cherry pick: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick run: | cd ${{ github.workspace }}/action/package/ + git config user.name "EggmanBot" + git config user.email "jarbasai@mailfence.com" git checkout -b temp-branch "tags/$VERSION" git cherry-pick ${{ env.COMMIT_SHA }} CHERRYPICKED_SHA=$(git rev-parse HEAD) From a86975be9d43a81b32e016350f785688a54888b0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 02:41:21 +0100 Subject: [PATCH 257/357] tests --- .github/workflows/release_semver_start.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 046d324ec..fa878daca 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -334,6 +334,10 @@ jobs: git cherry-pick ${{ env.COMMIT_SHA }} CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV + - name: Push changes to testing branch + run: | + cd ${{ github.workspace }}/action/package/ + git push --force origin testing - name: Generate Release Changelog (cropped) id: release_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 @@ -363,7 +367,3 @@ jobs: with: password: ${{secrets.PYPI_TOKEN}} packages-dir: action/dist/ - - name: Push changes to testing branch - run: | - cd ${{ github.workspace }}/action/package/ - git push --force origin testing From a6ed4a34c10f692993fb8c6484aaff67a23c217d Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 03:10:46 +0100 Subject: [PATCH 258/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index fa878daca..9ce8ab3b2 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -337,7 +337,7 @@ jobs: - name: Push changes to testing branch run: | cd ${{ github.workspace }}/action/package/ - git push --force origin testing + git push --force origin temp-branch:testing - name: Generate Release Changelog (cropped) id: release_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 From e66d8eb967fbf54ef4a43983303710cf717b71ca Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 15:15:00 +0100 Subject: [PATCH 259/357] tests --- .github/workflows/release_alpha.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 1e048aed2..fb74dce71 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -37,6 +37,7 @@ jobs: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: ${{ inputs.release_type == 'alpha' && 'dev' || 'testing' }} GITHUB_REPOSITORY: ${{ github.repository }} + GH_PAT: ${{ secrets.GH_PAT }} steps: - name: Checkout Scripts Repo uses: actions/checkout@v4 From d6376a20f083fcea29697274bfcda953354347b6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 15:55:53 +0100 Subject: [PATCH 260/357] tests --- .github/workflows/release_semver_start.yml | 5 +++++ scripts/semver_release_version.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 9ce8ab3b2..3148938d5 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -345,6 +345,11 @@ jobs: token: ${{ secrets.GH_PAT }} unreleasedOnly: true futureRelease: ${{ env.NEXT_VERSION}} + # not globally bumped, only for the release + - name: Bump to next alpha version (for the release) + run: | + cd ${{ github.workspace }}/action/package/ + python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.NEXT_VERSION }} --save ${{ inputs.version_file }} - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index 675d3d2a7..9b6d89de2 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -376,7 +376,8 @@ def filter_versions(self, release_type: Optional[str] = None, cycle_only: bool = if (args.save or args.fsave) and version is not None: file = args.file or args.save or args.fsave - if version > base or args.fsave: + if (version > base or args.fsave) or \ + all( arg is False for arg in [args.first, args.last, args.next] ): version.to_file(file) if version is not None: From e68f64e119c3256262be24a36c7a93a74c30f9d1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 16:23:20 +0100 Subject: [PATCH 261/357] tests --- .github/workflows/release_semver_start.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 3148938d5..4991d2bb8 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -259,6 +259,10 @@ jobs: cherry_pick_testing: needs: [parse_conventional_commits, autotranslate] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' }} + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GH_PAT: ${{ secrets.GH_PAT }} + VERSION_FILE: ${{ github.workspace}}/action/package/${{ inputs.version_file }} runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -348,8 +352,7 @@ jobs: # not globally bumped, only for the release - name: Bump to next alpha version (for the release) run: | - cd ${{ github.workspace }}/action/package/ - python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.NEXT_VERSION }} --save ${{ inputs.version_file }} + python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.NEXT_VERSION }} --save ${{ env.VERSION_FILE }} - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 From b9972a534e52421fc3a33be6593dc8ae416a5d17 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 19:07:11 +0100 Subject: [PATCH 262/357] tests --- scripts/semver_release_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index 9b6d89de2..9df4ea425 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -341,7 +341,7 @@ def filter_versions(self, release_type: Optional[str] = None, cycle_only: bool = if cycle_only: filtered_versions = filter(lambda v: v.major == self.major and v.minor == self.minor and v.patch == self.patch, filtered_versions) elif release_type is None: - filtered_versions = self.history[:] + filtered_versions = [v for v in self.history if v <= self] if cycle_only: filtered_versions = filter(lambda v: v.major == self.major, filtered_versions) else: From f83d52ed6c88ec2ff30dad351ff97d627635ade4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 20:33:53 +0100 Subject: [PATCH 263/357] tests --- .github/workflows/release_semver_start.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 4991d2bb8..0b2192421 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -292,6 +292,7 @@ jobs: cd ${{ github.workspace }}/action/package/ git checkout testing TEST_VERSION=$(python setup.py --version) + echo "TEST_VERSION=${TEST_VERSION}" >> $GITHUB_ENV VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) echo "VERSION=${VERSION}" >> $GITHUB_ENV @@ -333,7 +334,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git config user.name "EggmanBot" - git config user.email "jarbasai@mailfence.com" + git config user.email "openvoiceos@gmail.com" git checkout -b temp-branch "tags/$VERSION" git cherry-pick ${{ env.COMMIT_SHA }} CHERRYPICKED_SHA=$(git rev-parse HEAD) @@ -341,7 +342,7 @@ jobs: - name: Push changes to testing branch run: | cd ${{ github.workspace }}/action/package/ - git push --force origin temp-branch:testing + git push --force origin temp-branch:testing - name: Generate Release Changelog (cropped) id: release_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 @@ -349,6 +350,14 @@ jobs: token: ${{ secrets.GH_PAT }} unreleasedOnly: true futureRelease: ${{ env.NEXT_VERSION}} + - name: (Re)Set Release Version + run: | + python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} + - name: Commit Release Version + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Increment Version to ${{ env.TEST_VERSION }} + repository: action/package/ # not globally bumped, only for the release - name: Bump to next alpha version (for the release) run: | From 5a82c891b084f162908d0fd49a1befeba769258d Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 21:21:34 +0100 Subject: [PATCH 264/357] tests --- scripts/semver_release_version.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/semver_release_version.py b/scripts/semver_release_version.py index 9df4ea425..8fabdbfd7 100644 --- a/scripts/semver_release_version.py +++ b/scripts/semver_release_version.py @@ -36,7 +36,7 @@ --last # get the last release of that release type in the current cycle --next # get the next release of the upcoming release type --first # get the first release of that release type in the current cycle - none of above # get the latest version released + --latest # get the latest version released """ @@ -108,18 +108,18 @@ def __str__(self) -> str: return f"{self.__prefix}{super().__str__()}" def next(self, rtype: Optional[str], alpha_marker: str = ALPHA_MARKER)\ - -> Optional["OVOSReleases"]: + -> "OVOSReleases": rtype = rtype or "prerelease" - - return self.next_version(rtype, alpha_marker) + next_v = self.next_version(rtype, alpha_marker) + return OVOSReleases(next_v.major, next_v.minor, next_v.patch, next_v.prerelease, next_v.build) - def latest(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + def latest(self, rtype: Optional[str] = None) -> "OVOSReleases": if rtype and not self.history: raise ValueError("No release history") release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) - latest_version = None + latest_version = OVOSReleases(0, 0, 0) if rtype is None and (not self.history or not release_versions): latest_version = self elif release_versions: @@ -127,25 +127,25 @@ def latest(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: return latest_version - def last(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + def last(self, rtype: Optional[str] = None) -> "OVOSReleases": if not self.history: raise ValueError("No release history") release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) - last_version = None + last_version = OVOSReleases(0, 0, 0) if release_versions: last_version = release_versions[min(1, len(release_versions) - 1)] return last_version - def first(self, rtype: Optional[str] = None) -> Optional["OVOSReleases"]: + def first(self, rtype: Optional[str] = None) -> "OVOSReleases": if not self.history: raise ValueError("No release history") release_versions = self.filter_versions(rtype, RESTRICT_TO_CYCLE) - first_version = None + first_version = OVOSReleases(0, 0, 0) if release_versions: first_version = min(release_versions) @@ -351,7 +351,7 @@ def filter_versions(self, release_type: Optional[str] = None, cycle_only: bool = # instanciate the class history -releases = OVOSReleases.from_repo(REPOSITORY, getenv("GH_PAT")) +releases = OVOSReleases.from_repo(REPOSITORY, getenv("GH_PAT") or getenv("GITHUB_TOKEN")) # if version or file is provided, get the version from the repository history or use the provided version if args.version or args.file: base = releases.get(args.version, args.file) @@ -371,8 +371,10 @@ def filter_versions(self, release_type: Optional[str] = None, cycle_only: bool = version = base.last(RELEASE_TYPE) elif args.next: version = base.next(RELEASE_TYPE) -else: +elif args.latest: version = base.latest(RELEASE_TYPE) +else: + version = base if (args.save or args.fsave) and version is not None: file = args.file or args.save or args.fsave From a739a1543d68407181b59a7bbb5f5de3e7433f35 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 21:32:54 +0100 Subject: [PATCH 265/357] tests --- .github/workflows/release_alpha.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index fb74dce71..d87821295 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -81,7 +81,7 @@ jobs: if: ${{ inputs.release_type == 'alpha' }} run: | cd ${{ github.workspace }}/action/github/scripts/ - VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) + VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }} --latest) echo "VERSION=${VERSION}" >> $GITHUB_ENV NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV From 68c76e60fa69dc3a5c470b4903c6500926560446 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 22:06:55 +0100 Subject: [PATCH 266/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 0b2192421..003bb57ea 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -358,6 +358,7 @@ jobs: with: commit_message: Increment Version to ${{ env.TEST_VERSION }} repository: action/package/ + branch: testing # not globally bumped, only for the release - name: Bump to next alpha version (for the release) run: | From f7cbe833c7d4329c414d18d855add2464980a014 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 22:29:11 +0100 Subject: [PATCH 267/357] tests --- .github/workflows/release_semver_start.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 003bb57ea..59e9e4bb2 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -353,12 +353,11 @@ jobs: - name: (Re)Set Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} - - name: Commit Release Version - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Increment Version to ${{ env.TEST_VERSION }} - repository: action/package/ - branch: testing + git commit -am "Increment Version to ${{ env.TEST_VERSION }}" + - name: Push Release Version to testing branch + run: | + cd ${{ github.workspace }}/action/package/ + git push origin temp-branch:testing # not globally bumped, only for the release - name: Bump to next alpha version (for the release) run: | From c2c9476205892bbc739c2a6dc9c7a408f39b52d3 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 22:47:18 +0100 Subject: [PATCH 268/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 59e9e4bb2..ef4b66c6e 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -353,6 +353,7 @@ jobs: - name: (Re)Set Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} + cd ${{ github.workspace }}/action/package/ git commit -am "Increment Version to ${{ env.TEST_VERSION }}" - name: Push Release Version to testing branch run: | From 6338d26887d6cd0bb43b17d36c5bfaf61480b088 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 15 Mar 2024 22:48:40 +0100 Subject: [PATCH 269/357] tests --- .github/workflows/release_semver_start.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index ef4b66c6e..0453ff581 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -350,14 +350,11 @@ jobs: token: ${{ secrets.GH_PAT }} unreleasedOnly: true futureRelease: ${{ env.NEXT_VERSION}} - - name: (Re)Set Release Version + - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} cd ${{ github.workspace }}/action/package/ git commit -am "Increment Version to ${{ env.TEST_VERSION }}" - - name: Push Release Version to testing branch - run: | - cd ${{ github.workspace }}/action/package/ git push origin temp-branch:testing # not globally bumped, only for the release - name: Bump to next alpha version (for the release) From 56825bf9b838fcdd5cd2ab061eb8e7a765930a84 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 00:50:43 +0100 Subject: [PATCH 270/357] tests --- .github/workflows/release_alpha.yml | 32 +++++++--------------- .github/workflows/release_semver_start.yml | 12 -------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index d87821295..7bf655b7c 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -20,14 +20,11 @@ on: description: 'The file location to write the changelog to' type: string default: CHANGELOG.md - # internal workflow variables + # release type other than alpha would need some semver foo release_type: description: 'The type of release to make' type: string default: alpha - cherrypick_sha: - description: 'the SHA(s) to cherry pick as a whitespace separated string' - type: string jobs: @@ -61,29 +58,20 @@ jobs: ref: ${{ env.SOURCE_BRANCH }} path: action/package/ fetch-depth: 0 - # testing specific steps. Those releases belong to the previous release cycle - # to be included in the major/minor/patch release we have to step down to the last release tag and increase from there - - name: Checkout At Tag And Cherry Pick - if: ${{ inputs.release_type == 'release' }} - run: | - cd ${{ github.workspace }}/action/package/ - TEST_VERSION=$(python setup.py --version) - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - git checkout "tags/$VERSION" - git cherry-pick ${{ inputs.cherrypick_sha }} - CHERRYPICKED_SHA=$(git rev-parse HEAD) - echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV + # note: "bump-prelease" is a special operation, as it doesn't preserve order + # normally the next prerelease version of 0.3.0 would be 0.3.1-a.1 + # so we bump the patch/minor/major version and then add the prerelease - name: Increment Version id: version - if: ${{ inputs.release_type == 'alpha' }} run: | cd ${{ github.workspace }}/action/github/scripts/ VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }} --latest) + if [ "${{ inputs.release_type }}" != 'alpha' ]; then + NEXT_VERSION=$( python semver_release_version.py --next --type ${{ inputs.release_type }} --file ${{ env.VERSION_FILE }} --bump-prerelease --save) + else + NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) + fi echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: change working directory run: cd ${{ github.workspace }}/action/package/ @@ -116,7 +104,7 @@ jobs: body: | Changes in this Release ${{ steps.release_changelog.outputs.changelog }} - commit: ${{ env.CHERRYPICKED_SHA || 'dev' }} + commit: dev prerelease: true - name: Build Distribution Packages run: | diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 0453ff581..57a5b2f06 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -67,18 +67,6 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - # check if the last commit was due to a PR merge (needed below) - - name: PR merge? - id: pr_check - run: | - cd ${{ github.workspace }}/action/package/ - LAST_COMMIT_SHA=$(git rev-parse HEAD) - PR_NUMBER=$(curl --silent --show-error --header "Authorization: Bearer ${{ secrets.GH_PAT }}" "https://api.github.com/repos/${{ github.repository }}/commits/${LAST_COMMIT_SHA}/pulls" | jq '.[0].number') - if [ -n "$PR_NUMBER" ]; then - echo "was_pr=true" >> $GITHUB_OUTPUT - else - echo "was_pr=false" >> $GITHUB_OUTPUT - fi - name: Parse Conventional Commits id: parse run: | From 84b08bd34caf06415cd42361674809c85b65aa6a Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 15:48:14 +0100 Subject: [PATCH 271/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_pull_master.yml | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 7bf655b7c..9c6d5d4df 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -60,7 +60,7 @@ jobs: fetch-depth: 0 # note: "bump-prelease" is a special operation, as it doesn't preserve order # normally the next prerelease version of 0.3.0 would be 0.3.1-a.1 - # so we bump the patch/minor/major version and then add the prerelease + # so we bump the patch/minor/major version and then add the prerelease (0.3.0 -> 0.3.0-a.1) - name: Increment Version id: version run: | diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 51e580965..fec773d82 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -52,7 +52,7 @@ jobs: pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - name: Get Version run: | - VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) + export VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Parse Release Type run: | @@ -62,11 +62,6 @@ jobs: run: | LATEST_RELEASE=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type ${{ env.RELEASE_TYPE }}) echo "LATEST_RELEASE=${LATEST_RELEASE}" >> $GITHUB_ENV - - name: Bump Changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - name: create release changelog (cropped) id: changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 From 6cd60256371ce8f83592913e92208239d2b1d166 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 16:00:24 +0100 Subject: [PATCH 272/357] tests --- .github/workflows/release_semver_start.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 57a5b2f06..544903817 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -338,6 +338,7 @@ jobs: token: ${{ secrets.GH_PAT }} unreleasedOnly: true futureRelease: ${{ env.NEXT_VERSION}} + sinceTag: ${{ env.VERSION }} - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} From ae5c5f50b8e3aae132953a2d0fec858f921a6a69 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 17:40:13 +0100 Subject: [PATCH 273/357] tests --- .github/workflows/release_semver_start.yml | 1 - scripts/check_PRs.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 544903817..913e03c5c 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -336,7 +336,6 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - unreleasedOnly: true futureRelease: ${{ env.NEXT_VERSION}} sinceTag: ${{ env.VERSION }} - name: (Re)Set and push Release Version diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 9f88eb0e9..a03b9e99e 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -56,7 +56,6 @@ def parse_cc(desc: str) -> Optional[pccc.ConventionalCommitRunner]: def check_cc_labels(desc: str) -> List[str]: - global cc_missing labels = set() _type = cc_type(desc) From 03132af959322ab75fb4d9aff8de743260942f7c Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 18:20:50 +0100 Subject: [PATCH 274/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 913e03c5c..8dc3a60b2 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -336,7 +336,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - futureRelease: ${{ env.NEXT_VERSION}} + unreleasedLabel: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} - name: (Re)Set and push Release Version run: | From e6503f5de1d41db271d6a8af94976090bda5b53e Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 19:51:59 +0100 Subject: [PATCH 275/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 8dc3a60b2..82f281eb5 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -292,7 +292,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.NEXT_VERSION}} + futureRelease: ${{ env.NEXT_VERSION }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -336,7 +336,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - unreleasedLabel: ${{ env.NEXT_VERSION }} + futureRelease: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} - name: (Re)Set and push Release Version run: | From 0ef4fed347a8ef8de72385243f5f45475a28091b Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 20:05:49 +0100 Subject: [PATCH 276/357] tests --- .github/workflows/release_semver_start.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 82f281eb5..8a2977ed2 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -67,6 +67,17 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt + - name : Check if latest commit is a merge commit + id: merge_check + run: | + cd ${{ github.workspace }}/action/package/ + LATEST_COMMIT_HASH=$(git rev-parse HEAD) + PARENTS_COUNT=$(git cat-file -p $LATEST_COMMIT_HASH | grep -c parent) + if [ $PARENTS_COUNT -gt 1 ]; then + echo "was_pr=true" >> $GITHUB_ENV + else + echo "was_pr=false" >> $GITHUB_ENV + fi - name: Parse Conventional Commits id: parse run: | @@ -300,7 +311,7 @@ jobs: repository: action/package/ # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered - # note: COMMITS=2 (PR commit + version bump) + # note: COMMITS=2 (commit + version bump) - name: get commit shas to cherry pick id: get_commit_sha run: | @@ -308,6 +319,9 @@ jobs: git fetch --unshallow git checkout dev COMMITS=2 + if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then + COMMITS=$((COMMITS + 1)) + fi if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi From 738c6367792f4a94d9dd6109a5f673274a8acb30 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 20:14:27 +0100 Subject: [PATCH 277/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 8a2977ed2..8faa6a36e 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -351,7 +351,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} futureRelease: ${{ env.NEXT_VERSION }} - sinceTag: ${{ env.VERSION }} + unreleasedOnly: true - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} From 86e9bb123f3a2221743b535c0fe4402ea7668469 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 20:49:03 +0100 Subject: [PATCH 278/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 8faa6a36e..9ee4a92a3 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -329,8 +329,8 @@ jobs: echo "--------------" git log --pretty=format:"%h %s" -s HEAD --max-count=10 echo -e "\n--------------" - COMMIT_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" --no-merges | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tr '\n' ' ') - echo "Considerring $COMMITS (non-merge) commits to cherry pick: $COMMIT_SHA" + COMMIT_SHA=$(git log -n "$COMMITS" --pretty=format:"%H" | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' | tr '\n' ' ') + echo "Considerring $COMMITS commits to cherry pick: $COMMIT_SHA" echo "COMMIT_SHA=${COMMIT_SHA}" >> $GITHUB_ENV - name: Checkout At Tag And Cherry Pick run: | From f8b1d0d96abc9c8e75a57baeefa01d296dbadc61 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 21:03:36 +0100 Subject: [PATCH 279/357] tests --- scripts/check_PRs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index a03b9e99e..3d9a3878d 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -88,9 +88,10 @@ def check_cc_labels(desc: str) -> List[str]: # clear the test flag if the PR adresses a release. Ie. gets added to the test if SINGLE_PR: - if cc_type(pr_description) == "release": + if cc_type(pr_description) == "release" or \ + cc_scope(pr_description) == "release": ongoing_test = False - elif cc_type(pr_description) == "unknown": + if cc_type(pr_description) == "unknown": cc_missing = True # nuke status check (if requested) From ecc72148a23a47f598554e511163552dbe56a0bd Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 16 Mar 2024 21:22:42 +0100 Subject: [PATCH 280/357] tests --- scripts/check_PRs.py | 3 ++- scripts/parse_semver_release.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 3d9a3878d..3c7494506 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -60,6 +60,7 @@ def check_cc_labels(desc: str) -> List[str]: labels = set() _type = cc_type(desc) _scope = cc_scope(desc) + test_relevant_cc = ["feat", "fix", "refactor", "breaking"] if _type == "unknown": return [PR_LABELS.get("need_cc", "CC missing")] if _type == "breaking": @@ -68,7 +69,7 @@ def check_cc_labels(desc: str) -> List[str]: labels.add(PR_LABELS.get(_scope)) if _type in PR_LABELS: labels.add(PR_LABELS.get(_type)) - if ongoing_test and not any(t == "release" for t in [_type, _scope]): + if ongoing_test and any(t in test_relevant_cc for t in [_type, _scope]): labels.add("ongoing test") return list(labels) diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index 9a33a3ce1..b1fa6212d 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -39,7 +39,7 @@ def semver_from_cc(): return "release" elif ccr.header.get("type") == "feat": return "minor" - elif ccr.header.get("type") == "fix": + elif ccr.header.get("type") in ["fix", "refactor"]: return "patch" elif ccr.header.get("type") not in ["ci", "docs", "style", "test"]: return "alpha" From c6eefbb7a675bbba541b91597f8ae3566b46342e Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 15:26:50 +0100 Subject: [PATCH 281/357] tests --- .github/workflows/release_semver_start.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 9ee4a92a3..7cd6fb979 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -41,7 +41,7 @@ jobs: parse_conventional_commits: outputs: release_type: ${{ steps.parse.outputs.release_type }} - was_pr: ${{ steps.pr_check.outputs.was_pr }} + was_pr: ${{ steps.merge_check.outputs.was_pr }} runs-on: ubuntu-latest steps: - name: Checkout Package Repo @@ -338,7 +338,16 @@ jobs: git config user.name "EggmanBot" git config user.email "openvoiceos@gmail.com" git checkout -b temp-branch "tags/$VERSION" - git cherry-pick ${{ env.COMMIT_SHA }} + IFS=' ' read -ra ADDR <<< "${{ env.COMMIT_SHA }}" + for i in "${ADDR[@]}"; do + PARENTS_COUNT=$(git cat-file -p $i | grep -c parent) + if [ $PARENTS_COUNT -gt 1 ] + then + git cherry-pick -m 1 $i + else + git cherry-pick $i + fi + done CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV - name: Push changes to testing branch From a05234d1db58228d70fa7a7e43edacc5e7484fc1 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 15:44:31 +0100 Subject: [PATCH 282/357] tests --- .github/workflows/check_conventional_commits.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index c56d5adb0..700db5565 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -48,10 +48,12 @@ jobs: python -m pip install --upgrade pip pip install -r ${{ github.workspace }}/requirements/pip_base.txt - name: Get Cached Test Status - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: test-status key: test-status- + restore-keys: | + test-status- - name: Status Check run: | python scripts/check_PRs.py From 394d3800199bfb560002457068098c0161a58739 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 15:52:03 +0100 Subject: [PATCH 283/357] tests --- scripts/check_PRs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 3c7494506..0ad609681 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -15,6 +15,7 @@ PR_LABELS = json.loads(open(join(dirname(dirname(__file__)), "pr_labels.json")).read()) test_phase_cache = os.getenv('TEST_PHASE_CACHE', '') +print(f"TEST_PHASE_CACHE: {test_phase_cache}") if not isfile(test_phase_cache): ongoing_test = False if test_phase_cache: @@ -22,6 +23,7 @@ else: with open(test_phase_cache, 'r') as f: ongoing_test = f.read().strip() == "testing" + print(f"ongoing_test: {ongoing_test}") def cc_type(desc: str) -> str: From 87c468d52e8feddcc3edcb727562e6039e69bd16 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 15:53:01 +0100 Subject: [PATCH 284/357] tests --- scripts/check_PRs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 0ad609681..0f2acb370 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -22,8 +22,8 @@ print("The file specified in TEST_PHASE_FILE does not exist.") else: with open(test_phase_cache, 'r') as f: + print(f"file content: {f.read().strip()}") ongoing_test = f.read().strip() == "testing" - print(f"ongoing_test: {ongoing_test}") def cc_type(desc: str) -> str: From 9f7a617fb60fe79ae40cd030ad24fed904cb84b8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 16:01:49 +0100 Subject: [PATCH 285/357] tests --- .github/workflows/release_semver_start.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 7cd6fb979..dca0b3a52 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -238,14 +238,13 @@ jobs: # set cached test status to active - name: Set test status run: | + cd ${{ github.workspace }} echo "testing" > test-status - name: Cache test status - uses: actions/cache@v4 + uses: actions/cache/save@v4 with: path: test-status key: test-status-${{ github.run_id }} - restore-keys: | - test-status- - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From 73317504916d947d66958172a271c16203ac08bd Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 16:33:51 +0100 Subject: [PATCH 286/357] tests --- .github/workflows/check_conventional_commits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 700db5565..c4754f0d4 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -51,7 +51,7 @@ jobs: uses: actions/cache/restore@v4 with: path: test-status - key: test-status- + key: test-status-${{ github.run_id }} restore-keys: | test-status- - name: Status Check From 395184a0bfedd0323df294f8b2936394b191acb8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 16:43:08 +0100 Subject: [PATCH 287/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index dca0b3a52..f4478a252 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -74,9 +74,9 @@ jobs: LATEST_COMMIT_HASH=$(git rev-parse HEAD) PARENTS_COUNT=$(git cat-file -p $LATEST_COMMIT_HASH | grep -c parent) if [ $PARENTS_COUNT -gt 1 ]; then - echo "was_pr=true" >> $GITHUB_ENV + echo "was_pr=true" >> $GITHUB_OUTPUT else - echo "was_pr=false" >> $GITHUB_ENV + echo "was_pr=false" >> $GITHUB_OUTPUT fi - name: Parse Conventional Commits id: parse From 9e5b3807d449ca0a0dd8cfb564c5bb8e21903fe6 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 16:59:32 +0100 Subject: [PATCH 288/357] tests --- .github/workflows/check_conventional_commits.yml | 2 +- .github/workflows/clear_test_phase.yml | 2 +- .github/workflows/release_semver_publish.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index c4754f0d4..f75b71553 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -51,7 +51,7 @@ jobs: uses: actions/cache/restore@v4 with: path: test-status - key: test-status-${{ github.run_id }} + key: test-status-${{ runner.os }} restore-keys: | test-status- - name: Status Check diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml index a2707cd56..e5d0ffe4b 100644 --- a/.github/workflows/clear_test_phase.yml +++ b/.github/workflows/clear_test_phase.yml @@ -13,4 +13,4 @@ jobs: uses: actions/cache/save@v4 with: path: test-status - key: test-status-${{ github.run_id }} + key: test-status-${{ runner.os }} diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 211351313..f6f21f0bc 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -99,7 +99,7 @@ jobs: uses: actions/cache@v4 with: path: test-status - key: test-status-${{ github.run_id }} + key: test-status-${{ runner.os }} restore-keys: | test-status- # NO REBASE APPROACH!! diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index f4478a252..fd137b44a 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -244,7 +244,7 @@ jobs: uses: actions/cache/save@v4 with: path: test-status - key: test-status-${{ github.run_id }} + key: test-status-${{ runner.os }} - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From 1cb896b8f6baf463411329d8dcda15f074c1a05f Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 17:27:31 +0100 Subject: [PATCH 289/357] tests --- .github/workflows/release_semver_start.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index fd137b44a..21a79197e 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -342,9 +342,9 @@ jobs: PARENTS_COUNT=$(git cat-file -p $i | grep -c parent) if [ $PARENTS_COUNT -gt 1 ] then - git cherry-pick -m 1 $i + git cherry-pick -m 1 --allow-empty $i else - git cherry-pick $i + git cherry-pick --allow-empty $i fi done CHERRYPICKED_SHA=$(git rev-parse HEAD) From e860e807f0318698b1772e57c4aab33ed1c94f02 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 18:31:22 +0100 Subject: [PATCH 290/357] tests --- scripts/check_PRs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 0f2acb370..bf026d8e0 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -22,8 +22,9 @@ print("The file specified in TEST_PHASE_FILE does not exist.") else: with open(test_phase_cache, 'r') as f: - print(f"file content: {f.read().strip()}") - ongoing_test = f.read().strip() == "testing" + content = f.read().strip("\n").strip() + print(f"file content: {content}, eq: {content == 'testing'}") + ongoing_test = content == "testing" def cc_type(desc: str) -> str: From e4a72cde643ae05d3c7493af0b94ca3acb571e31 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 18:36:06 +0100 Subject: [PATCH 291/357] tests --- pr_labels.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_labels.json b/pr_labels.json index f642ffe06..4f8924af3 100644 --- a/pr_labels.json +++ b/pr_labels.json @@ -18,7 +18,7 @@ "core": "core", "audio": "audio", "tts": "TTS", - "sst": "STT", + "stt": "STT", "nlp": "NLP", "plugins": "plugins", "localisation": "localisation", From 5f5b143a2cd6f870818ea56c221c96a61c0b7320 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 19:35:36 +0100 Subject: [PATCH 292/357] tests --- .github/workflows/release_semver_start.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 21a79197e..731df10bf 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -41,7 +41,6 @@ jobs: parse_conventional_commits: outputs: release_type: ${{ steps.parse.outputs.release_type }} - was_pr: ${{ steps.merge_check.outputs.was_pr }} runs-on: ubuntu-latest steps: - name: Checkout Package Repo @@ -67,17 +66,6 @@ jobs: - name: Install Common Python Requirements run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name : Check if latest commit is a merge commit - id: merge_check - run: | - cd ${{ github.workspace }}/action/package/ - LATEST_COMMIT_HASH=$(git rev-parse HEAD) - PARENTS_COUNT=$(git cat-file -p $LATEST_COMMIT_HASH | grep -c parent) - if [ $PARENTS_COUNT -gt 1 ]; then - echo "was_pr=true" >> $GITHUB_OUTPUT - else - echo "was_pr=false" >> $GITHUB_OUTPUT - fi - name: Parse Conventional Commits id: parse run: | @@ -310,7 +298,7 @@ jobs: repository: action/package/ # based on the workflow run, set the number of commits to cherry pick # note: merge commits are filtered - # note: COMMITS=2 (commit + version bump) + # note: COMMITS=2 (commit/merge commit + version bump) - name: get commit shas to cherry pick id: get_commit_sha run: | @@ -318,9 +306,6 @@ jobs: git fetch --unshallow git checkout dev COMMITS=2 - if [ "${{ needs.parse_conventional_commits.outputs.was_pr }}" == "true" ]; then - COMMITS=$((COMMITS + 1)) - fi if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then COMMITS=$((COMMITS + 1)) fi From d54801961008077056f2222b046a9083577d8e22 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 21:45:47 +0100 Subject: [PATCH 293/357] tests --- .github/workflows/release_semver_start.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 731df10bf..984887e80 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -324,13 +324,7 @@ jobs: git checkout -b temp-branch "tags/$VERSION" IFS=' ' read -ra ADDR <<< "${{ env.COMMIT_SHA }}" for i in "${ADDR[@]}"; do - PARENTS_COUNT=$(git cat-file -p $i | grep -c parent) - if [ $PARENTS_COUNT -gt 1 ] - then - git cherry-pick -m 1 --allow-empty $i - else - git cherry-pick --allow-empty $i - fi + git merge --allow-unrelated-histories --commit --no-ff $i done CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV From 0c6ff8a99e220cdc4bd5a6dc9fb11bb1b49cef64 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 22:06:33 +0100 Subject: [PATCH 294/357] tests --- .github/workflows/release_semver_start.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 984887e80..123ec0907 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -343,8 +343,7 @@ jobs: run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} cd ${{ github.workspace }}/action/package/ - git commit -am "Increment Version to ${{ env.TEST_VERSION }}" - git push origin temp-branch:testing + git diff --exit-code || (git commit -am "Increment Version to ${{ env.TEST_VERSION }}" && git push origin temp-branch:testing) # not globally bumped, only for the release - name: Bump to next alpha version (for the release) run: | From 3401b0e0fb6d051332636d426b777fd0fb888f3d Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 22:37:53 +0100 Subject: [PATCH 295/357] tests --- .github/workflows/release_semver_start.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 123ec0907..7d0a4d9e5 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,6 +291,7 @@ jobs: token: ${{ secrets.GH_PAT }} output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.NEXT_VERSION }} + verbose: true - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -324,7 +325,13 @@ jobs: git checkout -b temp-branch "tags/$VERSION" IFS=' ' read -ra ADDR <<< "${{ env.COMMIT_SHA }}" for i in "${ADDR[@]}"; do - git merge --allow-unrelated-histories --commit --no-ff $i + PARENTS_COUNT=$(git cat-file -p $i | grep -c parent) + if [ $PARENTS_COUNT -gt 1 ] + then + git cherry-pick -m 1 --allow-empty $i + else + git cherry-pick --allow-empty $i + fi done CHERRYPICKED_SHA=$(git rev-parse HEAD) echo "CHERRYPICKED_SHA=${CHERRYPICKED_SHA}" >> $GITHUB_ENV From 28a88a80caccac88a3f3a9e55da31c11ec84821f Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 23:11:11 +0100 Subject: [PATCH 296/357] tests --- .github/workflows/release_alpha.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 9c6d5d4df..1dddd7218 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,7 @@ jobs: uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} - unreleasedLabel: ${{ env.NEXT_VERSION }} + futureRelease: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} - name: Generate Full Changelog id: full_changelog From 6bc327abbd6909a2445ab84f36c3127f99482944 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 23:11:48 +0100 Subject: [PATCH 297/357] tests --- .github/workflows/release_semver_start.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 7d0a4d9e5..c85b3cb17 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -345,7 +345,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} futureRelease: ${{ env.NEXT_VERSION }} - unreleasedOnly: true + sinceTag: ${{ env.VERSION }} - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} From fda3548c425db21640f96bc0709b67ca5b2c15a0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 17 Mar 2024 23:22:11 +0100 Subject: [PATCH 298/357] tests --- .github/workflows/release_alpha.yml | 2 ++ .github/workflows/release_semver_publish.yml | 1 + .github/workflows/release_semver_pull_master.yml | 1 + .github/workflows/release_semver_start.yml | 2 ++ 4 files changed, 6 insertions(+) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 1dddd7218..5cff58b15 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -82,12 +82,14 @@ jobs: token: ${{ secrets.GH_PAT }} futureRelease: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} + stripGeneratorNotice: true - name: Generate Full Changelog id: full_changelog uses: heinrichreimer/github-changelog-generator-action@v2.3 with: token: ${{ secrets.GH_PAT }} futureRelease: ${{ env.NEXT_VERSION}} + stripGeneratorNotice: true output: action/package/${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index f6f21f0bc..98791cd29 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -71,6 +71,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} sinceTag: ${{ env.SINCE_TAG }} + stripGeneratorNotice: true - name: Create Release id: create_release uses: ncipollo/release-action@v1 diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index fec773d82..dcb8e4b54 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -68,6 +68,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} sinceTag: ${{ env.LATEST_RELEASE }} + stripGeneratorNotice: true # if there is no master branch, create one from the tag of the latest release or first commit - name: Create Master Branch (if necessary) run: | diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c85b3cb17..048f3fa5d 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,6 +291,7 @@ jobs: token: ${{ secrets.GH_PAT }} output: action/package/${{ inputs.changelog_file }} futureRelease: ${{ env.NEXT_VERSION }} + stripGeneratorNotice: true verbose: true - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 @@ -346,6 +347,7 @@ jobs: token: ${{ secrets.GH_PAT }} futureRelease: ${{ env.NEXT_VERSION }} sinceTag: ${{ env.VERSION }} + stripGeneratorNotice: true - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} From cd78e6df9fdd280497cf7128c1fba41bdfb5646e Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 05:56:38 +0100 Subject: [PATCH 299/357] tests --- .../workflows/release_semver_pull_master.yml | 1 + .github/workflows/release_semver_start.yml | 2 + cliff.toml | 90 +++++++++++++++++++ pyproject.toml => pccc.toml | 12 +-- scripts/parse_semver_release.py | 29 ++++-- 5 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 cliff.toml rename pyproject.toml => pccc.toml (91%) diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index dcb8e4b54..6d924a7b6 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -21,6 +21,7 @@ jobs: env: TARGET_BRANCH: 'master' GITHUB_REPOSITORY: ${{ github.repository }} + PCCC_CONFIG_FILE: ${{ github.workspace }}/action/github/pccc.toml GH_PAT: ${{ secrets.GH_PAT }} steps: - name: Checkout Repository diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 048f3fa5d..71f0be260 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -42,6 +42,8 @@ jobs: outputs: release_type: ${{ steps.parse.outputs.release_type }} runs-on: ubuntu-latest + env: + PCCC_CONFIG_FILE: ${{ github.workspace }}/action/github/pccc.toml steps: - name: Checkout Package Repo uses: actions/checkout@v4 diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 000000000..da89bd553 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,90 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/tree/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} + ### [Full Changelog](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/compare/{{ previous.version }}...{{ version }}) +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + { pattern = '', replace_command = 'sed "s##$GITHUB_REPOSITORY#"' }, +] + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com//pull/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix|^release", group = "🐛 Bug Fixes" }, + { message = "^doc", group = "📚 Documentation" }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", group = "🎨 Styling" }, + { message = "^test", group = "🧪 Testing" }, + { message = "^chore\\(release\\): prepare for", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore|^ci", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] +# protect breaking changes from being skipped due to matching a skipping commit_parser +protect_breaking_commits = false +# filter out the commits that are not matched by commit parsers +filter_commits = false +# regex for matching git tags +# tag_pattern = "v[0-9].*" +# regex for skipping tags +# skip_tags = "" +# regex for ignoring tags +# ignore_tags = "" +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" +# limit the number of commits included in the changelog. +# limit_commits = 42 diff --git a/pyproject.toml b/pccc.toml similarity index 91% rename from pyproject.toml rename to pccc.toml index ed9199aa9..7ecfb08da 100644 --- a/pyproject.toml +++ b/pccc.toml @@ -1,7 +1,5 @@ # configuration keys for the Python Conventional Commit Checker -[tool] - -[tool.pccc] +[pccc] header_length = 50 body_length = 72 @@ -26,14 +24,14 @@ types = [ "docs", "style", "test", + "release", "chore", "build", "feat", "fix", "perf", "refactor", - "revert", - "release" + "revert" ] scopes = [ @@ -49,7 +47,9 @@ scopes = [ "nlp", "plugins", "localisation", - "resources" + "resources", + "release", + "testing" ] footers = [ diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index b1fa6212d..c0cbe452d 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -1,5 +1,7 @@ import re -from os import environ +from os import environ, devnull +from os.path import isfile +import sys import pccc import semver @@ -9,6 +11,15 @@ """ +CONFIG_FILE = environ.get("PCCC_CONFIG_FILE") +if CONFIG_FILE and not isfile(CONFIG_FILE): + print(f"Config file {CONFIG_FILE} not found.") + exit(1) + +TITLE = environ.get("TITLE") +BODY = environ.get("BODY") + + def get_version(): # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" version = environ.get("VERSION", "") @@ -21,7 +32,17 @@ def get_version(): def semver_from_cc(): ccr = pccc.ConventionalCommitRunner() - ccr.options.load() + if CONFIG_FILE is None: + # Redirect stdout to null + original_stdout = sys.stdout + sys.stdout = open(devnull, 'w') + + ccr.options.load((f"-o{CONFIG_FILE}",) if CONFIG_FILE else None) + + if CONFIG_FILE is None: + # Restore original stdout + sys.stdout = original_stdout + ccr.raw = f"{TITLE}\n{BODY}" ccr.clean() try: @@ -62,9 +83,7 @@ def semver_from_version(): return "minor" elif version.major != 0: return "major" - -TITLE = environ.get("TITLE") -BODY = environ.get("BODY") + VERSION = get_version() if VERSION: From 7790f8c4ee63b2f0c92312894cd2c74e1a82e87b Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 14:43:29 +0100 Subject: [PATCH 300/357] tests --- .github/workflows/release_alpha.yml | 34 +++++++++++-------- .github/workflows/release_semver_start.yml | 38 +++++++++++++--------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 5cff58b15..d259e4e2d 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -75,22 +75,28 @@ jobs: echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: change working directory run: cd ${{ github.workspace }}/action/package/ - - name: Generate Release Changelog (cropped) + - name: Prepend changelog id: release_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 + uses: orhun/git-cliff-action@v3 with: - token: ${{ secrets.GH_PAT }} - futureRelease: ${{ env.NEXT_VERSION }} - sinceTag: ${{ env.VERSION }} - stripGeneratorNotice: true - - name: Generate Full Changelog - id: full_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - futureRelease: ${{ env.NEXT_VERSION}} - stripGeneratorNotice: true - output: action/package/${{ inputs.changelog_file }} + config: ${{ github.workspace }}/action/github/cliff.toml + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --tag ${{ env.NEXT_VERSION }} + # - name: Generate Release Changelog (cropped) + # id: release_changelog + # uses: heinrichreimer/github-changelog-generator-action@v2.3 + # with: + # token: ${{ secrets.GH_PAT }} + # futureRelease: ${{ env.NEXT_VERSION }} + # sinceTag: ${{ env.VERSION }} + # stripGeneratorNotice: true + # - name: Generate Full Changelog + # id: full_changelog + # uses: heinrichreimer/github-changelog-generator-action@v2.3 + # with: + # token: ${{ secrets.GH_PAT }} + # futureRelease: ${{ env.NEXT_VERSION}} + # stripGeneratorNotice: true + # output: action/package/${{ inputs.changelog_file }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 71f0be260..40908fae0 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -286,15 +286,21 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV git checkout dev - - name: Generate Full Changelog - id: full_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 + - name: Prepend changelog + id: release_changelog + uses: orhun/git-cliff-action@v3 with: - token: ${{ secrets.GH_PAT }} - output: action/package/${{ inputs.changelog_file }} - futureRelease: ${{ env.NEXT_VERSION }} - stripGeneratorNotice: true - verbose: true + config: ${{ github.workspace }}/action/github/cliff.toml + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --tag ${{ env.NEXT_VERSION }} + # - name: Generate Full Changelog + # id: full_changelog + # uses: heinrichreimer/github-changelog-generator-action@v2.3 + # with: + # token: ${{ secrets.GH_PAT }} + # output: action/package/${{ inputs.changelog_file }} + # futureRelease: ${{ env.NEXT_VERSION }} + # stripGeneratorNotice: true + # verbose: true - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -342,14 +348,14 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git push --force origin temp-branch:testing - - name: Generate Release Changelog (cropped) - id: release_changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - futureRelease: ${{ env.NEXT_VERSION }} - sinceTag: ${{ env.VERSION }} - stripGeneratorNotice: true + # - name: Generate Release Changelog (cropped) + # id: release_changelog + # uses: heinrichreimer/github-changelog-generator-action@v2.3 + # with: + # token: ${{ secrets.GH_PAT }} + # futureRelease: ${{ env.NEXT_VERSION }} + # sinceTag: ${{ env.VERSION }} + # stripGeneratorNotice: true - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} From f83377d63b454a2d2e5f477681b15fd056752ded Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 14:59:33 +0100 Subject: [PATCH 301/357] tests --- .github/workflows/release_alpha.yml | 4 +++- .github/workflows/release_semver_start.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index d259e4e2d..2856b43b0 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,9 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --tag ${{ env.NEXT_VERSION }} + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --t ${{ env.NEXT_VERSION }} --w ${{ github.workspace }}/action/package/ + env: + OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) # id: release_changelog # uses: heinrichreimer/github-changelog-generator-action@v2.3 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 40908fae0..009c94601 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,7 +291,9 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --tag ${{ env.NEXT_VERSION }} + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --t ${{ env.NEXT_VERSION }} --w ${{ github.workspace }}/action/package/ + env: + OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog # id: full_changelog # uses: heinrichreimer/github-changelog-generator-action@v2.3 From 5cd569cef1de4a26821e3f56f63c60d3c96fde5b Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 15:10:15 +0100 Subject: [PATCH 302/357] tests --- scripts/check_PRs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index bf026d8e0..8974551ae 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -6,6 +6,12 @@ from github import Github, PullRequest import pccc + +CONFIG_FILE = os.environ.get("PCCC_CONFIG_FILE") +if CONFIG_FILE and not isfile(CONFIG_FILE): + print(f"Config file {CONFIG_FILE} not found.") + exit(1) + TOKEN = os.getenv('GH_PAT') or os.getenv('GITHUB_TOKEN') REPOSITORY = os.getenv('GITHUB_REPOSITORY') PR_LABELS: dict = json.loads(os.getenv('PR_LABELS', '{}')) @@ -47,7 +53,7 @@ def cc_scope(desc: str) -> str: def parse_cc(desc: str) -> Optional[pccc.ConventionalCommitRunner]: ccr = pccc.ConventionalCommitRunner() - ccr.options.load() + ccr.options.load((f"-o{CONFIG_FILE}",) if CONFIG_FILE else None) ccr.raw = desc ccr.clean() try: From af0ed6b24c2a68b5dbd34ca56038fa97d481ac33 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 15:19:34 +0100 Subject: [PATCH 303/357] tests --- pccc.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pccc.toml b/pccc.toml index 7ecfb08da..ac8a405ba 100644 --- a/pccc.toml +++ b/pccc.toml @@ -43,7 +43,7 @@ scopes = [ "core", "audio", "tts", - "sst", + "stt", "nlp", "plugins", "localisation", From 49f37e5e85034013d0b5d4177d6f2fcd147c71f5 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 15:36:05 +0100 Subject: [PATCH 304/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 2856b43b0..911b21e36 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --t ${{ env.NEXT_VERSION }} --w ${{ github.workspace }}/action/package/ + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w ${{ github.workspace }}/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 009c94601..cfcdc028f 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,7 +291,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} --t ${{ env.NEXT_VERSION }} --w ${{ github.workspace }}/action/package/ + args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w ${{ github.workspace }}/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog From bc86c78a94cd2dbf0689693aaf1f5bd34d60b112 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 16:06:41 +0100 Subject: [PATCH 305/357] tests --- .github/workflows/release_alpha.yml | 4 ++-- .github/workflows/release_semver_start.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 911b21e36..54ff8b1c5 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -79,8 +79,8 @@ jobs: id: release_changelog uses: orhun/git-cliff-action@v3 with: - config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w ${{ github.workspace }}/action/package/ + config: /github/workspace/action/github/cliff.toml + args: --unreleased --prepend --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index cfcdc028f..f038646f7 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -290,8 +290,8 @@ jobs: id: release_changelog uses: orhun/git-cliff-action@v3 with: - config: ${{ github.workspace }}/action/github/cliff.toml - args: --unreleased --prepend ${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w ${{ github.workspace }}/action/package/ + config: /github/workspace/action/github/cliff.toml + args: --unreleased --prepend --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog From f464ee13b49e30d8ab1e2ee560a12c80a20793c4 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 16:15:24 +0100 Subject: [PATCH 306/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 54ff8b1c5..f8e0ee8dd 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: /github/workspace/action/github/cliff.toml - args: --unreleased --prepend --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ + args: --unreleased --prepend /github/workspace/action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index f038646f7..3ab87edc2 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,7 +291,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: /github/workspace/action/github/cliff.toml - args: --unreleased --prepend --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ + args: --unreleased --prepend /github/workspace/action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ env: OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog From ec460b4c2093d1e036ce4b9d5cbd666878384167 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 16:29:56 +0100 Subject: [PATCH 307/357] tests --- .github/workflows/release_alpha.yml | 6 +++--- .github/workflows/release_semver_start.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index f8e0ee8dd..7487f3b49 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -79,10 +79,10 @@ jobs: id: release_changelog uses: orhun/git-cliff-action@v3 with: - config: /github/workspace/action/github/cliff.toml - args: --unreleased --prepend /github/workspace/action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ + config: action/github/cliff.toml + args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} env: - OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} + OUTPUT: action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) # id: release_changelog # uses: heinrichreimer/github-changelog-generator-action@v2.3 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 3ab87edc2..dfe057a7f 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -290,10 +290,10 @@ jobs: id: release_changelog uses: orhun/git-cliff-action@v3 with: - config: /github/workspace/action/github/cliff.toml - args: --unreleased --prepend /github/workspace/action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w /github/workspace/action/package/ + config: action/github/cliff.toml + args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} env: - OUTPUT: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} + OUTPUT: action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog # id: full_changelog # uses: heinrichreimer/github-changelog-generator-action@v2.3 From 11205e37dc7e5983b01c0a203808c6fe1d4a04bc Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 18 Mar 2024 16:38:57 +0100 Subject: [PATCH 308/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 7487f3b49..eca7ec45e 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: action/github/cliff.toml - args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} + args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} --include-path action/package/** env: OUTPUT: action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index dfe057a7f..75c13cb55 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -291,7 +291,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: action/github/cliff.toml - args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} + args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} --include-path action/package/** env: OUTPUT: action/package/${{ inputs.changelog_file }} # - name: Generate Full Changelog From b6503208d8b42242e3c268019c7eb0176252060c Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 19 Mar 2024 00:44:58 +0100 Subject: [PATCH 309/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 43 ++++++------- cliff.toml | 2 +- requirements/pip_base.txt | 1 + scripts/changelog_postprocess.py | 72 ++++++++++++++++++++++ 5 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 scripts/changelog_postprocess.py diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index eca7ec45e..a85a91c7f 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -80,7 +80,7 @@ jobs: uses: orhun/git-cliff-action@v3 with: config: action/github/cliff.toml - args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} --include-path action/package/** + args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w action/package/ env: OUTPUT: action/package/${{ inputs.changelog_file }} # - name: Generate Release Changelog (cropped) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 75c13cb55..3a777838e 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -286,23 +286,19 @@ jobs: echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV git checkout dev - - name: Prepend changelog - id: release_changelog - uses: orhun/git-cliff-action@v3 - with: - config: action/github/cliff.toml - args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} --include-path action/package/** + - name: Create Changelog + run: | + cd ${{ github.workspace }}/action/package/ + export CLIFF_CHANGELOG_CONTEXT=$( git-cliff --unreleased --context) + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py env: - OUTPUT: action/package/${{ inputs.changelog_file }} - # - name: Generate Full Changelog - # id: full_changelog - # uses: heinrichreimer/github-changelog-generator-action@v2.3 - # with: - # token: ${{ secrets.GH_PAT }} - # output: action/package/${{ inputs.changelog_file }} - # futureRelease: ${{ env.NEXT_VERSION }} - # stripGeneratorNotice: true - # verbose: true + GIT_CLIFF_CHANGELOG_ITEMS: "unreleased" + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} + GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -350,14 +346,13 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ git push --force origin temp-branch:testing - # - name: Generate Release Changelog (cropped) - # id: release_changelog - # uses: heinrichreimer/github-changelog-generator-action@v2.3 - # with: - # token: ${{ secrets.GH_PAT }} - # futureRelease: ${{ env.NEXT_VERSION }} - # sinceTag: ${{ env.VERSION }} - # stripGeneratorNotice: true + - name: Create Changelog + id: release_changelog + run: | + cd ${{ github.workspace }}/action/package/ + unset GIT_CLIFF_PREPEND + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py) + echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} diff --git a/cliff.toml b/cliff.toml index da89bd553..8469d304e 100644 --- a/cliff.toml +++ b/cliff.toml @@ -37,7 +37,7 @@ footer = """ trim = true # postprocessors postprocessors = [ - { pattern = '', replace_command = 'sed "s##$GITHUB_REPOSITORY#"' }, + { pattern = '', replace_command = 'sed "s##$GITHUB_REPO#"' }, ] [git] diff --git a/requirements/pip_base.txt b/requirements/pip_base.txt index d864019bc..5fee76d95 100644 --- a/requirements/pip_base.txt +++ b/requirements/pip_base.txt @@ -1,3 +1,4 @@ PyGithub semver pccc +git-cliff diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py new file mode 100644 index 000000000..70182d1c0 --- /dev/null +++ b/scripts/changelog_postprocess.py @@ -0,0 +1,72 @@ +from os import environ +from os.path import join, dirname +import subprocess +import json +import re + +#CONTEXT = environ.get("CLIFF_CHANGELOG_CONTEXT") +CONTEXT = '[{"version":"0.3.40a3","commits":[{"id":"e08c27d1c5dd61ddcd4fc28c50b1c756b68dd691","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710769697},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710769697},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"a99aab5ab0aac932035d784a204bf7c9fc58fa6a","message":"bump ([#198](https://github.com//pull/198))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710769744},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710769744},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"6d9e906d98700a3cd85483ed062a1a3bebb212e7","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710770502},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710770502},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"eda66264a83ef35d4df23f10026cc0257187ece5","message":"bump ([#199](https://github.com//pull/199))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710770515},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710770515},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"def1f9ee70493cdc9cf929981c95e4dbcab3249b","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710771913},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710771913},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"3d884ec8542bd005364e39ec748831e9186f3c0b","message":"bump ([#200](https://github.com//pull/200))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710771970},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710771970},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"2cfad089d4bee39fef7f9d324ffdb7cb4642f583","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"nlp","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710772746},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710772746},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"5ad5da3faae949761edb42c31d189dc7dfe816bb","message":"bump ([#201](https://github.com//pull/201))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"nlp","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710772790},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710772790},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"fa626505a36ef89e74582b370df82c9c2d3dbaf2","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774510},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774510},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"55147702f7980338cd976d67a2b39e5d07673943","message":"bump ([#202](https://github.com//pull/202))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710774531},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710774531},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"5300b035186ff3c9fdd9d668d0334c79074d33e5","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774960},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774960},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"6fb2ae48ca55dcf2b39733af612be8d12092447d","message":"bump ([#203](https://github.com//pull/203))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710774986},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710774986},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"602c9e970a3f01887e6856bc1b4d1797ebda3cc2","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":null,"links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710775850},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710775850},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"2fadd7b14552ced76f9065f94af4eb46085fa0b1","message":"bump ([#204](https://github.com//pull/204))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":null,"links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710775870},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710775870},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"e9a225a5e9a70c58daa4ff7f3ec2e7e29c7219ed","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776421},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776421},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"a47647e286cbd76cf8670ffb7e6fde5b152e0e05","message":"bump ([#205](https://github.com//pull/205))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710776448},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710776448},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"1b04cc8a3c8feba12f0c06dee23beeed826ebb03","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776662},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776662},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"3cc132a8bf01d571e430ab85a01875ada9cb523c","message":"bump ([#206](https://github.com//pull/206))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710776695},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710776695},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}}],"commit_id":"3cc132a8bf01d571e430ab85a01875ada9cb523c","timestamp":1710791950,"previous":{"version":"0.3.40a2","commits":[],"commit_id":"2fcf56b470755c63d417c3c905fac274bdfe4203","timestamp":1710714924,"previous":null,"github":{"contributors":[]}},"github":{"contributors":[]}}]' +PULL_LINK_PATTERN = r' \(\[#\d+\]\(https://github\.com/.+?/pull/\d+\)\)' +CLIFF_IGNORE_FILE = join(environ.get("REPO_BASEDIR", ""), ".cliffignore") +CHANGELOG_ITEMS = environ.get("GIT_CLIFF_CHANGELOG_ITEMS", "full") + + +def escape_control_characters(s): + return re.sub(r'[\x00-\x1f\x7f-\x9f]', lambda c: "\\u{0:04x}".format(ord(c.group())), s) + +def strip_pull_request_links(text): + return re.sub(PULL_LINK_PATTERN, '', text).strip() + +def in_git_repo(file_path): + try: + subprocess.check_output(['git', '-C', dirname(file_path), 'rev-parse']) + return True + except subprocess.CalledProcessError: + return False + +if CONTEXT is None: + print("You need to provide a changlog context (json)") + exit(1) +if not in_git_repo(CLIFF_IGNORE_FILE): + print("you have to run this script in a git repository or provide a proper `REPO_BASEDIR` environment variable.") + exit(1) +else: + # empty the file + with open(CLIFF_IGNORE_FILE, 'w') as f: + f.write("") +if not CHANGELOG_ITEMS in ["full", "latest", "current", "unreleased"]: + raise ValueError("GIT_CLIFF_CHANGELOG_ITEMS must be one of 'latest', 'current', 'unreleased'") + +escaped_json_string = escape_control_characters(CONTEXT) +changelog_context = json.loads(escaped_json_string) +last_message = None + +# write the commits to be ignored in the postprocessing step +for entry in changelog_context: + message = entry['message'] + if not(last_message and re.match(PULL_LINK_PATTERN)): + continue + + # the merge_commit flag is not safe to use by itself + # double check + stripped_message = strip_pull_request_links(message) + if message.get('merge_commit', False) or stripped_message == last_message: + with open(CLIFF_IGNORE_FILE, 'a') as f: + f.write(f"{message["id"]}\n") + + + + + + + + + + + + + + + + + From 83b95a7e2d76b5414d92605e70e3dfce4b589e60 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 19 Mar 2024 00:55:22 +0100 Subject: [PATCH 310/357] tests --- scripts/changelog_postprocess.py | 88 +++++++++++++++++++------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 70182d1c0..ac66b2785 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -1,12 +1,12 @@ from os import environ from os.path import join, dirname +import sys import subprocess import json import re -#CONTEXT = environ.get("CLIFF_CHANGELOG_CONTEXT") -CONTEXT = '[{"version":"0.3.40a3","commits":[{"id":"e08c27d1c5dd61ddcd4fc28c50b1c756b68dd691","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710769697},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710769697},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"a99aab5ab0aac932035d784a204bf7c9fc58fa6a","message":"bump ([#198](https://github.com//pull/198))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710769744},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710769744},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"6d9e906d98700a3cd85483ed062a1a3bebb212e7","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710770502},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710770502},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"eda66264a83ef35d4df23f10026cc0257187ece5","message":"bump ([#199](https://github.com//pull/199))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710770515},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710770515},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"def1f9ee70493cdc9cf929981c95e4dbcab3249b","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710771913},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710771913},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"3d884ec8542bd005364e39ec748831e9186f3c0b","message":"bump ([#200](https://github.com//pull/200))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710771970},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710771970},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"2cfad089d4bee39fef7f9d324ffdb7cb4642f583","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"nlp","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710772746},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710772746},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"5ad5da3faae949761edb42c31d189dc7dfe816bb","message":"bump ([#201](https://github.com//pull/201))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"nlp","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710772790},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710772790},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"fa626505a36ef89e74582b370df82c9c2d3dbaf2","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774510},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774510},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"55147702f7980338cd976d67a2b39e5d07673943","message":"bump ([#202](https://github.com//pull/202))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710774531},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710774531},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"5300b035186ff3c9fdd9d668d0334c79074d33e5","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774960},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710774960},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"6fb2ae48ca55dcf2b39733af612be8d12092447d","message":"bump ([#203](https://github.com//pull/203))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710774986},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710774986},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"602c9e970a3f01887e6856bc1b4d1797ebda3cc2","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":null,"links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710775850},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710775850},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"2fadd7b14552ced76f9065f94af4eb46085fa0b1","message":"bump ([#204](https://github.com//pull/204))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":null,"links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710775870},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710775870},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"e9a225a5e9a70c58daa4ff7f3ec2e7e29c7219ed","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776421},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776421},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"a47647e286cbd76cf8670ffb7e6fde5b152e0e05","message":"bump ([#205](https://github.com//pull/205))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"tts","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710776448},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710776448},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"1b04cc8a3c8feba12f0c06dee23beeed826ebb03","message":"bump","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776662},"committer":{"name":"emphasize","email":"swen_g@t-online.de","timestamp":1710776662},"conventional":true,"merge_commit":false,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}},{"id":"3cc132a8bf01d571e430ab85a01875ada9cb523c","message":"bump ([#206](https://github.com//pull/206))","body":null,"footers":[],"group":"🐛 Bug Fixes","breaking_description":null,"breaking":false,"scope":"stt","links":[],"author":{"name":"Swen Gross","email":"25036977+emphasize@users.noreply.github.com","timestamp":1710776695},"committer":{"name":"GitHub","email":"noreply@github.com","timestamp":1710776695},"conventional":true,"merge_commit":true,"github":{"username":null,"pr_title":null,"pr_number":null,"pr_labels":[],"is_first_time":false}}],"commit_id":"3cc132a8bf01d571e430ab85a01875ada9cb523c","timestamp":1710791950,"previous":{"version":"0.3.40a2","commits":[],"commit_id":"2fcf56b470755c63d417c3c905fac274bdfe4203","timestamp":1710714924,"previous":null,"github":{"contributors":[]}},"github":{"contributors":[]}}]' -PULL_LINK_PATTERN = r' \(\[#\d+\]\(https://github\.com/.+?/pull/\d+\)\)' +CONTEXT = environ.get("CLIFF_CHANGELOG_CONTEXT") +PULL_LINK_PATTERN = r' \(\[#\d+\]\(https:\/\/github\.com\/.+?\/pull\/\d+\)\)' CLIFF_IGNORE_FILE = join(environ.get("REPO_BASEDIR", ""), ".cliffignore") CHANGELOG_ITEMS = environ.get("GIT_CLIFF_CHANGELOG_ITEMS", "full") @@ -23,13 +23,18 @@ def in_git_repo(file_path): return True except subprocess.CalledProcessError: return False + +def valid_json(s): + try: + json.loads(escape_control_characters(s)) + return True + except json.JSONDecodeError: + return False -if CONTEXT is None: - print("You need to provide a changlog context (json)") - exit(1) +if not valid_json(CONTEXT): + raise Exception("You need to provide a valid changelog context (json)") if not in_git_repo(CLIFF_IGNORE_FILE): - print("you have to run this script in a git repository or provide a proper `REPO_BASEDIR` environment variable.") - exit(1) + raise Exception("You have to run this script in a git repository or provide a proper `REPO_BASEDIR` environment variable.") else: # empty the file with open(CLIFF_IGNORE_FILE, 'w') as f: @@ -37,36 +42,47 @@ def in_git_repo(file_path): if not CHANGELOG_ITEMS in ["full", "latest", "current", "unreleased"]: raise ValueError("GIT_CLIFF_CHANGELOG_ITEMS must be one of 'latest', 'current', 'unreleased'") + escaped_json_string = escape_control_characters(CONTEXT) changelog_context = json.loads(escaped_json_string) last_message = None -# write the commits to be ignored in the postprocessing step for entry in changelog_context: - message = entry['message'] - if not(last_message and re.match(PULL_LINK_PATTERN)): - continue - - # the merge_commit flag is not safe to use by itself - # double check - stripped_message = strip_pull_request_links(message) - if message.get('merge_commit', False) or stripped_message == last_message: - with open(CLIFF_IGNORE_FILE, 'a') as f: - f.write(f"{message["id"]}\n") - - - - - - - - - - - - - - - - - + last_commit = None + for commit in entry.get('commits', []): + + message = commit['message'] + if not (last_commit and re.search(PULL_LINK_PATTERN, message)): + last_commit = commit + continue + + stripped_message = strip_pull_request_links(message) + if stripped_message == last_commit['message'] and \ + commit.get('scope') == last_commit.get('scope'): + # add to ignored commits (as the merge commit will be part of the changelog) + with open(CLIFF_IGNORE_FILE, 'a') as f: + f.write(f"{last_commit['id']}\n") + last_message = message + +if not open(CLIFF_IGNORE_FILE, 'r').read(): + print("No commits to ignore. No need to postprocess.") + exit(0) + +command = ["git", "cliff"] + +if CHANGELOG_ITEMS == "unreleased": + command.append("--unreleased") +elif CHANGELOG_ITEMS == "latest": + command.append("--latest") +elif CHANGELOG_ITEMS == "current": + command.append("--current") + + +process = subprocess.Popen(command, env=environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +# interact with the subprocess's standard output and error streams +stdout, stderr = process.communicate() + +# print the subprocess's standard output and error streams +print(stdout.decode()) +print(stderr.decode(), file=sys.stderr) From 84576d0d048178d9b335f794e5e1c2f579813bdb Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 19 Mar 2024 00:58:24 +0100 Subject: [PATCH 311/357] tests --- scripts/changelog_postprocess.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index ac66b2785..4ac0ba9e0 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -45,7 +45,6 @@ def valid_json(s): escaped_json_string = escape_control_characters(CONTEXT) changelog_context = json.loads(escaped_json_string) -last_message = None for entry in changelog_context: last_commit = None @@ -62,7 +61,7 @@ def valid_json(s): # add to ignored commits (as the merge commit will be part of the changelog) with open(CLIFF_IGNORE_FILE, 'a') as f: f.write(f"{last_commit['id']}\n") - last_message = message + last_commit = commit if not open(CLIFF_IGNORE_FILE, 'r').read(): print("No commits to ignore. No need to postprocess.") From 48a44072c28bf289a4fe5f603305f3fba6e9ba2f Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 14:18:29 +0100 Subject: [PATCH 312/357] tests --- .github/workflows/release_semver_start.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 3a777838e..c8ea9d533 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -311,7 +311,9 @@ jobs: id: get_commit_sha run: | cd ${{ github.workspace }}/action/package/ - git fetch --unshallow + if [ -f .git/shallow ]; then + git fetch --unshallow + fi git checkout dev COMMITS=2 if [ "${{ needs.autotranslate.outputs.translated }}" == "1" ]; then From 855792d3b42e620aefedcf417f79075e74f51096 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 18:03:54 +0100 Subject: [PATCH 313/357] tests --- .github/workflows/release_semver_start.yml | 6 +- scripts/changelog_postprocess.py | 80 +++++++++++++++------- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c8ea9d533..d1d1b0281 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -289,10 +289,8 @@ jobs: - name: Create Changelog run: | cd ${{ github.workspace }}/action/package/ - export CLIFF_CHANGELOG_CONTEXT=$( git-cliff --unreleased --context) - python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased env: - GIT_CLIFF_CHANGELOG_ITEMS: "unreleased" GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} @@ -353,7 +351,7 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ unset GIT_CLIFF_PREPEND - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py) + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT - name: (Re)Set and push Release Version run: | diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 4ac0ba9e0..8cf2b604f 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -1,22 +1,33 @@ from os import environ -from os.path import join, dirname +from os.path import join, dirname, isfile import sys import subprocess import json import re +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument("--context", "-c", help="Path to the changelog context file") +parser.add_argument("--items", "-i", choices=["latest", "current", "unreleased", "first unreleased"], default="full") +args = parser.parse_args() + -CONTEXT = environ.get("CLIFF_CHANGELOG_CONTEXT") PULL_LINK_PATTERN = r' \(\[#\d+\]\(https:\/\/github\.com\/.+?\/pull\/\d+\)\)' CLIFF_IGNORE_FILE = join(environ.get("REPO_BASEDIR", ""), ".cliffignore") -CHANGELOG_ITEMS = environ.get("GIT_CLIFF_CHANGELOG_ITEMS", "full") +GIT_CLIFF_OUTPUT = environ.get("GIT_CLIFF_OUTPUT") +if GIT_CLIFF_OUTPUT: + del environ["GIT_CLIFF_OUTPUT"] def escape_control_characters(s): return re.sub(r'[\x00-\x1f\x7f-\x9f]', lambda c: "\\u{0:04x}".format(ord(c.group())), s) + def strip_pull_request_links(text): return re.sub(PULL_LINK_PATTERN, '', text).strip() + def in_git_repo(file_path): try: subprocess.check_output(['git', '-C', dirname(file_path), 'rev-parse']) @@ -24,13 +35,53 @@ def in_git_repo(file_path): except subprocess.CalledProcessError: return False + def valid_json(s): try: json.loads(escape_control_characters(s)) return True except json.JSONDecodeError: return False - + + +def run_cliff(get_context = False): + command = ["git", "cliff"] + mute = False + + if args.items == "unreleased": + command.append("--unreleased") + elif args.items == "latest": + command.append("--latest") + elif args.items == "current": + command.append("--current") + + if get_context: + command.append("--context") + mute = True + elif GIT_CLIFF_OUTPUT: + command.append("--output") + command.append(GIT_CLIFF_OUTPUT) + + print(f"Running cliff with command: {' '.join(command)}") + process = subprocess.Popen(command, env=environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # interact with the subprocess's standard output and error streams + stdout, stderr = process.communicate() + + # print the subprocess's standard output and error streams + if not mute: + print(stdout.decode()) + print(stderr.decode(), file=sys.stderr) + + return stdout.decode().strip() + +if not args.context or not isfile(args.context): + print(f"No context file passed, creating context with {args.items} items.") + CONTEXT = run_cliff(get_context=True) +else: + with open(args.context, 'r') as f: + CONTEXT = f.read() + if not valid_json(CONTEXT): raise Exception("You need to provide a valid changelog context (json)") if not in_git_repo(CLIFF_IGNORE_FILE): @@ -39,9 +90,6 @@ def valid_json(s): # empty the file with open(CLIFF_IGNORE_FILE, 'w') as f: f.write("") -if not CHANGELOG_ITEMS in ["full", "latest", "current", "unreleased"]: - raise ValueError("GIT_CLIFF_CHANGELOG_ITEMS must be one of 'latest', 'current', 'unreleased'") - escaped_json_string = escape_control_characters(CONTEXT) changelog_context = json.loads(escaped_json_string) @@ -67,21 +115,5 @@ def valid_json(s): print("No commits to ignore. No need to postprocess.") exit(0) -command = ["git", "cliff"] - -if CHANGELOG_ITEMS == "unreleased": - command.append("--unreleased") -elif CHANGELOG_ITEMS == "latest": - command.append("--latest") -elif CHANGELOG_ITEMS == "current": - command.append("--current") - - -process = subprocess.Popen(command, env=environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - -# interact with the subprocess's standard output and error streams -stdout, stderr = process.communicate() -# print the subprocess's standard output and error streams -print(stdout.decode()) -print(stderr.decode(), file=sys.stderr) +run_cliff() From 7906d0a8b11b94bd8b0104231e465378b646e145 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 18:16:07 +0100 Subject: [PATCH 314/357] tests --- .github/workflows/release_alpha.yml | 40 ++++++++++++----------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index a85a91c7f..a539b327a 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -75,35 +75,29 @@ jobs: echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: change working directory run: cd ${{ github.workspace }}/action/package/ - - name: Prepend changelog - id: release_changelog - uses: orhun/git-cliff-action@v3 - with: - config: action/github/cliff.toml - args: --unreleased --prepend action/package/${{ inputs.changelog_file }} --github-repo ${{ github.repository }} --github-token ${{ secrets.GH_PAT }} -t ${{ env.NEXT_VERSION }} -w action/package/ + - name: Create Changelog + run: | + cd ${{ github.workspace }}/action/package/ + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased env: - OUTPUT: action/package/${{ inputs.changelog_file }} - # - name: Generate Release Changelog (cropped) - # id: release_changelog - # uses: heinrichreimer/github-changelog-generator-action@v2.3 - # with: - # token: ${{ secrets.GH_PAT }} - # futureRelease: ${{ env.NEXT_VERSION }} - # sinceTag: ${{ env.VERSION }} - # stripGeneratorNotice: true - # - name: Generate Full Changelog - # id: full_changelog - # uses: heinrichreimer/github-changelog-generator-action@v2.3 - # with: - # token: ${{ secrets.GH_PAT }} - # futureRelease: ${{ env.NEXT_VERSION}} - # stripGeneratorNotice: true - # output: action/package/${{ inputs.changelog_file }} + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} + GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} repository: action/package/ + - name: Create Release Changelog + id: release_changelog + run: | + cd ${{ github.workspace }}/action/package/ + unset GIT_CLIFF_PREPEND + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) + echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 From a402bba5aa90edefa42db3369b56ccbc54f594b2 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 18:49:53 +0100 Subject: [PATCH 315/357] tests --- scripts/changelog_postprocess.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 8cf2b604f..233b10edf 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -117,3 +117,6 @@ def run_cliff(get_context = False): run_cliff() + +# delete the ignore file +subprocess.run(["rm", CLIFF_IGNORE_FILE]) From 028efdefa721930884d778cec73d4d5c7c482878 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 18:55:11 +0100 Subject: [PATCH 316/357] tests --- scripts/changelog_postprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 233b10edf..437bf6b45 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -62,7 +62,6 @@ def run_cliff(get_context = False): command.append("--output") command.append(GIT_CLIFF_OUTPUT) - print(f"Running cliff with command: {' '.join(command)}") process = subprocess.Popen(command, env=environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # interact with the subprocess's standard output and error streams From fa266e8d2bdc3419aa8817bab0bfc06d9d933eb0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 18:58:22 +0100 Subject: [PATCH 317/357] tests --- scripts/changelog_postprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 437bf6b45..d6c6cb037 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -111,7 +111,6 @@ def run_cliff(get_context = False): last_commit = commit if not open(CLIFF_IGNORE_FILE, 'r').read(): - print("No commits to ignore. No need to postprocess.") exit(0) From 89a28015ccc178a02f39d162eabf6afc766f6d46 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 19:02:44 +0100 Subject: [PATCH 318/357] tests --- scripts/changelog_postprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index d6c6cb037..f7cb4c710 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -75,7 +75,6 @@ def run_cliff(get_context = False): return stdout.decode().strip() if not args.context or not isfile(args.context): - print(f"No context file passed, creating context with {args.items} items.") CONTEXT = run_cliff(get_context=True) else: with open(args.context, 'r') as f: From dd0de491340c2f669ce15f085a9e6f82a7d19a3d Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 19:08:17 +0100 Subject: [PATCH 319/357] tests --- scripts/changelog_postprocess.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index f7cb4c710..29f0a5b15 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -109,9 +109,6 @@ def run_cliff(get_context = False): f.write(f"{last_commit['id']}\n") last_commit = commit -if not open(CLIFF_IGNORE_FILE, 'r').read(): - exit(0) - run_cliff() From 28a279edfb605c6eacbeb2bc3505e43bce95d856 Mon Sep 17 00:00:00 2001 From: emphasize Date: Sat, 23 Mar 2024 19:53:36 +0100 Subject: [PATCH 320/357] tests --- .github/actions/notify_testing_matrix/action.yml | 3 --- .github/workflows/release_alpha.yml | 13 ++++++------- .github/workflows/release_semver_start.yml | 13 ++++++------- scripts/changelog_postprocess.py | 6 ++++++ 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index 7e872c11b..d739a38b6 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -4,9 +4,6 @@ inputs: matrix_token: description: 'The token to use to authenticate with the Matrix server' required: true - # kickoff_pr: - # description: 'Kick off pull request' - # default: 'false' subject: description: 'Subject of the message' default: 'no subject' diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index a539b327a..b6b00b896 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -33,8 +33,10 @@ jobs: env: VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} SOURCE_BRANCH: ${{ inputs.release_type == 'alpha' && 'dev' || 'testing' }} - GITHUB_REPOSITORY: ${{ github.repository }} - GH_PAT: ${{ secrets.GH_PAT }} + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} steps: - name: Checkout Scripts Repo uses: actions/checkout@v4 @@ -80,12 +82,8 @@ jobs: cd ${{ github.workspace }}/action/package/ python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased env: - GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml - GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - GITHUB_REPO: ${{ github.repository }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -95,9 +93,10 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - unset GIT_CLIFF_PREPEND CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + env: + GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - name: Create Pre-release id: create_release uses: ncipollo/release-action@v1 diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index d1d1b0281..c3bba1d8d 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -248,9 +248,11 @@ jobs: needs: [parse_conventional_commits, autotranslate] if: ${{ needs.parse_conventional_commits.outputs.release_type == 'release' }} env: - GITHUB_REPOSITORY: ${{ github.repository }} - GH_PAT: ${{ secrets.GH_PAT }} VERSION_FILE: ${{ github.workspace}}/action/package/${{ inputs.version_file }} + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -291,12 +293,8 @@ jobs: cd ${{ github.workspace }}/action/package/ python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased env: - GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml - GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - GITHUB_REPO: ${{ github.repository }} - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -350,9 +348,10 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - unset GIT_CLIFF_PREPEND CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + env: + GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - name: (Re)Set and push Release Version run: | python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version ${{ env.TEST_VERSION }} --save ${{ env.VERSION_FILE }} diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 29f0a5b15..4af9ffce7 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -18,6 +18,9 @@ GIT_CLIFF_OUTPUT = environ.get("GIT_CLIFF_OUTPUT") if GIT_CLIFF_OUTPUT: del environ["GIT_CLIFF_OUTPUT"] +GIT_CLIFF_PREPEND = environ.get("GIT_CLIFF_PREPEND") +if GIT_CLIFF_PREPEND: + del environ["GIT_CLIFF_PREPEND"] def escape_control_characters(s): @@ -61,6 +64,9 @@ def run_cliff(get_context = False): elif GIT_CLIFF_OUTPUT: command.append("--output") command.append(GIT_CLIFF_OUTPUT) + elif GIT_CLIFF_PREPEND: + command.append("--prepend") + command.append(GIT_CLIFF_PREPEND) process = subprocess.Popen(command, env=environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From d9c00841febb03f17d4d92c07160ac6cb7b28cfd Mon Sep 17 00:00:00 2001 From: emphasize Date: Sun, 24 Mar 2024 20:53:19 +0100 Subject: [PATCH 321/357] tests --- .github/workflows/release_alpha.yml | 8 ++-- .github/workflows/release_semver_publish.yml | 42 +++++++++++-------- .../workflows/release_semver_pull_master.yml | 34 ++++++++------- .github/workflows/release_semver_start.yml | 12 +++--- cliff.toml | 4 +- scripts/changelog_postprocess.py | 23 +++++++++- scripts/parse_semver_release.py | 2 +- 7 files changed, 79 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index b6b00b896..208e5042c 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -67,13 +67,15 @@ jobs: id: version run: | cd ${{ github.workspace }}/action/github/scripts/ - VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }} --latest) + PREVIOUS_VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }} --latest) if [ "${{ inputs.release_type }}" != 'alpha' ]; then NEXT_VERSION=$( python semver_release_version.py --next --type ${{ inputs.release_type }} --file ${{ env.VERSION_FILE }} --bump-prerelease --save) else NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) fi - echo "VERSION=${VERSION}" >> $GITHUB_ENV + RELEASE_CYCLE_START=$( python semver_release_version.py cycle --version "$VERSION" --latest --type patch) + echo "RELEASE_CYCLE_START=${RELEASE_CYCLE_START}" >> $GITHUB_ENV + echo "PREVIOUS_VERSION=${PREVIOUS_VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - name: change working directory run: cd ${{ github.workspace }}/action/package/ @@ -84,7 +86,7 @@ jobs: env: GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + - name: Commit Prerelease Changes (${{ env.PREVIOUS_VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 98791cd29..bfda89b82 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -29,8 +29,11 @@ on: jobs: build_and_publish: env: - GITHUB_REPOSITORY: ${{ github.repository }} - GH_PAT: ${{ secrets.GH_PAT }} + PCCC_CONFIG_FILE: ${{ github.workspace }}/action/github/pccc.toml + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} runs-on: ubuntu-latest steps: - name: set up release type @@ -57,31 +60,34 @@ jobs: run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt pip install -r ${{ github.workspace }}/action/github/requirements/pip_build_tools.txt - - name: Get Version + - name: Get Release Version run: | - VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Check Latest Release + RELEASE_VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV + - name: Parse Release Type run: | - SINCE_TAG=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest) - echo "SINCE_TAG=${SINCE_TAG}" >> $GITHUB_ENV - - name: create release changelog (cropped) - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.SINCE_TAG }} - stripGeneratorNotice: true + RELEASE_TYPE=$(python ${{ github.workspace }}/action/github/scripts/parse_semver_release.py) + echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV + RELEASE_CYCLE_START=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type "$RELEASE_TYPE" ) + echo "RELEASE_CYCLE_START=${RELEASE_CYCLE_START}" >> $GITHUB_ENV + - name: Create Release Changelog + id: release_changelog + run: | + cd ${{ github.workspace }}/action/package/ + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.RELEASE_CYCLE_START }} ) + echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + env: + GIT_CLIFF_TAG: ${{ env.RELEASE_VERSION }} - name: Create Release id: create_release uses: ncipollo/release-action@v1 with: token: ${{ secrets.GH_PAT }} - tag: ${{ env.VERSION }} - name: Release ${{ env.VERSION }} + tag: ${{ env.RELEASE_VERSION }} + name: Release ${{ env.RELEASE_VERSION }} body: | Changes in this Release - ${{ steps.changelog.outputs.changelog }} + ${{ steps.release_changelog.outputs.changelog }} - name: Build Distribution Packages run: | mkdir -p ${{ github.workspace }}/action/dist diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 6d924a7b6..8cb39e866 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -20,9 +20,11 @@ jobs: runs-on: ubuntu-latest env: TARGET_BRANCH: 'master' - GITHUB_REPOSITORY: ${{ github.repository }} PCCC_CONFIG_FILE: ${{ github.workspace }}/action/github/pccc.toml - GH_PAT: ${{ secrets.GH_PAT }} + GIT_CLIFF_CONFIG: ${{ github.workspace }}/action/github/cliff.toml + GIT_CLIFF_WORKDIR: ${{ github.workspace }}/action/package/ + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_REPO: ${{ github.repository }} steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -51,26 +53,26 @@ jobs: - name: Install Python Dependencies run: | pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: Get Version + - name: Get Release Version run: | - export VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV + RELEASE_VERSION=$(python ${{ github.workspace }}/action/package/setup.py --version) + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV - name: Parse Release Type run: | RELEASE_TYPE=$(python ${{ github.workspace }}/action/github/scripts/parse_semver_release.py) echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Check Latest Release run: | - LATEST_RELEASE=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type ${{ env.RELEASE_TYPE }}) + LATEST_RELEASE=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --latest --type ${{ env.RELEASE_TYPE }}) echo "LATEST_RELEASE=${LATEST_RELEASE}" >> $GITHUB_ENV - - name: create release changelog (cropped) - id: changelog - uses: heinrichreimer/github-changelog-generator-action@v2.3 - with: - token: ${{ secrets.GH_PAT }} - sinceTag: ${{ env.LATEST_RELEASE }} - stripGeneratorNotice: true - # if there is no master branch, create one from the tag of the latest release or first commit + - name: Create Release Changelog (Proposal) + id: release_changelog + run: | + cd ${{ github.workspace }}/action/package/ + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.LATEST_RELEASE }} ) + echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + env: + GIT_CLIFF_TAG: ${{ env.RELEASE_VERSION }} - name: Create Master Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ @@ -88,10 +90,10 @@ jobs: run: | cd ${{ github.workspace }}/action/package/ gh pr create \ - --title "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.VERSION }})" \ + --title "ci(release): declare ${{ env.RELEASE_TYPE }} release stable (${{ env.RELEASE_VERSION }})" \ --body "$(cat << EOF Included changes: - ${{ steps.changelog.outputs.changelog }} + ${{ steps.release_changelog.outputs.changelog }} EOF )" \ --base ${{ env.TARGET_BRANCH }} \ diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index c3bba1d8d..1da012d82 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -283,9 +283,11 @@ jobs: git checkout testing TEST_VERSION=$(python setup.py --version) echo "TEST_VERSION=${TEST_VERSION}" >> $GITHUB_ENV - VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) - NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$VERSION" --next --type alpha) - echo "VERSION=${VERSION}" >> $GITHUB_ENV + PREVIOUS_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) + NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$PREVIOUS_VERSION" --next --type alpha) + RELEASE_CYCLE_START=$( python ${{ github.workspace }}/action/github/scripts/semver_release_version.py cycle --version "$TEST_VERSION" --latest --type patch) + echo "RELEASE_CYCLE_START=${RELEASE_CYCLE_START}" >> $GITHUB_ENV + echo "PREVIOUS_VERSION=${PREVIOUS_VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV git checkout dev - name: Create Changelog @@ -295,7 +297,7 @@ jobs: env: GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - - name: Commit Prerelease Changes (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) + - name: Commit Prerelease Changes (${{ env.PREVIOUS_VERSION }} -> ${{ env.NEXT_VERSION }}) uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Prerelease Changes ${{ env.NEXT_VERSION }} @@ -327,7 +329,7 @@ jobs: cd ${{ github.workspace }}/action/package/ git config user.name "EggmanBot" git config user.email "openvoiceos@gmail.com" - git checkout -b temp-branch "tags/$VERSION" + git checkout -b temp-branch "tags/$PREVIOUS_VERSION" IFS=' ' read -ra ADDR <<< "${{ env.COMMIT_SHA }}" for i in "${ADDR[@]}"; do PARENTS_COUNT=$(git cat-file -p $i | grep -c parent) diff --git a/cliff.toml b/cliff.toml index 8469d304e..1db620d6d 100644 --- a/cliff.toml +++ b/cliff.toml @@ -16,7 +16,8 @@ All notable changes to this project will be documented in this file.\n body = """ {% if version %}\ ## [{{ version | trim_start_matches(pat="v") }}](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/tree/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} - ### [Full Changelog](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/compare/{{ previous.version }}...{{ version }}) + ### [Latest Changes](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/compare/{{ previous.version }}...{{ version }}) + ### [Release Cycle Changelog](https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/compare/...{{ version }}) {% else %}\ ## [unreleased] {% endif %}\ @@ -38,6 +39,7 @@ trim = true # postprocessors postprocessors = [ { pattern = '', replace_command = 'sed "s##$GITHUB_REPO#"' }, + { pattern = '', replace_command = 'if [ -z "$RELEASE_CYCLE_START" ]; then sed "//d"; else sed "s##$RELEASE_CYCLE_START#"; fi' }, ] [git] diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 4af9ffce7..0c6d206f7 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -9,10 +9,16 @@ parser = argparse.ArgumentParser() parser.add_argument("--context", "-c", help="Path to the changelog context file") -parser.add_argument("--items", "-i", choices=["latest", "current", "unreleased", "first unreleased"], default="full") -args = parser.parse_args() +release_group = parser.add_mutually_exclusive_group() +release_group.add_argument("--items", "-i", choices=["unreleased", "latest", "current"], help="Items to include in the changelog", default="full") +release_group.add_argument("--since", "-s", help="Include items since a specific version") + +args = parser.parse_args() +if args.since == "0.0.0": + args.since = None + PULL_LINK_PATTERN = r' \(\[#\d+\]\(https:\/\/github\.com\/.+?\/pull\/\d+\)\)' CLIFF_IGNORE_FILE = join(environ.get("REPO_BASEDIR", ""), ".cliffignore") GIT_CLIFF_OUTPUT = environ.get("GIT_CLIFF_OUTPUT") @@ -39,6 +45,14 @@ def in_git_repo(file_path): return False +def is_tag(tag): + try: + subprocess.check_output(['git', 'rev-parse', tag]) + return True + except subprocess.CalledProcessError: + return False + + def valid_json(s): try: json.loads(escape_control_characters(s)) @@ -57,6 +71,9 @@ def run_cliff(get_context = False): command.append("--latest") elif args.items == "current": command.append("--current") + elif args.since: + last_commit = subprocess.check_output(["git", "log", "-1", "--pretty=format:'%h'"]).decode().strip() + command.append(f"{args.since}..{last_commit}") if get_context: command.append("--context") @@ -90,6 +107,8 @@ def run_cliff(get_context = False): raise Exception("You need to provide a valid changelog context (json)") if not in_git_repo(CLIFF_IGNORE_FILE): raise Exception("You have to run this script in a git repository or provide a proper `REPO_BASEDIR` environment variable.") +elif args.since and not is_tag(args.since): + raise Exception(f"The tag provided {args.since} doesn't exist.") else: # empty the file with open(CLIFF_IGNORE_FILE, 'w') as f: diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index c0cbe452d..3f7ef0781 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -22,7 +22,7 @@ def get_version(): # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" - version = environ.get("VERSION", "") + version = environ.get("VERSION") or environ.get("PREVIOUS_VERSION") match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", version) if match: return f"{match.group(1)}-{match.group(2)}" From a5aaf2e8d20b0d1b0b946619efc3d10415d3d1e7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 25 Mar 2024 16:59:04 +0100 Subject: [PATCH 322/357] tests --- .github/workflows/release_alpha.yml | 4 ++-- .github/workflows/release_semver_start.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 208e5042c..591b4cabe 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -82,7 +82,7 @@ jobs: - name: Create Changelog run: | cd ${{ github.workspace }}/action/package/ - python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} env: GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} @@ -95,7 +95,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} ) echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT env: GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 1da012d82..89c939913 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -293,7 +293,7 @@ jobs: - name: Create Changelog run: | cd ${{ github.workspace }}/action/package/ - python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} env: GIT_CLIFF_PREPEND: ${{ github.workspace }}/action/package/${{ inputs.changelog_file }} GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} @@ -350,7 +350,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --items unreleased ) + CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} ) echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT env: GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} From 708fcc80e5ef6c094a4aff2b5ea5390d21f1b21c Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 15:58:34 +0100 Subject: [PATCH 323/357] tests --- scripts/parse_semver_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index 3f7ef0781..b585256a0 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -22,7 +22,7 @@ def get_version(): # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" - version = environ.get("VERSION") or environ.get("PREVIOUS_VERSION") + version = environ.get("VERSION", "") or environ.get("PREVIOUS_VERSION", "") match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", version) if match: return f"{match.group(1)}-{match.group(2)}" From 9993a5bca88eebfc38f1d8a7a7ff23630a509891 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 16:09:17 +0100 Subject: [PATCH 324/357] tests --- .github/workflows/release_alpha.yml | 2 +- .github/workflows/release_semver_start.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 591b4cabe..a85bd3711 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -73,7 +73,7 @@ jobs: else NEXT_VERSION=$(python semver_release_version.py --next --type alpha --file ${{ env.VERSION_FILE }} --save) fi - RELEASE_CYCLE_START=$( python semver_release_version.py cycle --version "$VERSION" --latest --type patch) + RELEASE_CYCLE_START=$( python semver_release_version.py --version "$VERSION" --latest --type patch) echo "RELEASE_CYCLE_START=${RELEASE_CYCLE_START}" >> $GITHUB_ENV echo "PREVIOUS_VERSION=${PREVIOUS_VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 89c939913..0b64b7872 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -285,7 +285,7 @@ jobs: echo "TEST_VERSION=${TEST_VERSION}" >> $GITHUB_ENV PREVIOUS_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest) NEXT_VERSION=$(python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$PREVIOUS_VERSION" --next --type alpha) - RELEASE_CYCLE_START=$( python ${{ github.workspace }}/action/github/scripts/semver_release_version.py cycle --version "$TEST_VERSION" --latest --type patch) + RELEASE_CYCLE_START=$( python ${{ github.workspace }}/action/github/scripts/semver_release_version.py --version "$TEST_VERSION" --latest --type patch) echo "RELEASE_CYCLE_START=${RELEASE_CYCLE_START}" >> $GITHUB_ENV echo "PREVIOUS_VERSION=${PREVIOUS_VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV From 81035cb2e3e9ed7728c190f1bba24ffb06d05d80 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 17:01:10 +0100 Subject: [PATCH 325/357] tests --- scripts/changelog_postprocess.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 0c6d206f7..99e49c1c6 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -1,5 +1,5 @@ -from os import environ -from os.path import join, dirname, isfile +from os import environ, getcwd +from os.path import join, isfile import sys import subprocess import json @@ -20,7 +20,8 @@ args.since = None PULL_LINK_PATTERN = r' \(\[#\d+\]\(https:\/\/github\.com\/.+?\/pull\/\d+\)\)' -CLIFF_IGNORE_FILE = join(environ.get("REPO_BASEDIR", ""), ".cliffignore") +CLIFF_WORKDIR = environ.get("GIT_CLIFF_WORKDIR", getcwd()) +CLIFF_IGNORE_FILE = join(CLIFF_WORKDIR, ".cliffignore") GIT_CLIFF_OUTPUT = environ.get("GIT_CLIFF_OUTPUT") if GIT_CLIFF_OUTPUT: del environ["GIT_CLIFF_OUTPUT"] @@ -37,9 +38,9 @@ def strip_pull_request_links(text): return re.sub(PULL_LINK_PATTERN, '', text).strip() -def in_git_repo(file_path): +def in_git_repo(): try: - subprocess.check_output(['git', '-C', dirname(file_path), 'rev-parse']) + subprocess.check_output(['git', '-C', CLIFF_WORKDIR, 'rev-parse']) return True except subprocess.CalledProcessError: return False @@ -47,7 +48,7 @@ def in_git_repo(file_path): def is_tag(tag): try: - subprocess.check_output(['git', 'rev-parse', tag]) + subprocess.check_output(['git', '-C', CLIFF_WORKDIR, 'rev-parse', tag]) return True except subprocess.CalledProcessError: return False @@ -72,7 +73,7 @@ def run_cliff(get_context = False): elif args.items == "current": command.append("--current") elif args.since: - last_commit = subprocess.check_output(["git", "log", "-1", "--pretty=format:'%h'"]).decode().strip() + last_commit = subprocess.check_output(["git", "-C", CLIFF_WORKDIR, "log", "-1", "--pretty=format:%h"]).decode().strip() command.append(f"{args.since}..{last_commit}") if get_context: @@ -105,8 +106,8 @@ def run_cliff(get_context = False): if not valid_json(CONTEXT): raise Exception("You need to provide a valid changelog context (json)") -if not in_git_repo(CLIFF_IGNORE_FILE): - raise Exception("You have to run this script in a git repository or provide a proper `REPO_BASEDIR` environment variable.") +if not in_git_repo(): + raise Exception("You have to run this script in a git repository or provide a proper `GIT_CLIFF_WORKDIR` environment variable.") elif args.since and not is_tag(args.since): raise Exception(f"The tag provided {args.since} doesn't exist.") else: @@ -138,4 +139,4 @@ def run_cliff(get_context = False): run_cliff() # delete the ignore file -subprocess.run(["rm", CLIFF_IGNORE_FILE]) +subprocess.run(["rm", "-f", CLIFF_IGNORE_FILE]) From ef06d3a91f369d6cc3d853d7c1b0991eda6739df Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 17:45:46 +0100 Subject: [PATCH 326/357] tests --- scripts/changelog_postprocess.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 99e49c1c6..6a1e640af 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -93,8 +93,7 @@ def run_cliff(get_context = False): # print the subprocess's standard output and error streams if not mute: - print(stdout.decode()) - print(stderr.decode(), file=sys.stderr) + print(stdout.decode().replace('\n', '%0A')) return stdout.decode().strip() From 957750a5ba95054074c23573ebc3ed14c21bb87e Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 17:52:26 +0100 Subject: [PATCH 327/357] tests --- scripts/changelog_postprocess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 6a1e640af..3f5032b88 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -91,9 +91,11 @@ def run_cliff(get_context = False): # interact with the subprocess's standard output and error streams stdout, stderr = process.communicate() - # print the subprocess's standard output and error streams if not mute: - print(stdout.decode().replace('\n', '%0A')) + if stderr.strip(): + print(stderr.decode().replace('\n', '%0A')) + else: + print(stdout.decode().replace('\n', '%0A')) return stdout.decode().strip() From 671462fd3b975ad890999e6c46eac812a5141852 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 20:11:53 +0100 Subject: [PATCH 328/357] tests --- .github/actions/notify_testing_matrix/action.yml | 4 +++- .github/workflows/check_conventional_commits.yml | 6 +++--- .github/workflows/clear_test_phase.yml | 4 ++-- .github/workflows/notify_testing_matrix.yml | 5 +---- .github/workflows/release_semver_publish.yml | 6 ++---- .github/workflows/release_semver_start.yml | 5 +++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/actions/notify_testing_matrix/action.yml b/.github/actions/notify_testing_matrix/action.yml index d739a38b6..b94f33de8 100644 --- a/.github/actions/notify_testing_matrix/action.yml +++ b/.github/actions/notify_testing_matrix/action.yml @@ -13,6 +13,8 @@ inputs: version: description: 'The version of the release' default: "version unknown" + repository: + description: 'The repository that triggered the action' runs: using: 'composite' @@ -23,4 +25,4 @@ runs: homeserver: 'matrix.org' token: ${{ inputs.matrix_token }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: "Testing QA phase of a ${{ inputs.release_type }} release (${{ inputs.version }}) started ($GITHUB_REPOSITORY). Subject to test: ${{ inputs.subject }}" \ No newline at end of file + message: "Testing QA phase in repo ${{ inputs.repository }} of a ${{ inputs.release_type }} release (${{ inputs.version }}) started. Subject to test: ${{ inputs.subject }}" \ No newline at end of file diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index f75b71553..ccdf3c889 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -48,12 +48,12 @@ jobs: python -m pip install --upgrade pip pip install -r ${{ github.workspace }}/requirements/pip_base.txt - name: Get Cached Test Status - uses: actions/cache/restore@v4 + uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }} + key: test-status-${{ runner.os }}-${{ github.run_number }} restore-keys: | - test-status- + test-status-${{ runner.os }}- - name: Status Check run: | python scripts/check_PRs.py diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml index e5d0ffe4b..d62097e63 100644 --- a/.github/workflows/clear_test_phase.yml +++ b/.github/workflows/clear_test_phase.yml @@ -10,7 +10,7 @@ jobs: run: | echo "complete" > test-status - name: Cache test status - uses: actions/cache/save@v4 + uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }} + key: test-status-${{ runner.os }}-${{ github.run_number }} diff --git a/.github/workflows/notify_testing_matrix.yml b/.github/workflows/notify_testing_matrix.yml index 4ce4cccfa..39a0dce94 100644 --- a/.github/workflows/notify_testing_matrix.yml +++ b/.github/workflows/notify_testing_matrix.yml @@ -1,12 +1,9 @@ -name: Notify Matrix Chat +name: Notify Matrix Chat (Test State) # only triggers on pull request closed events on: workflow_call: inputs: - # kickoff_pr: - # type: boolean - # default: true subject: type: string default: 'no subject' diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index bfda89b82..57abc361e 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -103,12 +103,10 @@ jobs: run: | echo "complete" > test-status - name: Cache test status - uses: actions/cache@v4 + uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }} - restore-keys: | - test-status- + key: test-status-${{ runner.os }}-${{ github.run_number }} # NO REBASE APPROACH!! # - name: Rebase dev onto master # run: | diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 0b64b7872..1960da68b 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -231,10 +231,10 @@ jobs: cd ${{ github.workspace }} echo "testing" > test-status - name: Cache test status - uses: actions/cache/save@v4 + uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }} + key: test-status-${{ runner.os }}-${{ github.run_number }} - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: @@ -242,6 +242,7 @@ jobs: subject: ${{ inputs.subject }} release_type: ${{ env.RELEASE_TYPE }} version: ${{ env.NEXT_VERSION }} + repository: ${{ github.repository }} # testing release # note: autotranslate as dependency requires the job running without restrictions! cherry_pick_testing: From f2add8920255904596b0729fe22983dd98d92b63 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 20:33:31 +0100 Subject: [PATCH 329/357] tests --- .github/workflows/check_conventional_commits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index ccdf3c889..9f4fc21a1 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -37,6 +37,7 @@ jobs: PR_NUMBER: ${{ inputs.pr_number }} ERROR_ON_FAILURE: ${{ inputs.error_on_failure }} TEST_PHASE_CACHE: ${{ github.workspace }}/test-status + PCCC_CONFIG_FILE: ${{ github.workspace }}/pccc.toml steps: - name: Checkout Scripts Repo uses: actions/checkout@v4 From e6996cc18fd95a7bf701d095cc3fff3b10e9047f Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 21:25:56 +0100 Subject: [PATCH 330/357] tests --- scripts/changelog_postprocess.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 3f5032b88..704bc8da9 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -29,6 +29,8 @@ if GIT_CLIFF_PREPEND: del environ["GIT_CLIFF_PREPEND"] +GITHUB_ACTION_OUTPUT = environ.get("GITHUB_OUTPUT") + def escape_control_characters(s): return re.sub(r'[\x00-\x1f\x7f-\x9f]', lambda c: "\\u{0:04x}".format(ord(c.group())), s) @@ -93,9 +95,15 @@ def run_cliff(get_context = False): if not mute: if stderr.strip(): - print(stderr.decode().replace('\n', '%0A')) + output = stderr.decode() + else: + output = stdout.decode() + + if GITHUB_ACTION_OUTPUT: + with open(GITHUB_ACTION_OUTPUT, 'a') as f: + f.write(f"changelog={output}\n") else: - print(stdout.decode().replace('\n', '%0A')) + print(output) return stdout.decode().strip() From d11cf69e2bf1faa26f825c71cd8c8f5f65dcaa06 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 21:37:15 +0100 Subject: [PATCH 331/357] tests --- .github/workflows/release_alpha.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index a85bd3711..0af756ef2 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -95,8 +95,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} ) - echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} env: GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - name: Create Pre-release From 19d73c813105a2c8226451b6dcc582eb85196eba Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 21:46:39 +0100 Subject: [PATCH 332/357] tests --- scripts/changelog_postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 704bc8da9..b6f9be004 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -101,7 +101,7 @@ def run_cliff(get_context = False): if GITHUB_ACTION_OUTPUT: with open(GITHUB_ACTION_OUTPUT, 'a') as f: - f.write(f"changelog={output}\n") + f.write(f'changelog="{output}"') else: print(output) From b3e5cc51f0b1f5eaba5f36400bf2c6500169eff8 Mon Sep 17 00:00:00 2001 From: emphasize Date: Tue, 26 Mar 2024 22:05:36 +0100 Subject: [PATCH 333/357] tests --- scripts/changelog_postprocess.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index b6f9be004..6fb4b1856 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -1,10 +1,11 @@ -from os import environ, getcwd +from os import environ, getcwd, urandom from os.path import join, isfile import sys import subprocess import json import re import argparse +import base64 parser = argparse.ArgumentParser() @@ -100,8 +101,11 @@ def run_cliff(get_context = False): output = stdout.decode() if GITHUB_ACTION_OUTPUT: + delimiter = base64.b64encode(urandom(15)).decode('utf-8') with open(GITHUB_ACTION_OUTPUT, 'a') as f: - f.write(f'changelog="{output}"') + f.write(f'changelog<<{delimiter}\n') + f.write(f'{output}\n') + f.write(f'{delimiter}\n') else: print(output) From a09d566701bf6ce4b05c67a128003294e384849e Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 14:34:06 +0100 Subject: [PATCH 334/357] tests --- .github/workflows/release_semver_publish.yml | 3 +-- .github/workflows/release_semver_pull_master.yml | 3 +-- .github/workflows/release_semver_start.yml | 3 +-- scripts/changelog_postprocess.py | 2 ++ 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 57abc361e..628b2f1b7 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -74,8 +74,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.RELEASE_CYCLE_START }} ) - echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.RELEASE_CYCLE_START }} env: GIT_CLIFF_TAG: ${{ env.RELEASE_VERSION }} - name: Create Release diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index 8cb39e866..a0f571a27 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -69,8 +69,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.LATEST_RELEASE }} ) - echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.LATEST_RELEASE }} env: GIT_CLIFF_TAG: ${{ env.RELEASE_VERSION }} - name: Create Master Branch (if necessary) diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index 1960da68b..aef89b2e8 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -351,8 +351,7 @@ jobs: id: release_changelog run: | cd ${{ github.workspace }}/action/package/ - CHANGELOG=$( python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} ) - echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT + python ${{ github.workspace }}/action/github/scripts/changelog_postprocess.py --since ${{ env.PREVIOUS_VERSION }} env: GIT_CLIFF_TAG: ${{ env.NEXT_VERSION }} - name: (Re)Set and push Release Version diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 6fb4b1856..5feeeeeb3 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -83,9 +83,11 @@ def run_cliff(get_context = False): command.append("--context") mute = True elif GIT_CLIFF_OUTPUT: + GITHUB_ACTION_OUTPUT = None command.append("--output") command.append(GIT_CLIFF_OUTPUT) elif GIT_CLIFF_PREPEND: + GITHUB_ACTION_OUTPUT = None command.append("--prepend") command.append(GIT_CLIFF_PREPEND) From e3ca54c81ca5d880d2d150bbecb0ac0b46404587 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 14:41:24 +0100 Subject: [PATCH 335/357] tests --- scripts/changelog_postprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 5feeeeeb3..ebdb4e877 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -31,6 +31,8 @@ del environ["GIT_CLIFF_PREPEND"] GITHUB_ACTION_OUTPUT = environ.get("GITHUB_OUTPUT") +if GIT_CLIFF_OUTPUT or GIT_CLIFF_PREPEND: + GITHUB_ACTION_OUTPUT = None def escape_control_characters(s): @@ -83,11 +85,9 @@ def run_cliff(get_context = False): command.append("--context") mute = True elif GIT_CLIFF_OUTPUT: - GITHUB_ACTION_OUTPUT = None command.append("--output") command.append(GIT_CLIFF_OUTPUT) elif GIT_CLIFF_PREPEND: - GITHUB_ACTION_OUTPUT = None command.append("--prepend") command.append(GIT_CLIFF_PREPEND) From 240cc7e7f9fe94d87c4e56fb4d7e3778b43a9da0 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 15:03:51 +0100 Subject: [PATCH 336/357] tests --- scripts/parse_semver_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index b585256a0..b40142de8 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -22,7 +22,9 @@ def get_version(): # note: this is a PEP 440 compliant version, so alpha versions come in "1.0.0a1" - version = environ.get("VERSION", "") or environ.get("PREVIOUS_VERSION", "") + version = environ.get("VERSION", "") or \ + environ.get("RELEASE_VERSION", "") or \ + environ.get("PREVIOUS_VERSION", "") match = re.match(r"(\d+\.\d+\.\d+)([aA-zZ].*)", version) if match: return f"{match.group(1)}-{match.group(2)}" From cad2ff8424373dd108fb4b5283277944609aca41 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 15:44:00 +0100 Subject: [PATCH 337/357] tests --- scripts/changelog_postprocess.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index ebdb4e877..18b95c4eb 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -60,11 +60,12 @@ def is_tag(tag): def valid_json(s): - try: - json.loads(escape_control_characters(s)) - return True - except json.JSONDecodeError: - return False + #try: + json.loads(escape_control_characters(s)) + return True + # return True + # except json.JSONDecodeError: + # return False def run_cliff(get_context = False): From 085bfb6c1a1dab13d6911b64fa499694eaf46d0c Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 15:47:47 +0100 Subject: [PATCH 338/357] tests --- scripts/changelog_postprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 18b95c4eb..7d7413ac9 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -61,6 +61,7 @@ def is_tag(tag): def valid_json(s): #try: + print(s) json.loads(escape_control_characters(s)) return True # return True From f3f762e735d77a75925c553d765c19d440d311fd Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 16:00:51 +0100 Subject: [PATCH 339/357] tests --- scripts/changelog_postprocess.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 7d7413ac9..9d121f25d 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -60,13 +60,11 @@ def is_tag(tag): def valid_json(s): - #try: - print(s) - json.loads(escape_control_characters(s)) - return True - # return True - # except json.JSONDecodeError: - # return False + try: + json.loads(escape_control_characters(s)) + return True + except json.JSONDecodeError: + return False def run_cliff(get_context = False): @@ -98,6 +96,8 @@ def run_cliff(get_context = False): # interact with the subprocess's standard output and error streams stdout, stderr = process.communicate() + print(command) + print(stdout) if not mute: if stderr.strip(): output = stderr.decode() From b10cc3436b924fba1034c83ad6f9cebc711f059e Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 16:07:03 +0100 Subject: [PATCH 340/357] tests --- .github/workflows/release_semver_pull_master.yml | 1 + scripts/changelog_postprocess.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index a0f571a27..c6e2e531f 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -31,6 +31,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} ref: testing + fetch-depth: 0 path: action/package/ - name: Checkout Scripts Repo uses: actions/checkout@v4 diff --git a/scripts/changelog_postprocess.py b/scripts/changelog_postprocess.py index 9d121f25d..ebdb4e877 100644 --- a/scripts/changelog_postprocess.py +++ b/scripts/changelog_postprocess.py @@ -96,8 +96,6 @@ def run_cliff(get_context = False): # interact with the subprocess's standard output and error streams stdout, stderr = process.communicate() - print(command) - print(stdout) if not mute: if stderr.strip(): output = stderr.decode() From 4100827da7af6eccfd7de2673def072569f7ba51 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 16:11:51 +0100 Subject: [PATCH 341/357] tests --- .github/workflows/release_semver_pull_master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_semver_pull_master.yml b/.github/workflows/release_semver_pull_master.yml index c6e2e531f..b89a9bc49 100644 --- a/.github/workflows/release_semver_pull_master.yml +++ b/.github/workflows/release_semver_pull_master.yml @@ -76,6 +76,7 @@ jobs: - name: Create Master Branch (if necessary) run: | cd ${{ github.workspace }}/action/package/ + git fetch origin dev:dev TARGET_BRANCH_EXISTS=$(git ls-remote --heads origin ${{ env.TARGET_BRANCH }} | wc -l) INITIAL_COMMIT=$(git rev-list --max-parents=0 dev) if [ "$TARGET_BRANCH_EXISTS" -eq "0" ]; then From 545dff6f0213a513931006dea2f32a4715937e16 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 16:36:51 +0100 Subject: [PATCH 342/357] tests --- .github/workflows/release_semver_publish.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 628b2f1b7..aa597ef25 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -36,10 +36,6 @@ jobs: GITHUB_REPO: ${{ github.repository }} runs-on: ubuntu-latest steps: - - name: set up release type - run: | - RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') - echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - name: Checkout Repository uses: actions/checkout@v4 with: @@ -106,10 +102,3 @@ jobs: with: path: test-status key: test-status-${{ runner.os }}-${{ github.run_number }} - # NO REBASE APPROACH!! - # - name: Rebase dev onto master - # run: | - # cd ${{ github.workspace }}/action/package - # git checkout dev - # git rebase master - # git push origin dev --force From 3e6fc213d508691cc888d5732f6c9cfa8f11d374 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 16:54:26 +0100 Subject: [PATCH 343/357] tests --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 58e3762a4..2544772a9 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,13 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling --------- - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. - - General rule: PRs/commits are dev only, other branches will be protected. - - _Testing QA_: PRs/commits that directly address issues with that release need a prefix/scope `fix(testing):`. _Those_ will be part of the patch/minor/major release. The version of the `testing` branch is to be held steady. A fix for later releases has to be commited as usual with `fix: ...`. - - Release-tags: The tag resembles the semantic versioning (eg. `1.2.3`; no prefix!, alphas will be formatted appropriately) + - General rule: PRs/commits are _dev only_, other branches will be protected. + - PRs/commits that directly address issues with a **release** need a prefix/scope `fix(release):`/`release():`. _Those_ will be part of the patch/minor/major release. + - The version of the `testing` branch is to be held steady. + - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing" + - The PR title of release PRs that conclude the testing phase (_Proposals_) mustn't be edited + - If those proposals need additional changes, the PR is to be closed until the changes are commited (to dev). + - Release-tags: The tag resembles the semantic versioning (eg. `1.2.3`; no prefix!, alphas will be formatted appropriately) TODO (per repo): - `setup.py`: setuptools cant cope with semver compliance: `x.x.x-alpha...` stays `x.x.xax` for now From 8cb802fa792b519dac77c289e00d0d917713479f Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 17:14:55 +0100 Subject: [PATCH 344/357] tests --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2544772a9..57822a935 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,10 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. - General rule: PRs/commits are _dev only_, other branches will be protected. - PRs/commits that directly address issues with a **release** need a prefix/scope `fix(release):`/`release():`. _Those_ will be part of the patch/minor/major release. + - remember: `ci`/`style`/`test`/`docs` commits don't get a release. - The version of the `testing` branch is to be held steady. - - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing" + - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". + - All pending PRs get a daily check (00:00 UTC) and labeling is adjusted accordingly. - The PR title of release PRs that conclude the testing phase (_Proposals_) mustn't be edited - If those proposals need additional changes, the PR is to be closed until the changes are commited (to dev). - Release-tags: The tag resembles the semantic versioning (eg. `1.2.3`; no prefix!, alphas will be formatted appropriately) From c4ddf14b1b876f83267741810c0e1547f21fe598 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 17:16:23 +0100 Subject: [PATCH 345/357] tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57822a935..3bfc965bb 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - PRs/commits that directly address issues with a **release** need a prefix/scope `fix(release):`/`release():`. _Those_ will be part of the patch/minor/major release. - remember: `ci`/`style`/`test`/`docs` commits don't get a release. - The version of the `testing` branch is to be held steady. - - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". + - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". Usually it`s best ti include them into the ongoing release/test. - All pending PRs get a daily check (00:00 UTC) and labeling is adjusted accordingly. - The PR title of release PRs that conclude the testing phase (_Proposals_) mustn't be edited - If those proposals need additional changes, the PR is to be closed until the changes are commited (to dev). From 10a9919b7b8dc999c50e5920567e6f0897f6caec Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 17:17:02 +0100 Subject: [PATCH 346/357] tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bfc965bb..255c44d27 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - PRs/commits that directly address issues with a **release** need a prefix/scope `fix(release):`/`release():`. _Those_ will be part of the patch/minor/major release. - remember: `ci`/`style`/`test`/`docs` commits don't get a release. - The version of the `testing` branch is to be held steady. - - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". Usually it`s best ti include them into the ongoing release/test. + - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". Usually it`s best to include them into the ongoing release/test. - All pending PRs get a daily check (00:00 UTC) and labeling is adjusted accordingly. - The PR title of release PRs that conclude the testing phase (_Proposals_) mustn't be edited - If those proposals need additional changes, the PR is to be closed until the changes are commited (to dev). From 94ff60cfa3268f9ade4bd439d00994072ab5dfea Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 17:29:50 +0100 Subject: [PATCH 347/357] tests --- scripts/check_PRs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index 8974551ae..d4f59f363 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -98,7 +98,7 @@ def check_cc_labels(desc: str) -> List[str]: # clear the test flag if the PR adresses a release. Ie. gets added to the test if SINGLE_PR: - if cc_type(pr_description) == "release" or \ + if cc_type(pr_description) in ["release", "ci", "style", "test", "docs"] or \ cc_scope(pr_description) == "release": ongoing_test = False if cc_type(pr_description) == "unknown": From 1642d72b6f4d3d0162f33b3494cc71d5d79133ab Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 20:07:34 +0100 Subject: [PATCH 348/357] tests --- .../workflows/check_conventional_commits.yml | 4 +--- .github/workflows/clear_test_phase.yml | 19 ++++++++++++++++++- .github/workflows/release_semver_publish.yml | 19 ++++++++++++++++++- .github/workflows/release_semver_start.yml | 19 ++++++++++++++++++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_conventional_commits.yml b/.github/workflows/check_conventional_commits.yml index 9f4fc21a1..ff4b49c53 100644 --- a/.github/workflows/check_conventional_commits.yml +++ b/.github/workflows/check_conventional_commits.yml @@ -52,9 +52,7 @@ jobs: uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }}-${{ github.run_number }} - restore-keys: | - test-status-${{ runner.os }}- + key: test-status-${{ runner.os }} - name: Status Check run: | python scripts/check_PRs.py diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml index d62097e63..482e3acad 100644 --- a/.github/workflows/clear_test_phase.yml +++ b/.github/workflows/clear_test_phase.yml @@ -9,8 +9,25 @@ jobs: - name: Set test status run: | echo "complete" > test-status + - name: Delete Test Tracking Cache + run: | + gh extension install actions/gh-actions-cache + unset GITHUB_TOKEN + echo "${{ secrets.GH_PAT }}" | gh auth login --with-token + + echo "Fetching list of cache key" + cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeys + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm + done + echo "Done" - name: Cache test status uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }}-${{ github.run_number }} + key: test-status-${{ runner.os }} diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index aa597ef25..5a335139c 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -97,8 +97,25 @@ jobs: - name: Set test status run: | echo "complete" > test-status + - name: Delete Test Tracking Cache + run: | + gh extension install actions/gh-actions-cache + unset GITHUB_TOKEN + echo "${{ secrets.GH_PAT }}" | gh auth login --with-token + + echo "Fetching list of cache key" + cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeys + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm + done + echo "Done" - name: Cache test status uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }}-${{ github.run_number }} + key: test-status-${{ runner.os }} diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index aef89b2e8..b8042d754 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -230,11 +230,28 @@ jobs: run: | cd ${{ github.workspace }} echo "testing" > test-status + - name: Delete Test Tracking Cache + run: | + gh extension install actions/gh-actions-cache + unset GITHUB_TOKEN + echo "${{ secrets.GH_PAT }}" | gh auth login --with-token + + echo "Fetching list of cache key" + cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeys + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm + done + echo "Done" - name: Cache test status uses: actions/cache@v2 with: path: test-status - key: test-status-${{ runner.os }}-${{ github.run_number }} + key: test-status-${{ runner.os }} - name: Notify Matrix Room uses: ./action/github/.github/actions/notify_testing_matrix with: From 664a0da34f42a0f8acf95b44ea05bc4a5b1bf782 Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Mar 2024 20:12:01 +0100 Subject: [PATCH 349/357] tests --- .github/workflows/clear_test_phase.yml | 4 ++-- .github/workflows/release_semver_publish.yml | 4 ++-- .github/workflows/release_semver_start.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clear_test_phase.yml b/.github/workflows/clear_test_phase.yml index 482e3acad..b5edb80e0 100644 --- a/.github/workflows/clear_test_phase.yml +++ b/.github/workflows/clear_test_phase.yml @@ -12,8 +12,6 @@ jobs: - name: Delete Test Tracking Cache run: | gh extension install actions/gh-actions-cache - unset GITHUB_TOKEN - echo "${{ secrets.GH_PAT }}" | gh auth login --with-token echo "Fetching list of cache key" cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) @@ -26,6 +24,8 @@ jobs: gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm done echo "Done" + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - name: Cache test status uses: actions/cache@v2 with: diff --git a/.github/workflows/release_semver_publish.yml b/.github/workflows/release_semver_publish.yml index 5a335139c..076951def 100644 --- a/.github/workflows/release_semver_publish.yml +++ b/.github/workflows/release_semver_publish.yml @@ -100,8 +100,6 @@ jobs: - name: Delete Test Tracking Cache run: | gh extension install actions/gh-actions-cache - unset GITHUB_TOKEN - echo "${{ secrets.GH_PAT }}" | gh auth login --with-token echo "Fetching list of cache key" cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) @@ -114,6 +112,8 @@ jobs: gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm done echo "Done" + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - name: Cache test status uses: actions/cache@v2 with: diff --git a/.github/workflows/release_semver_start.yml b/.github/workflows/release_semver_start.yml index b8042d754..aee966583 100644 --- a/.github/workflows/release_semver_start.yml +++ b/.github/workflows/release_semver_start.yml @@ -233,8 +233,6 @@ jobs: - name: Delete Test Tracking Cache run: | gh extension install actions/gh-actions-cache - unset GITHUB_TOKEN - echo "${{ secrets.GH_PAT }}" | gh auth login --with-token echo "Fetching list of cache key" cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -L 100 | cut -f 1 ) @@ -247,6 +245,8 @@ jobs: gh actions-cache delete $cacheKey -R ${{ github.repository }} --confirm done echo "Done" + env: + GH_TOKEN: ${{ secrets.GH_PAT }} - name: Cache test status uses: actions/cache@v2 with: From 6a10d64c194fd825b85d27bd55f19345ee261a79 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 28 Mar 2024 14:33:49 +0100 Subject: [PATCH 350/357] tests --- .../workflows/release_semver_versioning.yml | 77 ------------------- .../workflows/release_synchronize_testing.yml | 25 ------ 2 files changed, 102 deletions(-) delete mode 100644 .github/workflows/release_semver_versioning.yml delete mode 100644 .github/workflows/release_synchronize_testing.yml diff --git a/.github/workflows/release_semver_versioning.yml b/.github/workflows/release_semver_versioning.yml deleted file mode 100644 index d83806501..000000000 --- a/.github/workflows/release_semver_versioning.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Bump Version (after testing QA started) and announce testing phase - -# only triggers on pull request closed events -on: - workflow_call: - inputs: - action_branch: - description: 'The shared action branch to checkout' - type: string - default: main - python_version: - description: 'The python version to use' - type: string - default: "3.8" - version_file: - description: 'The file location to read the version from' - type: string - default: version.py - changelog_file: - type: string - default: CHANGELOG.md - subject: - type: string - description: 'Subject title of the push/pull-request event to parse the release type.' - required: true - -jobs: - bump_version_and_announce_testing: - env: - VERSION_FILE: ${{ github.workspace }}/action/package/${{ inputs.version_file }} - SOURCE_BRANCH: 'testing' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: set up release type - run: | - RELEASE_TYPE=$(echo "${{ inputs.subject }}" | grep -oP '(patch|minor|major)') - echo "RELEASE_TYPE=${RELEASE_TYPE}" >> $GITHUB_ENV - - name: Checkout Package - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_PAT }} - ref: dev - fetch-depth: 0 - path: action/package - - name: Checkout Scripts Repo - uses: actions/checkout@v4 - with: - repository: OpenVoiceOS/.github - ref: ${{ inputs.action_branch }} - path: action/github/ - - name: Set up python ${{ inputs.python_version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - name: Install Common Python Requirements - run: | - pip install -r ${{ github.workspace }}/action/github/requirements/pip_base.txt - - name: Bump to next ${{ env.RELEASE_TYPE }} version - id: version - run: | - cd ${{ github.workspace }}/action/github/scripts/ - VERSION=$(python semver_release_version.py --file ${{ env.VERSION_FILE }}) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - NEXT_VERSION=$(python semver_release_version.py --next --file ${{ env.VERSION_FILE }} --type ${{ env.RELEASE_TYPE }} --save) - echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - - name: Push Version Change (${{ env.VERSION }} -> ${{ env.NEXT_VERSION }}) - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Increment Version to ${{ env.NEXT_VERSION }} - repository: action/package/ - - name: Notify Matrix Room - uses: ./action/github/.github/actions/notify_testing_matrix - with: - release_type: ${{ env.RELEASE_TYPE }} - version: ${{ env.NEXT_VERSION }} diff --git a/.github/workflows/release_synchronize_testing.yml b/.github/workflows/release_synchronize_testing.yml deleted file mode 100644 index 2fbc9fc2c..000000000 --- a/.github/workflows/release_synchronize_testing.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Sync Branch 'Testing' with 'Dev' - -on: - workflow_call: - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout Testing - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_PAT }} - ref: testing - fetch-depth: 0 - path: action/package - - name: Synchronize testing with dev - run: | - cd ${{ github.workspace }}/action/package - git pull --rebase origin dev - if [ $? -ne 0 ]; then - git rebase --abort - git pull --rebase -Xtheirs origin dev - fi - git push -f origin testing \ No newline at end of file From 9b38c7be8297e3236bd56b00bada307d12276aec Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 28 Mar 2024 14:53:30 +0100 Subject: [PATCH 351/357] tests --- .github/workflows/notify_pr_matrix.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/notify_pr_matrix.yml b/.github/workflows/notify_pr_matrix.yml index 69278403c..772cd19a3 100644 --- a/.github/workflows/notify_pr_matrix.yml +++ b/.github/workflows/notify_pr_matrix.yml @@ -7,11 +7,17 @@ on: pr_id: type: number required: true + subject: + type: string + default: '' jobs: notify_pr_matrix: runs-on: ubuntu-latest steps: + - name: Get repo + run: | + echo REPO=$(echo ${{ github.repository}} | cut -d '/' -f 2) >> $GITHUB_ENV - uses: actions/checkout@v4 - name: Send message to Matrix bots channel id: matrix-chat-message @@ -21,4 +27,4 @@ jobs: token: ${{ secrets.MATRIX_TOKEN }} channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' message: | - new ${{ github.repository }} PR merged! https://github.com/${{ github.repository }}/pull/${{ inputs.pr_id }} \ No newline at end of file + ${{ inputs.subject }} PR merged! (${{ env.REPO }}) [PR](https://github.com/${{ github.repository }}/pull/${{ inputs.pr_id }}) \ No newline at end of file From 41e522f2f35d889d7eaad323ea899a31b99664d7 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 28 Mar 2024 14:57:30 +0100 Subject: [PATCH 352/357] tests --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 255c44d27..c0b815453 100644 --- a/README.md +++ b/README.md @@ -376,19 +376,18 @@ jobs: ```yaml name: Notify Matrix Chat -# only trigger on pull request closed events on: pull_request: types: [ closed ] jobs: notify_pr_matrix: - # this job will only run if the PR has been merged if: github.event.pull_request.merged == true secrets: inherit uses: openvoiceos/.github/.github/workflows/notify_pr_matrix.yml@main with: pr_id: ${{ github.event.number }} + subject: ${{ github.event.pull_request.title }} ``` ------------- From e2e51c9ed161408d983879cae1347b3ddb653761 Mon Sep 17 00:00:00 2001 From: emphasize Date: Thu, 28 Mar 2024 15:17:21 +0100 Subject: [PATCH 353/357] tests --- README.md | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index c0b815453..1804cf43f 100644 --- a/README.md +++ b/README.md @@ -81,12 +81,11 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling _Alpha releases are directly published without going through a test phase_ Strategy: 3-staged - - Manually propose a testing start or automatically with setting "Conventional Commits" (optionally as PR) - - Kickoff PR: If the testing phase of the release should start right away or being time-delayed (starting when PR is merged). Input `kickoff_pr` (true/false) - - Manually conclude testing phase, propose a stable release (PR) - - Publishing of a stable release (on merge) and feed back the changes to the dev branch + - _Manually_ propose a testing start or _automatically_ with setting "Conventional Commits" + - _Manually_ conclude testing phase, propose a stable release (PR) + - _Automatically_ publishing a stable release (on merge) -_(manual/auto)_ **Start release mechanism** +**Start release mechanism** ```yaml name: Start release mechanism @@ -127,32 +126,8 @@ jobs: update_intentfile: test/unittests/test_intent.yaml # the intent file resources gets tested against release_type: ${{ inputs.release_type || null }} # if manually triggered, set a release type subject: ${{ github.event.head_commit.message || null }} # on push, the commit message is used as release subject - kickoff_pr: true # if the release process should be started with a PR to summarize and visualize, default: false ``` -_(auto/optional)_ **Start testing phase** - -_If a kickoff PR is opened, for versioning and synchronization purposes_ -_(if no PR is used, the testing phase starts right away)_ -```yaml -name: Start Testing Phase -# only trigger on pull request closed events -on: - pull_request: - types: [ closed ] - branches: - - testing - -jobs: - start_testing_phase: - if: github.actor != 'EggmanBot' && github.event.pull_request.merged == true && contains(fromJson('["patch release", "minor release", "major release"]'), github.event.pull_request.title) - uses: openvoiceos/.github/.github/workflows/release_semver_versioning.yml@feat/shared_actions1 - secrets: inherit - with: - action_branch: feat/shared_actions1 # Shared action branch to use, default: main - version_file: ovos_testpkg/version.py # the file containing the version number - subject: ${{ github.event.pull_request.title }} -``` -_(manual)_ **Conclude testing phase** +**Conclude testing phase** _After the testing phase, a PR is opened to propose the stable release_ ```yaml @@ -168,7 +143,7 @@ jobs: action_branch: shared_actions1 # Shared action branch to use, default: main python_version: "3.10" # the python version to use ``` -_(auto)_ **Publishing stable release** +**Publishing stable release** ```yaml name: Publish Stable Release @@ -181,7 +156,12 @@ on: jobs: publish_stable_release: - if: github.actor != 'EggmanBot' && github.event.pull_request.merged == true && contains(fromJson('["patch release stable", "minor release stable", "major release stable"]'), github.event.pull_request.title) + if: > + github.event.pull_request.merged == true && + github.actor != 'EggmanBot' && + (contains(github.event.pull_request.title, 'patch release stable') || + contains(github.event.pull_request.title, 'minor release stable') || + contains(github.event.pull_request.title, 'major release stable')) uses: openvoiceos/.github/.github/workflows/release_semver_publish.yml@feat/shared_actions1 secrets: inherit with: From 48bfd1ff97607389ea6504003964b89147a12c24 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 29 Mar 2024 19:17:28 +0100 Subject: [PATCH 354/357] tests --- scripts/check_PRs.py | 42 +++++++++++++++++++++++++-------- scripts/parse_semver_release.py | 16 +++++++++++-- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index d4f59f363..c18b030a2 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -1,6 +1,7 @@ import os from os.path import join, dirname, isfile import json +import re from typing import List, Optional from github import Github, PullRequest @@ -33,22 +34,39 @@ ongoing_test = content == "testing" +def get_scope(title): + match = re.match(r"^[a-z]+\s*\((.+)\):", title) + if match: + return match.group(1) + return None + + +def strip_scope(title): + return re.sub(r"^([a-z]+)\s*\(([^)]+)\):", r"\1:", title) + + def cc_type(desc: str) -> str: - ccr = parse_cc(desc) + ccr = parse_cc(strip_scope(desc)) if ccr: - if ccr.breaking.get("flag") or ccr.breaking.get("token"): - return "breaking" return ccr.header.get("type") return "unknown" +def cc_breaking(desc: str) -> bool: + ccr = parse_cc(strip_scope(desc)) + if ccr: + return ccr.breaking.get("flag") or ccr.breaking.get("token") + + return False + + def cc_scope(desc: str) -> str: ccr = parse_cc(desc) if ccr: return ccr.header.get("scope") - return "unknown" + return get_scope(desc) or "unknown" def parse_cc(desc: str) -> Optional[pccc.ConventionalCommitRunner]: @@ -69,16 +87,20 @@ def check_cc_labels(desc: str) -> List[str]: labels = set() _type = cc_type(desc) _scope = cc_scope(desc) - test_relevant_cc = ["feat", "fix", "refactor", "breaking"] + test_relevant_cc = ["feat", "fix", "refactor"] if _type == "unknown": return [PR_LABELS.get("need_cc", "CC missing")] - if _type == "breaking": + if cc_breaking(desc): labels.add(PR_LABELS.get("breaking", "breaking change")) + if _type == "release": + labels.add("fix") + elif _type in PR_LABELS: + labels.add(PR_LABELS.get(_type)) if _scope in PR_LABELS: labels.add(PR_LABELS.get(_scope)) - if _type in PR_LABELS: - labels.add(PR_LABELS.get(_type)) - if ongoing_test and any(t in test_relevant_cc for t in [_type, _scope]): + elif _scope != "unknown": + labels.add(_scope) + if ongoing_test and (any(t in test_relevant_cc for t in [_type, _scope]) or cc_breaking(desc)): labels.add("ongoing test") return list(labels) @@ -96,7 +118,7 @@ def check_cc_labels(desc: str) -> List[str]: labels = check_cc_labels(pr_description) pr.set_labels(*labels) - # clear the test flag if the PR adresses a release. Ie. gets added to the test + # clear the test flag if the PR adresses a release or doesn't get a release at all. if SINGLE_PR: if cc_type(pr_description) in ["release", "ci", "style", "test", "docs"] or \ cc_scope(pr_description) == "release": diff --git a/scripts/parse_semver_release.py b/scripts/parse_semver_release.py index b40142de8..d38e572cd 100644 --- a/scripts/parse_semver_release.py +++ b/scripts/parse_semver_release.py @@ -16,8 +16,18 @@ print(f"Config file {CONFIG_FILE} not found.") exit(1) -TITLE = environ.get("TITLE") -BODY = environ.get("BODY") + +def get_scope(title): + match = re.match(r"^[a-z]+\s*\((.+)\):", title) + if match: + return match.group(1) + return None + + +def strip_scope(title): + if get_scope(title) != "release": + return re.sub(r"^([a-z]+)\s*\(([^)]+)\):", r"\1:", title) + return title def get_version(): @@ -86,6 +96,8 @@ def semver_from_version(): elif version.major != 0: return "major" +TITLE = strip_scope(environ.get("TITLE", "")) +BODY = environ.get("BODY") VERSION = get_version() if VERSION: From e220fefc9d9589b40c04458f993f548bbac74856 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 29 Mar 2024 19:53:41 +0100 Subject: [PATCH 355/357] tests --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1804cf43f..35d84df53 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,10 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - `plugins`: Changes to the plugin system, - `localisation`: Changes to the localisation files, - `resources`: Changes to the resource files, - - `release`: Everything related to the release process + - `release`: Everything related to the release process + + This is an excerpt [list of predefined scopes](https://github.com/OpenVoiceOS/.github/blob/feat/shared_actions1/pccc.toml). This list is not exclusive, but the main system components. + If you have to be more specific, feel free to use submodule names or more detailed scopes. -------------- ## Premises From 1b3b68c7ce38a0da1b6a74d50a02a8b3714fbb66 Mon Sep 17 00:00:00 2001 From: emphasize Date: Fri, 29 Mar 2024 19:59:09 +0100 Subject: [PATCH 356/357] tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35d84df53..c9417f6b4 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Using below prefixes (eg. `fix: ...`) will automate the versioning and labelling - Pushed or merged commits without a proper title/commit message (Conventional Commit spec) will get no release. - Translations are supposed to be prefixed as `fix` or `feat` to get a stable release. - General rule: PRs/commits are _dev only_, other branches will be protected. - - PRs/commits that directly address issues with a **release** need a prefix/scope `fix(release):`/`release():`. _Those_ will be part of the patch/minor/major release. + - PRs/commits that directly address issues with a **release** need a prefix/scope `release():`. _Those_ will be part of the patch/minor/major release. - remember: `ci`/`style`/`test`/`docs` commits don't get a release. - The version of the `testing` branch is to be held steady. - A fix for later releases has to be commited as usual with `fix: ...`. Those PRs get a warning label "test ongoing". Usually it`s best to include them into the ongoing release/test. From e57c29451a67c4ef04df3bf524a9e9938297111d Mon Sep 17 00:00:00 2001 From: emphasize Date: Mon, 1 Apr 2024 16:44:46 +0200 Subject: [PATCH 357/357] tests --- scripts/check_PRs.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/check_PRs.py b/scripts/check_PRs.py index c18b030a2..74a6ac9c1 100644 --- a/scripts/check_PRs.py +++ b/scripts/check_PRs.py @@ -4,7 +4,8 @@ import re from typing import List, Optional -from github import Github, PullRequest +from github import Github +from github.Repository import Repository import pccc @@ -106,6 +107,17 @@ def check_cc_labels(desc: str) -> List[str]: return list(labels) +def ensure_label_exists(repo: Repository, labels: List[str], color: str = 'ffffff'): + for label_name in labels: + if not any(label.name == label_name for label in repo.get_labels()): + repo.create_label(label_name, color) + + # switch the strings to label objects + for label in repo.get_labels(): + if label.name in labels: + labels[labels.index(label.name)] = label + + git = Github(TOKEN).get_repo(REPOSITORY) open_pulls = git.get_pulls(state='open') cc_missing = False @@ -116,6 +128,7 @@ def check_cc_labels(desc: str) -> List[str]: continue pr_description = f"{pr.title}\n{pr.body}" labels = check_cc_labels(pr_description) + ensure_label_exists(git, labels) pr.set_labels(*labels) # clear the test flag if the PR adresses a release or doesn't get a release at all.