From 93e54bd6028bf15eae79e45762c2c6a5b5439267 Mon Sep 17 00:00:00 2001 From: Hussam Suleiman <38970953+Hussam-Suleiman@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:28:25 +0300 Subject: [PATCH 1/6] Revert "Master depth calculation" From 3eef1be8dee4aa6f281f2b5daab7909319819686 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Fri, 8 Sep 2023 17:50:53 +0200 Subject: [PATCH 2/6] [ADD] update_depths script --- bin/update_depths.py | 228 +++++++++++++++++++++++++++++++++++++++++++ build | 3 +- update_depths | 15 +++ 3 files changed, 245 insertions(+), 1 deletion(-) create mode 100755 bin/update_depths.py create mode 100755 update_depths diff --git a/bin/update_depths.py b/bin/update_depths.py new file mode 100755 index 00000000..ec7d0d3a --- /dev/null +++ b/bin/update_depths.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python +# Version: v.21.05.30 +import os +import argparse +from string import Template +from subprocess import run, PIPE +from dotenv import load_dotenv + +import yaml +from waftlib import ( + PRIVATE, + REPOS_YAML, + SRC_DIR, + logger, + ODOO_VERSION, +) + +SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +os.environ["ODOO_WORK_DIR"] = os.path.realpath(os.path.join(SCRIPT_PATH, "../..")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) + + +parser = argparse.ArgumentParser( + prog="get_pins", + description="Fetch current pins of this instance", + epilog="Outputs pinned repo.yaml", +) + +parser.add_argument("--only-base") +args = parser.parse_args() + + +all_commits = {} +repo_min_depth = {} + + +class CustomDumper(yaml.Dumper): + """ + We want order of keys intact so that the output is the same order than source. + Unfortunately PyYAML orders yaml output alphabetically on dump. + pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our + wafts have a version < 5.1. + for this reason i made a custom dumper to have an "unordered" dump, without + affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. + Our output will therefore have the same order as given input. + """ + + def represent_dict_preserve_order(self, data): + return self.represent_dict(data.items()) + + +def split_line(line): + splitted_lines = line.split(" ") + for sl in splitted_lines: + if sl == "": + splitted_lines.pop(splitted_lines.index(sl)) + return splitted_lines + + +def is_in_history(value, history): + for commit in history: + if commit[:8] == value[:8]: + return True + return False + + +def decode_variables(string): + """ + pass a string and return variables replaced in it. + @returns string + """ + # may be int, str + string = str(string) + yaml_substituted = Template(string) + return yaml_substituted.substitute(os.environ) + + +def preprocess_merge(doc, repo, merge): + remotes = doc[repo]["remotes"].keys() + splitted_merge = [x for x in merge.split(" ") if x != ""] + if splitted_merge[0] not in remotes: + logger.debug("Invalid Remote on line: %s" % merge) + raise ValueError + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + return repo_path, splitted_merge + + +def get_branchname(splitted_merge, merge_type): + if merge_type in (1, 3): + return decode_variables(splitted_merge[1]) + return False + +def get_merge_type(splitted_merge, repo): + """ + Possible syntaxes for merges: + + merge_type=1 + merge_type=2 + merge_type=3 + merge_type=0 "invalid" + """ + if len(splitted_merge) == 3: + return 3 + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + return 2 + else: + return 1 + return 0 # unreachable. + + +def process_depth(splitted_merge, branchname, main_branch, main_branch_name, repo_path): + os.chdir(repo_path) + lastrev = ( + run( + [ + "git", + "merge-base", + "".join([main_branch[0], "/", main_branch_name]), + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + # we now calculate the needed depth of this branch + mindepth = ( + run( + [ + "git", + "rev-list", + "".join([main_branch[0], "/", main_branch_name]), + "^" + lastrev, + "--count", + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + return int(mindepth) + + +def main(): + """ + parsing directly repos.yaml, if something is not in addons.yaml, branch will still + be in folder, but may not be included in addons. Nothing changes. + """ + changes = '' + with open(REPOS_YAML) as yaml_file: + for doc in yaml.safe_load_all(yaml_file): + for repo in doc: + print("===>processing repo %s" % repo) + repo_min_depth[repo] = 0 + if repo in {PRIVATE, "ONLY", "ENV"}: + continue + target = doc[repo].get("target") or False + # main branch is defined as target or in absence of target, merge[0] + main_branch = split_line(target or doc[repo]["merges"][0]) + merge_type = get_merge_type(main_branch, repo) + main_branch_name = get_branchname(main_branch, merge_type) + for merge in doc[repo]["merges"]: + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + # this script cannot work on new ./builds it is written to keep + # depths of instances that have been built at least once with + # if one source folder is missing we skip it. + if not os.path.exists(repo_path): + continue + merge_type = get_merge_type(splitted_merge, repo) + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + # compute depth only for merges with branchname + min_depth = process_depth( + splitted_merge, + branchname, + main_branch, + main_branch_name, + repo_path, + ) + repo_min_depth[repo] = ( + min_depth > repo_min_depth[repo] + and min_depth + or repo_min_depth[repo] + ) + if repo_min_depth[repo] > 0: + waft_depth = decode_variables( + doc[repo]["defaults"].get("depth") + ) + # just in case the substitution didn't happen because variables + # are not explicitly loaded in env... + if waft_depth == "${WAFT_DEPTH_MERGE}": + # waft_depth_merge, if not specified in env defaults to 100 + waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 + if waft_depth == "${WAFT_DEPTH_DEFAULT}": + waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 + waft_depth = int(waft_depth) + if repo_min_depth[repo] > waft_depth: + changes += ("\n\t Increasing depth of %s from %s to %s" + % ( + repo, + doc[repo]["defaults"]["depth"], + str(repo_min_depth[repo]), + ) + ) + doc[repo]["defaults"]["depth"] = repo_min_depth[repo] + + CustomDumper.add_representer( + dict, CustomDumper.represent_dict_preserve_order + ) + + if changes: + print("========Applying Depth changes to repos.yaml:") + print(changes) + print("=======================================") + yaml_file = open(REPOS_YAML, "w") + yaml_file.write(yaml.dump(doc, Dumper=CustomDumper)) + yaml_file.close() + +if os.path.isfile(REPOS_YAML) and __name__ == "__main__": + main() +else: + logger.debug("no %s repository file found" % REPOS_YAML) + raise ValueError diff --git a/build b/build index 5355006c..734f9a56 100755 --- a/build +++ b/build @@ -28,7 +28,8 @@ fi pip install --quiet "setuptools<58" && \ pip install --quiet "pip>=20.3" && \ (pip uninstall --yes -r "${ODOO_WORK_DIR}/requirements-remove-default.txt" 2>/dev/null || /bin/true) && \ -pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements-default.txt" && \ +pip install --exists-action w -r "${ODOO_WORK_DIR}/requirements-default.txt" && \ pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements.txt" && \ +"${ODOO_WORK_DIR}/update_depths" && \ "${ODOO_WORK_DIR}/common/build" && \ python "${ODOO_WORK_DIR}/common/entrypoint" diff --git a/update_depths b/update_depths new file mode 100755 index 00000000..2d60db56 --- /dev/null +++ b/update_depths @@ -0,0 +1,15 @@ +#!/bin/sh + +SCRIPT_PATH="$(cd "$(/usr/bin/dirname "${0}")" && /bin/pwd)" +ODOO_WORK_DIR="${SCRIPT_PATH}" +. "${ODOO_WORK_DIR}/.env-default" ; \ +. "${ODOO_WORK_DIR}/.env-shared" ; \ +. "${ODOO_WORK_DIR}/.env-secret"; cd "${ODOO_WORK_DIR}" + +if [ "$(id -u)" -eq 0 ]; then + /bin/echo "Please dont use sudo command with script" + exit 1 +fi +echo "updating pin depth in repos.yaml" +. .venv/bin/activate && python bin/update_depths.py $@ + From 4344c8659c6521ea991610b0fff803164f4c7272 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Wed, 13 Sep 2023 15:58:40 +0200 Subject: [PATCH 3/6] [IMP] #9787 Update depth for new branches Also do not break on brand new remotes. --- bin/update_depths.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/bin/update_depths.py b/bin/update_depths.py index ec7d0d3a..94654e75 100755 --- a/bin/update_depths.py +++ b/bin/update_depths.py @@ -113,6 +113,18 @@ def get_merge_type(splitted_merge, repo): def process_depth(splitted_merge, branchname, main_branch, main_branch_name, repo_path): os.chdir(repo_path) + # make sure we have the latest branch available. + run( + [ + "git", + "fetch", + splitted_merge[0], + branchname, + ], + stdout=PIPE, + stderr=PIPE, + ) + # look at most recent common commit. lastrev = ( run( [ @@ -127,6 +139,8 @@ def process_depth(splitted_merge, branchname, main_branch, main_branch_name, rep .stdout.decode("utf-8") .replace("\n", "") ) + if not lastrev: + return 1024 # Can happen when remote not yet added. # we now calculate the needed depth of this branch mindepth = ( run( @@ -143,7 +157,11 @@ def process_depth(splitted_merge, branchname, main_branch, main_branch_name, rep .stdout.decode("utf-8") .replace("\n", "") ) - return int(mindepth) + try: + return int(mindepth) + except Exception: + # Should log/print some error here. + return 1024 def main(): @@ -166,8 +184,8 @@ def main(): main_branch_name = get_branchname(main_branch, merge_type) for merge in doc[repo]["merges"]: repo_path, splitted_merge = preprocess_merge(doc, repo, merge) - # this script cannot work on new ./builds it is written to keep - # depths of instances that have been built at least once with + # this script cannot work on new ./builds it is written to keep + # depths of instances that have been built at least once with # if one source folder is missing we skip it. if not os.path.exists(repo_path): continue @@ -218,7 +236,7 @@ def main(): print(changes) print("=======================================") yaml_file = open(REPOS_YAML, "w") - yaml_file.write(yaml.dump(doc, Dumper=CustomDumper)) + yaml_file.write(yaml.dump(doc, Dumper=CustomDumper)) yaml_file.close() if os.path.isfile(REPOS_YAML) and __name__ == "__main__": From 9840c5aedc3df04cc20c36b18d30196a722bd751 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Mon, 25 Sep 2023 15:41:43 +0200 Subject: [PATCH 4/6] [IMP] customize dumper to add blank lines to top-level --- bin/update_depths.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/update_depths.py b/bin/update_depths.py index 94654e75..159c2282 100755 --- a/bin/update_depths.py +++ b/bin/update_depths.py @@ -50,6 +50,13 @@ class CustomDumper(yaml.Dumper): def represent_dict_preserve_order(self, data): return self.represent_dict(data.items()) + def write_line_break(self, data=None): + # we override this dumper method to also insert + # a blank line for all top level lines. + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + def split_line(line): splitted_lines = line.split(" ") From 7b173c3ae7eaca116ad74d59fd6079a2f6983271 Mon Sep 17 00:00:00 2001 From: Giovanni Francesco Capalbo Date: Thu, 28 Sep 2023 17:42:26 +0200 Subject: [PATCH 5/6] [ADD] restore quiet switch --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 734f9a56..413cee64 100755 --- a/build +++ b/build @@ -28,7 +28,7 @@ fi pip install --quiet "setuptools<58" && \ pip install --quiet "pip>=20.3" && \ (pip uninstall --yes -r "${ODOO_WORK_DIR}/requirements-remove-default.txt" 2>/dev/null || /bin/true) && \ -pip install --exists-action w -r "${ODOO_WORK_DIR}/requirements-default.txt" && \ +pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements-default.txt" && \ pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements.txt" && \ "${ODOO_WORK_DIR}/update_depths" && \ "${ODOO_WORK_DIR}/common/build" && \ From aaa65aa96ccf7615fca986fbcce73fea68cd4961 Mon Sep 17 00:00:00 2001 From: Hussam Suleiman <38970953+Hussam-Suleiman@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:32:37 +0300 Subject: [PATCH 6/6] Remove unused arguments. --- update_depths | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_depths b/update_depths index 2d60db56..daa413d3 100755 --- a/update_depths +++ b/update_depths @@ -11,5 +11,5 @@ if [ "$(id -u)" -eq 0 ]; then exit 1 fi echo "updating pin depth in repos.yaml" -. .venv/bin/activate && python bin/update_depths.py $@ +. .venv/bin/activate && python bin/update_depths.py