diff --git a/bin/update_depths.py b/bin/update_depths.py new file mode 100755 index 00000000..159c2282 --- /dev/null +++ b/bin/update_depths.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# Version: v.21.05.30 +import os +import argparse +from string import Template +from subprocess import run, PIPE +from dotenv import load_dotenv + +import yaml +from waftlib import ( + PRIVATE, + REPOS_YAML, + SRC_DIR, + logger, + ODOO_VERSION, +) + +SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +os.environ["ODOO_WORK_DIR"] = os.path.realpath(os.path.join(SCRIPT_PATH, "../..")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-default")) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-shared"), override=True) +load_dotenv(os.path.join(os.environ["ODOO_WORK_DIR"], ".env-secret"), override=True) + + +parser = argparse.ArgumentParser( + prog="get_pins", + description="Fetch current pins of this instance", + epilog="Outputs pinned repo.yaml", +) + +parser.add_argument("--only-base") +args = parser.parse_args() + + +all_commits = {} +repo_min_depth = {} + + +class CustomDumper(yaml.Dumper): + """ + We want order of keys intact so that the output is the same order than source. + Unfortunately PyYAML orders yaml output alphabetically on dump. + pyyaml 5.1 has an option to disable alphabetical orderng of dumps, but often our + wafts have a version < 5.1. + for this reason i made a custom dumper to have an "unordered" dump, without + affecting the default behaviour of pyyaml dump. This script supports pyYAML<5.1. + Our output will therefore have the same order as given input. + """ + + def represent_dict_preserve_order(self, data): + return self.represent_dict(data.items()) + + def write_line_break(self, data=None): + # we override this dumper method to also insert + # a blank line for all top level lines. + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + +def split_line(line): + splitted_lines = line.split(" ") + for sl in splitted_lines: + if sl == "": + splitted_lines.pop(splitted_lines.index(sl)) + return splitted_lines + + +def is_in_history(value, history): + for commit in history: + if commit[:8] == value[:8]: + return True + return False + + +def decode_variables(string): + """ + pass a string and return variables replaced in it. + @returns string + """ + # may be int, str + string = str(string) + yaml_substituted = Template(string) + return yaml_substituted.substitute(os.environ) + + +def preprocess_merge(doc, repo, merge): + remotes = doc[repo]["remotes"].keys() + splitted_merge = [x for x in merge.split(" ") if x != ""] + if splitted_merge[0] not in remotes: + logger.debug("Invalid Remote on line: %s" % merge) + raise ValueError + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + return repo_path, splitted_merge + + +def get_branchname(splitted_merge, merge_type): + if merge_type in (1, 3): + return decode_variables(splitted_merge[1]) + return False + +def get_merge_type(splitted_merge, repo): + """ + Possible syntaxes for merges: + + merge_type=1 + merge_type=2 + merge_type=3 + merge_type=0 "invalid" + """ + if len(splitted_merge) == 3: + return 3 + else: + if is_in_history(splitted_merge[1], all_commits.get(repo, [])): + return 2 + else: + return 1 + return 0 # unreachable. + + +def process_depth(splitted_merge, branchname, main_branch, main_branch_name, repo_path): + os.chdir(repo_path) + # make sure we have the latest branch available. + run( + [ + "git", + "fetch", + splitted_merge[0], + branchname, + ], + stdout=PIPE, + stderr=PIPE, + ) + # look at most recent common commit. + lastrev = ( + run( + [ + "git", + "merge-base", + "".join([main_branch[0], "/", main_branch_name]), + "".join([splitted_merge[0], "/", branchname]), + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + if not lastrev: + return 1024 # Can happen when remote not yet added. + # we now calculate the needed depth of this branch + mindepth = ( + run( + [ + "git", + "rev-list", + "".join([main_branch[0], "/", main_branch_name]), + "^" + lastrev, + "--count", + ], + stdout=PIPE, + stderr=PIPE, + ) + .stdout.decode("utf-8") + .replace("\n", "") + ) + try: + return int(mindepth) + except Exception: + # Should log/print some error here. + return 1024 + + +def main(): + """ + parsing directly repos.yaml, if something is not in addons.yaml, branch will still + be in folder, but may not be included in addons. Nothing changes. + """ + changes = '' + with open(REPOS_YAML) as yaml_file: + for doc in yaml.safe_load_all(yaml_file): + for repo in doc: + print("===>processing repo %s" % repo) + repo_min_depth[repo] = 0 + if repo in {PRIVATE, "ONLY", "ENV"}: + continue + target = doc[repo].get("target") or False + # main branch is defined as target or in absence of target, merge[0] + main_branch = split_line(target or doc[repo]["merges"][0]) + merge_type = get_merge_type(main_branch, repo) + main_branch_name = get_branchname(main_branch, merge_type) + for merge in doc[repo]["merges"]: + repo_path, splitted_merge = preprocess_merge(doc, repo, merge) + # this script cannot work on new ./builds it is written to keep + # depths of instances that have been built at least once with + # if one source folder is missing we skip it. + if not os.path.exists(repo_path): + continue + merge_type = get_merge_type(splitted_merge, repo) + branchname = get_branchname(splitted_merge, merge_type) + if branchname: + # compute depth only for merges with branchname + min_depth = process_depth( + splitted_merge, + branchname, + main_branch, + main_branch_name, + repo_path, + ) + repo_min_depth[repo] = ( + min_depth > repo_min_depth[repo] + and min_depth + or repo_min_depth[repo] + ) + if repo_min_depth[repo] > 0: + waft_depth = decode_variables( + doc[repo]["defaults"].get("depth") + ) + # just in case the substitution didn't happen because variables + # are not explicitly loaded in env... + if waft_depth == "${WAFT_DEPTH_MERGE}": + # waft_depth_merge, if not specified in env defaults to 100 + waft_depth = os.environ.get("WAFT_DEPTH_MERGE") or 100 + if waft_depth == "${WAFT_DEPTH_DEFAULT}": + waft_depth = os.environ.get("WAFT_DEPTH_DEFAULT") or 1 + waft_depth = int(waft_depth) + if repo_min_depth[repo] > waft_depth: + changes += ("\n\t Increasing depth of %s from %s to %s" + % ( + repo, + doc[repo]["defaults"]["depth"], + str(repo_min_depth[repo]), + ) + ) + doc[repo]["defaults"]["depth"] = repo_min_depth[repo] + + CustomDumper.add_representer( + dict, CustomDumper.represent_dict_preserve_order + ) + + if changes: + print("========Applying Depth changes to repos.yaml:") + print(changes) + print("=======================================") + yaml_file = open(REPOS_YAML, "w") + yaml_file.write(yaml.dump(doc, Dumper=CustomDumper)) + yaml_file.close() + +if os.path.isfile(REPOS_YAML) and __name__ == "__main__": + main() +else: + logger.debug("no %s repository file found" % REPOS_YAML) + raise ValueError diff --git a/build b/build index 5355006c..413cee64 100755 --- a/build +++ b/build @@ -30,5 +30,6 @@ pip install --quiet "pip>=20.3" && \ (pip uninstall --yes -r "${ODOO_WORK_DIR}/requirements-remove-default.txt" 2>/dev/null || /bin/true) && \ pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements-default.txt" && \ pip install --quiet --exists-action w -r "${ODOO_WORK_DIR}/requirements.txt" && \ +"${ODOO_WORK_DIR}/update_depths" && \ "${ODOO_WORK_DIR}/common/build" && \ python "${ODOO_WORK_DIR}/common/entrypoint" diff --git a/update_depths b/update_depths new file mode 100755 index 00000000..daa413d3 --- /dev/null +++ b/update_depths @@ -0,0 +1,15 @@ +#!/bin/sh + +SCRIPT_PATH="$(cd "$(/usr/bin/dirname "${0}")" && /bin/pwd)" +ODOO_WORK_DIR="${SCRIPT_PATH}" +. "${ODOO_WORK_DIR}/.env-default" ; \ +. "${ODOO_WORK_DIR}/.env-shared" ; \ +. "${ODOO_WORK_DIR}/.env-secret"; cd "${ODOO_WORK_DIR}" + +if [ "$(id -u)" -eq 0 ]; then + /bin/echo "Please dont use sudo command with script" + exit 1 +fi +echo "updating pin depth in repos.yaml" +. .venv/bin/activate && python bin/update_depths.py +