From 9891c18ca404e8c80850c1381c7817f82dc4e61f Mon Sep 17 00:00:00 2001 From: fweikert Date: Fri, 11 Jan 2019 19:31:49 +0100 Subject: [PATCH 1/3] fweikert changes --- buildkite/README.md | 37 +++++++++++++- buildkite/bazelci.py | 112 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 135 insertions(+), 14 deletions(-) diff --git a/buildkite/README.md b/buildkite/README.md index 8317ee1920..43b713d76c 100644 --- a/buildkite/README.md +++ b/buildkite/README.md @@ -114,4 +114,39 @@ GOOD_BAZEL_COMMIT=b6ea3b6caa7f379778e74da33d1bd0ff6477f963 BAD_BAZEL_COMMIT=91eb3d207714af0ab1e5812252a0f10f40d6e4a8 ``` -Note: Bazel commit can only be set to commits after [63453bdbc6b05bd201375ee9e25b35010ae88aab](https://github.com/bazelbuild/bazel/commit/63453bdbc6b05bd201375ee9e25b35010ae88aab), Culprit Finder needs to download Bazel at specific commit, but we didn't prebuilt Bazel binaries before this commit. \ No newline at end of file +Note: Bazel commit can only be set to commits after [63453bdbc6b05bd201375ee9e25b35010ae88aab](https://github.com/bazelbuild/bazel/commit/63453bdbc6b05bd201375ee9e25b35010ae88aab), Culprit Finder needs to download Bazel at specific commit, but we didn't prebuilt Bazel binaries before this commit. + +## Running Buildifier on CI + +For each pipeline you can enable [Buildifier](https://github.com/bazelbuild/buildtools/tree/master/buildifier) to check whether all BUILD and .bzl files comply with the standard formatting convention. Simply add the following code to the top of the particular pipeline Yaml configuration (either locally in `.bazelci/presubmit.yml` or in https://github.com/bazelbuild/continuous-integration/tree/master/buildkite/pipelines): + +``` +buildifier: {} +``` + +As a consequence, every future build for this pipeline will contain an additional "Buildifier" step that runs the latest version of Buildifier on all BUILD and .bzl files in "lint" mode. + +### Running a specific version of Buildifier + +You can also specify which version of Buildifier should be run: + +``` +buildifier: + version: latest +``` + +The configuration value can be a hardcoded version string such as "0.20.0" or a dynamic reference such as "latest", "latest-1", etc. The latest Buildifier version will be used if the configuration does not contain the "version" field. + +### Specifying the set of input files + +By default Buildifier only processes BUILD and .bzl files. This can be changed by adding the "files" field to the configuration: + +``` +buildifier: + files: + - "*.bzl" + - "BUILD.bazel" + - "BUILD" +``` + +All values must be valid input for [find(1)](https://linux.die.net/man/1/find). diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index f871e682bd..88c36b26c3 100644 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -18,6 +18,7 @@ import base64 import codecs import datetime +from distutils.version import LooseVersion import hashlib import json import multiprocessing @@ -36,6 +37,7 @@ import uuid import yaml from urllib.request import url2pathname +from urllib.request import urlopen from urllib.parse import urlparse # Initialize the random number generator. @@ -309,6 +311,16 @@ }, } +LATEST_VERSION_PATTERN = re.compile(r"latest(-(?P\d+))?$") + +BUILDIFIER_RELEASE_PAGE = "https://api.github.com/repos/bazelbuild/buildtools/releases" + +BUILDIFIER_DEFAULT_VERSION = "latest" + +BUILDIFIER_DEFAULT_INPUT_FILES = ["BUILD", "*.bzl"] + +BUILDIFIER_DEFAULT_DOCKER_IMAGE = "gcr.io/bazel-untrusted/ubuntu1804:nojava" + # The platform used for various steps (e.g. stuff that formerly ran on the "pipeline" workers). DEFAULT_PLATFORM = "ubuntu1804" @@ -1052,7 +1064,21 @@ def execute_command_background(args): def create_step(label, commands, platform=DEFAULT_PLATFORM): host_platform = PLATFORMS[platform].get("host-platform", platform) if "docker-image" in PLATFORMS[platform]: + return create_docker_step(label, commands, PLATFORMS[platform]["docker-image"]) + else: return { + "label": label, + "command": commands, + "agents": { + "kind": "worker", + "java": PLATFORMS[platform]["java"], + "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"), + }, + } + + +def create_docker_step(label, commands, docker_image): + return { "label": label, "command": commands, "agents": {"kind": "docker", "os": "linux"}, @@ -1061,7 +1087,7 @@ def create_step(label, commands, platform=DEFAULT_PLATFORM): "always-pull": True, "debug": True, "environment": ["BUILDKITE_ARTIFACT_UPLOAD_DESTINATION", "BUILDKITE_GS_ACL"], - "image": PLATFORMS[platform]["docker-image"], + "image": docker_image, "privileged": True, "propagate-environment": True, "tmpfs": ["/home/bazel/.cache:exec,uid=999,gid=999"], @@ -1074,20 +1100,10 @@ def create_step(label, commands, platform=DEFAULT_PLATFORM): } }, } - else: - return { - "label": label, - "command": commands, - "agents": { - "kind": "worker", - "java": PLATFORMS[platform]["java"], - "os": rchop(host_platform, "_nojava", "_java8", "_java9", "_java10"), - }, - } def print_project_pipeline( - platform_configs, + configs, project_name, http_config, file_config, @@ -1096,6 +1112,7 @@ def print_project_pipeline( use_but, incompatible_flags, ): + platform_configs = configs.get("platforms", None) if not platform_configs: raise BuildkiteException("{0} pipeline configuration is empty.".format(project_name)) @@ -1122,6 +1139,10 @@ def print_project_pipeline( ) pipeline_steps.append(step) + buildifier_step = get_buildifier_step_if_requested(configs) + if buildifier_step: + pipeline_steps.append(buildifier_step) + pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG") all_downstream_pipeline_slugs = [] for _, config in DOWNSTREAM_PROJECTS.items(): @@ -1198,6 +1219,71 @@ def fetch_incompatible_flag_verbose_failures_command(): ) +def get_buildifier_step_if_requested(configs): + buildifier = configs.get("buildifier") + # An empty dictionary is allowed. + if buildifier is None: + return None + + version_from_config = buildifier.get("version", BUILDIFIER_DEFAULT_VERSION) + version, url = get_buildifier_version_and_url(version_from_config) + files = buildifier.get("files", BUILDIFIER_DEFAULT_INPUT_FILES) + return create_buildifier_step(version, url, files) + + +def get_buildifier_version_and_url(version_from_config): + releases = get_all_buildifier_releases() + + requested_release = None + if "latest" in version_from_config: + requested_release = get_latest_buildifier_release(releases.values(), version_from_config) + else: + requested_release = releases.get(version_from_config) + if not requested_release: + raise BuildkiteException("There is no Buildifier version '{}'.".format(version_from_config)) + + urls = [a["browser_download_url"] for a in requested_release["assets"] if a["name"] == "buildifier"] + if not urls: + raise BuildkiteException("There is no download URL for Buildifier release '{}'.".format(version_from_config)) + + return requested_release["tag_name"], urls[0] + + +def get_all_buildifier_releases(): + res = urlopen("{}?{}".format(BUILDIFIER_RELEASE_PAGE, int(time.time()))).read() + return {r["tag_name"] : r for r in json.loads(res.decode('utf-8')) if not r['prerelease']} + + +def get_latest_buildifier_release(releases, version_from_config): + match = LATEST_VERSION_PATTERN.match(version_from_config) + if not match: + raise BuildkiteException("Invalid version '{}'. In addition to using a version " + "number such as '0.20.0', you can use values such as " + "latest' and 'latest-N', with N being a non-negative " + "integer.".format(version_from_config)) + + offset = int(match.group("offset") or "0") + sorted_releases = sorted(releases, reverse=True, key= lambda r: LooseVersion(r["tag_name"])) + if offset >= len(sorted_releases): + version = "latest-{}".format(offset) if offset else "latest" + raise BuildkiteException("Cannot resolve version '{}': There are only {} Buildifier " + "releases.".format(version, len(sorted_releases))) + + return sorted_releases[offset] + + +def create_buildifier_step(version, url, files, os="ubuntu1604"): + commands = [ "curl -L {} -o buildifier".format(url), + "chmod +x buildifier", + create_buildifier_command(files) ] + return create_docker_step("Buildifier {}".format(version), commands, BUILDIFIER_DEFAULT_DOCKER_IMAGE) + + +def create_buildifier_command(files_to_lint): + find_args = " -or ".join('-iname "{}"'.format(f) for f in files_to_lint) + return "./buildifier --lint=warn $$(find . -type f \\( {} \\)) | grep -q \".\"".format(find_args) + + def upload_project_pipeline_step( project_name, git_repository, http_config, file_config, incompatible_flags ): @@ -1745,7 +1831,7 @@ def main(argv=None): elif args.subparsers_name == "project_pipeline": configs = fetch_configs(args.http_config, args.file_config) print_project_pipeline( - platform_configs=configs.get("platforms", None), + configs=configs, project_name=args.project_name, http_config=args.http_config, file_config=args.file_config, From cf3cb8adacada8fb8c693fae0e99605fc232ccf6 Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Fri, 11 Jan 2019 20:42:23 +0100 Subject: [PATCH 2/3] Execute shell commands before running Bazel --- buildkite/bazelci.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index 88c36b26c3..efe52e0bcc 100644 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -471,6 +471,11 @@ def execute_commands( else: git_repository = os.getenv("BUILDKITE_REPO") + if platform == "windows": + execute_batch_commands(config.get("batch_commands", None)) + else: + execute_shell_commands(config.get("shell_commands", None)) + if use_bazel_at_commit: print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit) bazel_binary = download_bazel_binary_at_commit(tmpdir, platform, use_bazel_at_commit) @@ -489,10 +494,6 @@ def execute_commands( for flag in incompatible_flags: eprint(flag + "\n") - if platform == "windows": - execute_batch_commands(config.get("batch_commands", None)) - else: - execute_shell_commands(config.get("shell_commands", None)) execute_bazel_run( bazel_binary, platform, config.get("run_targets", None), incompatible_flags ) From 85258f8caabddc2a563c0ca23f6f495eac9190d1 Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Fri, 11 Jan 2019 22:12:04 +0100 Subject: [PATCH 3/3] Fetch correct version of bazelci.py --- buildkite/bazelci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py index efe52e0bcc..7b0f56dcb7 100644 --- a/buildkite/bazelci.py +++ b/buildkite/bazelci.py @@ -386,7 +386,7 @@ def bazelcipy_url(): """ URL to the latest version of this script. """ - return "https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?{}".format( + return "https://raw.githubusercontent.com/fweikert/continuous-integration/nested2/buildkite/bazelci.py?{}".format( int(time.time()) )