diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java index 2c5bc990972059..475610fe86efc4 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java @@ -301,6 +301,9 @@ public enum WorkerProtocolFormat { public static final String DIFFERENTIATE_WORKSPACE_CACHE = "internal-differentiate-workspace-cache"; + /** Disables cgroups for a spawn */ + public static final String NO_SUPPORTS_CGROUPS = "no-supports-cgroups"; + /** * Indicates that the action is compatible with path mapping, e.g., removing the configuration * segment from the paths of all inputs and outputs. diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD index ab8aa8a0993a12..a88194f809117d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/BUILD @@ -1691,6 +1691,7 @@ java_library( ":config/run_under", ":config/starlark_defined_config_transition", ":platform_options", + ":test/test_configuration", "//src/main/java/com/google/devtools/build/lib/actions:action_environment", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/actions:build_configuration_event", diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationValue.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationValue.java index f54933eaafb5d2..d9e0e379a405ee 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationValue.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationValue.java @@ -41,6 +41,7 @@ import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.starlarkbuildapi.BuildConfigurationApi; import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.RegexFilter; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyValue; @@ -988,4 +989,16 @@ public static BuildEvent buildEvent(@Nullable BuildConfigurationValue configurat public ImmutableSet getReservedActionMnemonics() { return reservedActionMnemonics; } + + public Map getTestResources(com.google.devtools.build.lib.packages.TestSize size) { + if (!buildOptions.contains(com.google.devtools.build.lib.analysis.test.TestConfiguration.TestOptions.class)) { + return ImmutableMap.of(); + } + return buildOptions + .get(com.google.devtools.build.lib.analysis.test.TestConfiguration.TestOptions.class) + .testResources + .stream() + .collect(ImmutableMap.toImmutableMap(e -> e.getFirst(), e -> e.getSecond().get(size))); + } + } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java index 3a9e9a348bc4dd..02c646dd6bfabe 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java @@ -92,6 +92,21 @@ private static ResourceSet getResourceSetFromSize(TestSize size) { Map executionInfo = Maps.newLinkedHashMap(); executionInfo.putAll(TargetUtils.getExecutionInfo(rule)); + Map requestedResources; + try { + requestedResources = parseTags(ruleContext.getLabel(), executionInfo); + } catch (UserExecException e) { + requestedResources = new HashMap<>(); + } + + Map testResources = ruleContext.getConfiguration().getTestResources(size); + for (Map.Entry request: testResources.entrySet()) { + if (requestedResources.containsKey(request.getKey())) { + continue; + } + executionInfo.put(String.format("resources:%s:%f", request.getKey(), request.getValue()), ""); + } + boolean incompatibleExclusiveTestSandboxed = false; testConfiguration = ruleContext.getFragment(TestConfiguration.class); diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD index 6373e59a9342aa..c00328559326fc 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD +++ b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD @@ -38,6 +38,7 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity", "//src/main/java/com/google/devtools/build/lib/util", + "//src/main/java/com/google/devtools/build/lib/util:cpu_resource_converter", "//src/main/java/com/google/devtools/build/lib/util:ram_resource_converter", "//src/main/java/com/google/devtools/build/lib/util:resource_converter", "//src/main/java/com/google/devtools/build/lib/vfs", @@ -212,6 +213,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/actions:exec_exception", "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements", "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", @@ -223,11 +225,14 @@ java_library( "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/exec/local:options", "//src/main/java/com/google/devtools/build/lib/profiler", + "//src/main/java/com/google/devtools/build/lib/sandbox/cgroups", "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/util:pair", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/protobuf:failure_details_proto", + "//src/main/protobuf:failure_details_java_proto", "//third_party:flogger", "//third_party:guava", "//third_party:jsr305", diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilder.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilder.java index ac9d0ddcd9f688..cb6fcd1af887d7 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilder.java @@ -55,7 +55,7 @@ public class LinuxSandboxCommandLineBuilder { private boolean enablePseudoterminal = false; private String sandboxDebugPath = null; private boolean sigintSendsSigterm = false; - private String cgroupsDir; + private Set cgroupsDirs = ImmutableSet.of(); private LinuxSandboxCommandLineBuilder(Path linuxSandboxPath) { this.linuxSandboxPath = linuxSandboxPath; @@ -215,8 +215,8 @@ public LinuxSandboxCommandLineBuilder setSandboxDebugPath(String sandboxDebugPat * this directory, its parent directory, and the cgroup directory for the Bazel process. */ @CanIgnoreReturnValue - public LinuxSandboxCommandLineBuilder setCgroupsDir(String cgroupsDir) { - this.cgroupsDir = cgroupsDir; + public LinuxSandboxCommandLineBuilder setCgroupsDirs(Set cgroupsDirs) { + this.cgroupsDirs = cgroupsDirs; return this; } @@ -299,8 +299,8 @@ public ImmutableList buildForCommand(List commandArguments) { if (persistentProcess) { commandLineBuilder.add("-p"); } - if (cgroupsDir != null) { - commandLineBuilder.add("-C", cgroupsDir); + for (Path dir: cgroupsDirs) { + commandLineBuilder.add("-C", dir.toString()); } commandLineBuilder.add("--"); commandLineBuilder.addAll(commandArguments); diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java index 4f96f9c0317a8d..1850b09d4693c4 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java @@ -45,6 +45,8 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs; import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs; +import com.google.devtools.build.lib.sandbox.cgroups.VirtualCGroup; +import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.util.OS; @@ -59,6 +61,7 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.SortedMap; import java.util.Set; import java.util.TreeMap; @@ -74,6 +77,8 @@ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner { private static final AtomicBoolean warnedAboutUnsupportedModificationCheck = new AtomicBoolean(); + private java.util.concurrent.ConcurrentHashMap> cgroups; + /** * Returns whether the linux sandbox is supported on the local machine by running a small command * in it. @@ -166,10 +171,98 @@ private static boolean computeIsSupported(CommandEnvironment cmdEnv, Path linuxS this.localEnvProvider = new PosixLocalEnvProvider(cmdEnv.getClientEnv()); this.treeDeleter = treeDeleter; this.reporter = cmdEnv.getReporter(); + this.cgroups = new java.util.concurrent.ConcurrentHashMap<>(); this.slashTmp = cmdEnv.getRuntime().getFileSystem().getPath("/tmp"); this.knownPathsToMountUnderHermeticTmp = collectPathsToMountUnderHermeticTmp(cmdEnv); } + private Optional getCgroup(Spawn spawn, SpawnExecutionContext context) throws ExecException, IOException { + if (spawn.getExecutionInfo().get(ExecutionRequirements.NO_SUPPORTS_CGROUPS) != null) { + return Optional.empty(); + } + if (cgroups.containsKey(context.getId())) { + return cgroups.get(context.getId()); + } + + SandboxOptions sandboxOptions = getSandboxOptions(); + + VirtualCGroup cgroup = null; + long memoryLimit = sandboxOptions.memoryLimitMb * 1024L * 1024L; + float cpuLimit = sandboxOptions.cpuLimit; + + if (sandboxOptions.executionInfoLimit) { + ExecutionRequirements.ParseableRequirement requirement = ExecutionRequirements.RESOURCES; + for (String tag : spawn.getExecutionInfo().keySet()) { + try { + requirement = ExecutionRequirements.RESOURCES; + String name = null; + Float value = null; + + String extras = requirement.parseIfMatches(tag); + if (extras != null) { + int index = extras.indexOf(":"); + name = extras.substring(0, index); + value = Float.parseFloat(extras.substring(index + 1)); + } else { + requirement = ExecutionRequirements.CPU; + String cpus = requirement.parseIfMatches(tag); + if (cpus != null) { + name = "cpu"; + value = Float.parseFloat(cpus); + } + } + if (name == null) { + continue; + } + switch (name) { + case "memory": + memoryLimit = Math.round(value * 1024.0 * 1024.0); + break; + case "cpu": + cpuLimit = value; + break; + } + } catch (ExecutionRequirements.ParseableRequirement.ValidationException e) { + String message = + String.format( + "%s has a '%s' tag, but its value '%s' didn't pass validation: %s", + spawn.getTargetLabel(), + requirement.userFriendlyName(), + e.getTagValue(), + e.getMessage()); + FailureDetails.Spawn.Code code = FailureDetails.Spawn.Code.COMMAND_LINE_EXPANSION_FAILURE; + FailureDetails.FailureDetail details = FailureDetails.FailureDetail + .newBuilder() + .setMessage(message) + .setSpawn(FailureDetails.Spawn.newBuilder().setCode(code)) + .build(); + throw new UserExecException(e, details); + } + } + } + + // We put the sandbox inside a unique subdirectory using the context's ID. This ID is + // unique per spawn run by this spawn runner. + String scope = "sandbox_" + context.getId() + ".scope"; + if (memoryLimit > 0) { + if (cgroup == null) { + cgroup = VirtualCGroup.getInstance(this.reporter).child(scope); + } + cgroup.memory().setMaxBytes(memoryLimit); + } + + if (cpuLimit > 0) { + if (cgroup == null) { + cgroup = VirtualCGroup.getInstance(this.reporter).child(scope); + } + cgroup.cpu().setCpus(cpuLimit); + } + + cgroups.put(context.getId(), Optional.ofNullable(cgroup)); + + return cgroups.get(context.getId()); + } + private ImmutableSet collectPathsToMountUnderHermeticTmp(CommandEnvironment cmdEnv) { // If any path managed or tracked by Bazel is under /tmp, it needs to be explicitly mounted // into the sandbox when using hermetic /tmp. We attempt to collect an over-approximation of @@ -314,14 +407,12 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context commandLineBuilder.setSandboxDebugPath(sandboxDebugPath.getPathString()); } - if (sandboxOptions.memoryLimitMb > 0) { - CgroupsInfo cgroupsInfo = CgroupsInfo.getInstance(); - // We put the sandbox inside a unique subdirectory using the context's ID. This ID is - // unique per spawn run by this spawn runner. - cgroupsDir = - cgroupsInfo.createMemoryLimitCgroupDir( - "sandbox_" + context.getId(), sandboxOptions.memoryLimitMb); - commandLineBuilder.setCgroupsDir(cgroupsDir); + Optional cgroup = getCgroup(spawn, context); + if (cgroup.isPresent()) { + commandLineBuilder.setCgroupsDirs( + cgroup.get().paths().stream() + .map(p -> fileSystem.getPath(p.toString())) + .collect(ImmutableSet.toImmutableSet())); } if (!timeout.isZero()) { @@ -447,6 +538,13 @@ public void verifyPostCondition( if (getSandboxOptions().useHermetic) { checkForConcurrentModifications(context); } + Optional cgroup = cgroups.remove(context.getId()); + if (cgroup != null && cgroup.isPresent()) { + // We cannot leave the cgroups around and delete them only when we delete the sandboxes + // because linux has a hard limit of 65535 memory controllers. + // Ref. https://github.com/torvalds/linux/blob/58d4e450a490d5f02183f6834c12550ba26d3b47/include/linux/memcontrol.h#L69 + cgroup.get().delete(); + } } private void checkForConcurrentModifications(SpawnExecutionContext context) @@ -511,9 +609,7 @@ private boolean wasModifiedSinceDigest(FileContentsProxy proxy, Path path) throw @Override public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws IOException { - if (cgroupsDir != null) { - new File(cgroupsDir).delete(); - } + VirtualCGroup.deleteInstance(); // Delete the inaccessible files synchronously, bypassing the treeDeleter. They are only a // couple of files that can be deleted fast, and ensuring they are gone at the end of every // build avoids annoying permission denied errors if the user happens to run "rm -rf" on the diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java index 6a23863c87dd60..f5070b2d830bb4 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.devtools.build.lib.util.CpuResourceConverter; import com.google.devtools.build.lib.util.OptionsUtils; import com.google.devtools.build.lib.util.RamResourceConverter; import com.google.devtools.build.lib.util.ResourceConverter; @@ -381,6 +382,29 @@ public ImmutableSet getInaccessiblePaths(FileSystem fs) { + " Requires cgroups v1 or v2 and permissions for the users to the cgroups dir.") public int memoryLimitMb; + @Option( + name = "experimental_sandbox_cpu_limit", + defaultValue = "0", + documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + effectTags = {OptionEffectTag.EXECUTION}, + converter = CpuResourceConverter.class, + help = + "If > 0, each Linux sandbox will be limited to the given amount of cpus." + + " Requires cgroups v1 or v2 and permissions for the users to the cgroups dir.") + public float cpuLimit; + + @Option( + name = "experimental_sandbox_execution_info_limit", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "If true, resources declared in the execution info that match a cgroup controller" + + " will be used to apply the limits. For example a target that declares" + + " cpu:3 and resources:memory:10, will run with at most 3 cpus and 10" + + " megabytes of memory.") + public boolean executionInfoLimit; + /** Converter for the number of threads used for asynchronous tree deletion. */ public static final class AsyncTreeDeletesConverter extends ResourceConverter.IntegerConverter { public AsyncTreeDeletesConverter() { diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/BUILD b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/BUILD new file mode 100644 index 00000000000000..aec4f8b667265c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/BUILD @@ -0,0 +1,30 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "cgroups", + srcs = glob([ + "*.java", + "v1/*.java", + "v2/*.java", + ]), + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions:exec_exception", + "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/protobuf:failure_details_java_proto", + "//third_party:auto_value", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Controller.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Controller.java new file mode 100644 index 00000000000000..23aa9ed2151e87 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Controller.java @@ -0,0 +1,44 @@ +package com.google.devtools.build.lib.sandbox.cgroups; + +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.server.FailureDetails; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Path; + +public interface Controller { + default boolean isLegacy() throws IOException { + return !getPath().resolve("cgroup.controllers").toFile().exists(); + } + + static T getDefault(Class clazz) { + InvocationHandler handler = new InvocationHandler() { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws ExecException { + throw new ExecException("Cgroup requested by cgroups are not available!") { + protected FailureDetails.FailureDetail getFailureDetail(String message) { + return FailureDetails.FailureDetail.newBuilder().setMessage(message).build(); + } + }; + + } + }; + + return (T) Proxy.newProxyInstance(clazz.getClassLoader(), new Class[]{clazz}, handler); + } + + Path getPath() throws IOException; + + interface Memory extends Controller { + void setMaxBytes(long bytes) throws IOException; + long getMaxBytes() throws IOException; + } + interface Cpu extends Controller { + void setCpus(float cpus) throws IOException; + int getCpus() throws IOException; + } +} \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Hierarchy.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Hierarchy.java new file mode 100644 index 00000000000000..9db168ac410543 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Hierarchy.java @@ -0,0 +1,56 @@ +package com.google.devtools.build.lib.sandbox.cgroups; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@AutoValue +public abstract class Hierarchy { + public abstract Integer id(); + public abstract List controllers(); + public abstract Path path(); + public boolean isV2() { + return controllers().isEmpty() && id() == 0; + } + + /** + * A regexp that matches entries in {@code /proc/self/cgroup}. + * + * The format is documented in https://man7.org/linux/man-pages/man7/cgroups.7.html + */ + private static final Pattern PROC_CGROUPS_PATTERN = + Pattern.compile("^(?\\d+):(?[^:]*):(?.+)"); + + static Hierarchy create(Integer id, List controllers, String path) { + return new AutoValue_Hierarchy(id, controllers, Paths.get(path)); + } + + static List parse(File procCgroup) throws IOException { + ImmutableList.Builder hierarchies = ImmutableList.builder(); + for (String line : Files.readLines(procCgroup, StandardCharsets.UTF_8)) { + Matcher m = PROC_CGROUPS_PATTERN.matcher(line); + if (!m.matches()) { + continue; + } + + Integer id = Integer.parseInt(m.group("id")); + String path = m.group("file"); + List cs = ImmutableList.copyOf(m.group("controllers").split(",")); + hierarchies.add(Hierarchy.create(id, cs, path)); + } + return hierarchies.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Mount.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Mount.java new file mode 100644 index 00000000000000..4e86e1ec235de0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Mount.java @@ -0,0 +1,53 @@ +package com.google.devtools.build.lib.sandbox.cgroups; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@AutoValue +public abstract class Mount { + /** + * A regexp that matches cgroups entries in {@code /proc/mounts}. + * + * The format is documented in https://man7.org/linux/man-pages/man5/fstab.5.html + */ + private static final Pattern CGROUPS_MOUNT_PATTERN = + Pattern.compile("^[^\\s#]\\S*\\s+(?\\S*)\\s+(?cgroup2?)\\s+(?\\S*).*"); + + public abstract Path path(); + public abstract String type(); + public abstract List opts(); + public boolean isV2() { + return type().equals("cgroup2"); + } + + static Mount create(String path, String type, List opts) { + return new AutoValue_Mount(Paths.get(path), type, opts); + } + + static List parse(File procMounts) throws IOException { + ImmutableList.Builder mounts = ImmutableList.builder(); + + for (String mount: Files.readLines(procMounts, StandardCharsets.UTF_8)) { + Matcher m = CGROUPS_MOUNT_PATTERN.matcher(mount); + if (!m.matches()) { + continue; + } + + String path = m.group("file"); + String type = m.group("vfstype"); + List opts = ImmutableList.copyOf(m.group("mntops").split(",")); + mounts.add(Mount.create(path, type, opts)); + } + return mounts.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/VirtualCGroup.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/VirtualCGroup.java new file mode 100644 index 00000000000000..c1fed31b26c8cd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/VirtualCGroup.java @@ -0,0 +1,214 @@ +package com.google.devtools.build.lib.sandbox.cgroups; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.GoogleLogger; +import com.google.common.io.CharSink; +import com.google.common.io.Files; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.sandbox.cgroups.v1.LegacyCpu; +import com.google.devtools.build.lib.sandbox.cgroups.v1.LegacyMemory; +import com.google.devtools.build.lib.sandbox.cgroups.v2.UnifiedCpu; +import com.google.devtools.build.lib.sandbox.cgroups.v2.UnifiedMemory; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Scanner; +import java.util.concurrent.ConcurrentLinkedQueue; + + +/** + * This class creates and exposes a virtual cgroup for the bazel process and allows creating + * child cgroups. Resources are exposed as {@link Controller}s, each representing a + * subsystem within the virtual cgroup and that could in theory belong to different real cgroups. + */ +@AutoValue +public abstract class VirtualCGroup { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + private static final File PROC_SELF_MOUNTS_PATH = new File("/proc/self/mounts"); + private static final File PROC_SELF_CGROUP_PATH = new File("/proc/self/cgroup"); + + @Nullable + private static volatile VirtualCGroup instance; + + public abstract Controller.Cpu cpu(); + public abstract Controller.Memory memory(); + public abstract ImmutableSet paths(); + + private final Queue children = new ConcurrentLinkedQueue<>(); + + public static VirtualCGroup getInstance(EventHandler reporter) throws IOException { + if (instance == null) { + synchronized (VirtualCGroup.class) { + if (instance == null) { + instance = create(reporter); + } + } + } + return instance; + } + + public static void deleteInstance() { + if (instance != null) { + synchronized (VirtualCGroup.class) { + if (instance != null) { + instance.delete(); + instance = null; + } + } + } + } + + public static VirtualCGroup create(EventHandler reporter) throws IOException { + return create(PROC_SELF_MOUNTS_PATH, PROC_SELF_CGROUP_PATH, reporter); + } + + private static void copyControllersToSubtree(Path cgroup) throws IOException { + File subtree = cgroup.resolve("cgroup.subtree_control").toFile(); + File controllers = cgroup.resolve("cgroup.controllers").toFile(); + if (subtree.canWrite() && controllers.canRead()) { + CharSink sink = Files.asCharSink(subtree, StandardCharsets.UTF_8); + Scanner scanner = new Scanner(controllers); + while (scanner.hasNext()) { + sink.write("+" + scanner.next()); + } + } + } + + static VirtualCGroup create(File procMounts, File procCgroup, EventHandler reporter) throws IOException { + final List mounts = Mount.parse(procMounts); + final Map hierarchies = Hierarchy.parse(procCgroup) + .stream() + .flatMap(h -> h.controllers().stream().map(c -> Map.entry(c, h))) + // For cgroup v2, there are no controllers specified in the proc/pid/cgroup file + // So the keep will be empty and unique. For cgroup v1, there could potentially + // be multiple mounting points for the same controller, but they represent a + // "view of the same hierarchy" so it is ok to just keep one. + // Ref. https://man7.org/linux/man-pages/man7/cgroups.7.html + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + Controller.Memory memory = null; + Controller.Cpu cpu = null; + ImmutableSet.Builder paths = ImmutableSet.builder(); + + for (Mount m: mounts) { + if (memory != null && cpu != null) break; + + if (m.isV2()) { + Hierarchy h = hierarchies.get(""); + if (h == null) continue; + Path cgroup = m.path().resolve(Paths.get("/").relativize(h.path())); + if (!cgroup.equals(m.path())) { + // Because of the "no internal processes" rule, it is not possible to + // create a non-empty child cgroups on non-root cgroups with member processes + // Instead, we go up one level in the hierarchy and declare a sibling. + cgroup = cgroup.getParent(); + } + if (!cgroup.toFile().canWrite()) { + reporter.handle(Event.warn("Found non-writable cgroup v2 at " + cgroup)); + continue; + } + try (InputStream s = new FileInputStream(cgroup.resolve("cgroup.procs").toFile())) { + // Check if the cgroup is empty, i.e. there are no member processes + // before modifying the subtree control to respect the "no internal processes" + if (s.read() != -1) { + reporter.handle(Event.warn("Found non-empty cgroup v2 at " + cgroup)); + continue; + } + copyControllersToSubtree(cgroup); + } + + cgroup = cgroup.resolve("bazel_" + ProcessHandle.current().pid() + ".slice"); + cgroup.toFile().mkdirs(); + paths.add(cgroup); + + Scanner scanner = new Scanner(cgroup.resolve("cgroup.controllers").toFile()); + while (scanner.hasNext()) { + switch (scanner.next()) { + case "memory": + if (memory != null) continue; + logger.atInfo().log("Found cgroup v2 memory controller at %s", cgroup); + memory = new UnifiedMemory(cgroup); + break; + case "cpu": + if (cpu != null) continue; + logger.atInfo().log("Found cgroup v2 cpu controller at %s", cgroup); + cpu = new UnifiedCpu(cgroup); + break; + } + } + } else { + for (String opt : m.opts()) { + Hierarchy h = hierarchies.get(opt); + if (h == null) continue; + Path cgroup = m.path().resolve(Paths.get("/").relativize(h.path())); + if (!cgroup.toFile().canWrite()) { + reporter.handle(Event.warn("Found non-writable cgroup v1 at " + cgroup)); + continue; + } + cgroup = cgroup.resolve("bazel_" + ProcessHandle.current().pid() + ".slice"); + cgroup.toFile().mkdirs(); + paths.add(cgroup); + + switch (opt) { + case "memory": + if (memory != null) continue; + logger.atInfo().log("Found cgroup v1 memory controller at %s", cgroup); + memory = new LegacyMemory(cgroup); + break; + case "cpu": + if (cpu != null) continue; + logger.atInfo().log("Found cgroup v1 cpu controller at %s", cgroup); + cpu = new LegacyCpu(cgroup); + break; + } + } + } + } + + cpu = cpu != null ? cpu : Controller.getDefault(Controller.Cpu.class); + memory = memory != null ? memory : Controller.getDefault(Controller.Memory.class); + VirtualCGroup vcgroup = new AutoValue_VirtualCGroup(cpu, memory, paths.build()); + Runtime.getRuntime().addShutdownHook(new Thread(() -> vcgroup.delete())); + return vcgroup; + } + + public void delete() { + this.children.forEach(VirtualCGroup::delete); + this.paths().stream().map(Path::toFile).filter(File::exists).forEach(File::delete); + } + + public VirtualCGroup child(String name) throws IOException { + Controller.Cpu cpu = Controller.getDefault(Controller.Cpu.class); + Controller.Memory memory = Controller.getDefault(Controller.Memory.class); + ImmutableSet.Builder paths = ImmutableSet.builder(); + if (memory() != null && memory().getPath() != null) { + copyControllersToSubtree(memory().getPath()); + Path cgroup = memory().getPath().resolve(name); + cgroup.toFile().mkdirs(); + memory = memory().isLegacy() ? new LegacyMemory(cgroup) : new UnifiedMemory(cgroup); + paths.add(cgroup); + } + if (cpu() != null && cpu().getPath() != null) { + copyControllersToSubtree(cpu().getPath()); + Path cgroup = cpu().getPath().resolve(name); + cgroup.toFile().mkdirs(); + cpu = cpu().isLegacy() ? new LegacyCpu(cgroup) : new UnifiedCpu(cgroup); + paths.add(cgroup); + } + VirtualCGroup child = new AutoValue_VirtualCGroup(cpu, memory, paths.build()); + this.children.add(child); + return child; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyCpu.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyCpu.java new file mode 100644 index 00000000000000..cdcff1643ba070 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyCpu.java @@ -0,0 +1,33 @@ +package com.google.devtools.build.lib.sandbox.cgroups.v1; + +import com.google.devtools.build.lib.sandbox.cgroups.Controller; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class LegacyCpu implements Controller.Cpu { + private final Path path; + private final int period; + + public LegacyCpu(Path path) throws IOException { + this.path = path; + this.period = Integer.parseInt(Files.readString(path.resolve("cpu.cfs_period_us")).trim()); + } + + @Override + public Path getPath() { + return path; + } + + @Override + public void setCpus(float cpus) throws IOException { + int quota = Math.round(cpus * period); + Files.writeString(path.resolve("cpu.cfs_quota_us"), Integer.toString(quota)); + } + + @Override + public int getCpus() throws IOException { + return Integer.parseInt(Files.readString(path.resolve("cpu.cfs_quota_us")).trim()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyMemory.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyMemory.java new file mode 100644 index 00000000000000..873f728ccfde19 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyMemory.java @@ -0,0 +1,30 @@ +package com.google.devtools.build.lib.sandbox.cgroups.v1; + +import com.google.devtools.build.lib.sandbox.cgroups.Controller; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class LegacyMemory implements Controller.Memory { + private final Path path; + + @Override + public Path getPath() { + return path; + } + + public LegacyMemory(Path path) { + this.path = path; + } + + @Override + public void setMaxBytes(long bytes) throws IOException { + Files.writeString(path.resolve("memory.limit_in_bytes"), Long.toString(bytes)); + } + + @Override + public long getMaxBytes() throws IOException { + return Long.parseLong(Files.readString(path.resolve("memory.limit_in_bytes")).trim()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedCpu.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedCpu.java new file mode 100644 index 00000000000000..a3711f845c80cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedCpu.java @@ -0,0 +1,32 @@ +package com.google.devtools.build.lib.sandbox.cgroups.v2; + +import com.google.devtools.build.lib.sandbox.cgroups.Controller; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class UnifiedCpu implements Controller.Cpu { + private final Path path; + public UnifiedCpu(Path path) { + this.path = path; + } + + @Override + public Path getPath() { + return path; + } + + @Override + public void setCpus(float cpus) throws IOException { + int period = 1000_000; + int quota = Math.round(period * cpus); + String limit = String.format("%d %d", quota, period); + Files.writeString(path.resolve("cpu.max"), limit); + } + + @Override + public int getCpus() throws IOException { + return Integer.parseInt(Files.readString(path.resolve("cpu.max")).trim()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedMemory.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedMemory.java new file mode 100644 index 00000000000000..3e6e23e1d7e9bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedMemory.java @@ -0,0 +1,29 @@ +package com.google.devtools.build.lib.sandbox.cgroups.v2; + +import com.google.devtools.build.lib.sandbox.cgroups.Controller; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class UnifiedMemory implements Controller.Memory { + private final Path path; + public UnifiedMemory(Path path) { + this.path = path; + } + + @Override + public Path getPath() { + return path; + } + + @Override + public void setMaxBytes(long bytes) throws IOException { + Files.writeString(path.resolve("memory.max"), Long.toString(bytes)); + } + + @Override + public long getMaxBytes() throws IOException { + return Long.parseLong(Files.readString(path.resolve("memory.max")).trim()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/worker/BUILD b/src/main/java/com/google/devtools/build/lib/worker/BUILD index 284821909ef17e..2e8539a32ee5d9 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/BUILD +++ b/src/main/java/com/google/devtools/build/lib/worker/BUILD @@ -269,12 +269,14 @@ java_library( ":worker_exec_root", ":worker_key", ":worker_protocol", + "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/exec:tree_deleter", "//src/main/java/com/google/devtools/build/lib/sandbox:linux_sandbox", "//src/main/java/com/google/devtools/build/lib/sandbox:linux_sandbox_command_line_builder", "//src/main/java/com/google/devtools/build/lib/sandbox:linux_sandbox_util", "//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_helpers", + "//src/main/java/com/google/devtools/build/lib/sandbox/cgroups", "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", diff --git a/src/main/java/com/google/devtools/build/lib/worker/SandboxedWorker.java b/src/main/java/com/google/devtools/build/lib/worker/SandboxedWorker.java index befba2576a290a..fc3fbd04fb8e5c 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/SandboxedWorker.java +++ b/src/main/java/com/google/devtools/build/lib/worker/SandboxedWorker.java @@ -23,12 +23,13 @@ import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.exec.TreeDeleter; -import com.google.devtools.build.lib.sandbox.CgroupsInfo; +import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.sandbox.LinuxSandboxCommandLineBuilder; import com.google.devtools.build.lib.sandbox.LinuxSandboxUtil; import com.google.devtools.build.lib.sandbox.SandboxHelpers; import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs; import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs; +import com.google.devtools.build.lib.sandbox.cgroups.VirtualCGroup; import com.google.devtools.build.lib.shell.Subprocess; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; @@ -63,6 +64,8 @@ public abstract static class WorkerSandboxOptions { abstract int memoryLimit(); + abstract EventHandler reporter(); + abstract ImmutableSet inaccessiblePaths(); abstract ImmutableList> additionalMountPaths(); @@ -75,6 +78,7 @@ public static WorkerSandboxOptions create( ImmutableList tmpfsPath, ImmutableList writablePaths, int memoryLimit, + EventHandler reporter, ImmutableSet inaccessiblePaths, ImmutableList> sandboxAdditionalMounts) { return new AutoValue_SandboxedWorker_WorkerSandboxOptions( @@ -85,6 +89,7 @@ public static WorkerSandboxOptions create( writablePaths, sandboxBinary, memoryLimit, + reporter, inaccessiblePaths, sandboxAdditionalMounts); } @@ -191,12 +196,13 @@ protected Subprocess createProcess() throws IOException, UserExecException { .setCreateNetworkNamespace(NETNS); if (hardenedSandboxOptions.memoryLimit() > 0) { - CgroupsInfo cgroupsInfo = CgroupsInfo.getInstance(); - // We put the sandbox inside a unique subdirectory using the worker's ID. - cgroupsDir = - cgroupsInfo.createMemoryLimitCgroupDir( - "worker_sandbox_" + workerId, hardenedSandboxOptions.memoryLimit()); - commandLineBuilder.setCgroupsDir(cgroupsDir); + String name = "worker_sandbox_" + workerId; + VirtualCGroup cgroup = + VirtualCGroup.getInstance(hardenedSandboxOptions.reporter()).child(name); + cgroup.memory().setMaxBytes(hardenedSandboxOptions.memoryLimit() * 1024L * 1024L); + ImmutableSet.Builder paths = ImmutableSet.builder(); + cgroup.paths().forEach(p -> paths.add(workDir.getFileSystem().getPath(p.toString()))); + commandLineBuilder.setCgroupsDirs(paths.build()); } if (this.hardenedSandboxOptions.fakeUsername()) { diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java index 509907e3596e5a..3243819dcac750 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java @@ -107,6 +107,7 @@ public void buildStarting(BuildStartingEvent event) { ImmutableList.copyOf(sandboxOptions.sandboxTmpfsPath), ImmutableList.copyOf(sandboxOptions.sandboxWritablePath), sandboxOptions.memoryLimitMb, + env.getReporter(), sandboxOptions.getInaccessiblePaths(env.getRuntime().getFileSystem()), ImmutableList.copyOf(sandboxOptions.sandboxAdditionalMounts)); } else { diff --git a/src/main/tools/linux-sandbox-options.cc b/src/main/tools/linux-sandbox-options.cc index 6464d72a4bde24..e44eb7212a9b2a 100644 --- a/src/main/tools/linux-sandbox-options.cc +++ b/src/main/tools/linux-sandbox-options.cc @@ -234,7 +234,8 @@ static void ParseCommandLine(unique_ptr> args) { break; case 'C': ValidateIsAbsolutePath(optarg, args->front(), static_cast(c)); - opt.cgroups_dir.assign(optarg); + opt.cgroups_dirs.emplace_back(optarg); + opt.writable_files.emplace_back(optarg); break; case 'P': opt.enable_pty = true; diff --git a/src/main/tools/linux-sandbox-options.h b/src/main/tools/linux-sandbox-options.h index a4ed85d41cc381..3880da16f79db5 100644 --- a/src/main/tools/linux-sandbox-options.h +++ b/src/main/tools/linux-sandbox-options.h @@ -66,8 +66,8 @@ struct Options { bool hermetic; // The sandbox root directory (-s) std::string sandbox_root; - // Directory to use for cgroup control - std::string cgroups_dir; + // Directories to use for cgroup control + std::vector cgroups_dirs; // Command to run (--) std::vector args; }; diff --git a/src/main/tools/linux-sandbox-pid1.cc b/src/main/tools/linux-sandbox-pid1.cc index 97457ed1641d78..f0d4d83bd30e89 100644 --- a/src/main/tools/linux-sandbox-pid1.cc +++ b/src/main/tools/linux-sandbox-pid1.cc @@ -344,7 +344,7 @@ static bool ShouldBeWritable(const std::string &mnt_dir) { return true; } - if (mnt_dir == "/sys/fs/cgroup" && !opt.cgroups_dir.empty()) { + if (mnt_dir == "/sys/fs/cgroup" && !opt.cgroups_dirs.empty()) { return true; } @@ -589,9 +589,12 @@ static int WaitForChild() { } static void AddProcessToCgroup() { - if (!opt.cgroups_dir.empty()) { - PRINT_DEBUG("Adding process to cgroups dir %s", opt.cgroups_dir.c_str()); - WriteFile(opt.cgroups_dir + "/cgroup.procs", "1"); + for(const std::string &cgroups_dir : opt.cgroups_dirs) { + PRINT_DEBUG("Adding process to cgroup dir %s", cgroups_dir.c_str()); + // Writing the value 0 to a cgroup.procs file causes the writing + // process to be moved to the corresponding cgroup. + // Ref. https://man7.org/linux/man-pages/man7/cgroups.7.html + WriteFile(cgroups_dir + "/cgroup.procs", "0"); } } diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilderTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilderTest.java index d1790e620f1c1d..4e3d016d6cece1 100644 --- a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilderTest.java +++ b/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxCommandLineBuilderTest.java @@ -28,6 +28,8 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import java.nio.file.Paths; import java.time.Duration; import java.util.List; import org.junit.Before; @@ -133,7 +135,7 @@ public void testLinuxSandboxCommandLineBuilder_buildsWithOptionalArguments() { .put(bindMountTarget2, bindMountSource2) .buildOrThrow(); - String cgroupsDir = "/sys/fs/cgroups/something"; + Path cgroupsDir = fileSystem.getPath("/sys/fs/cgroups/something"); ImmutableList expectedCommandLine = ImmutableList.builder() @@ -158,7 +160,7 @@ public void testLinuxSandboxCommandLineBuilder_buildsWithOptionalArguments() { .add("-U") .add("-D", sandboxDebugPath.getPathString()) .add("-p") - .add("-C", cgroupsDir) + .add("-C", cgroupsDir.toString()) .add("--") .addAll(commandArguments) .build(); @@ -180,7 +182,7 @@ public void testLinuxSandboxCommandLineBuilder_buildsWithOptionalArguments() { .setUseFakeUsername(useFakeUsername) .setSandboxDebugPath(sandboxDebugPath.getPathString()) .setPersistentProcess(true) - .setCgroupsDir(cgroupsDir) + .setCgroupsDirs(ImmutableSet.of(cgroupsDir)) .buildForCommand(commandArguments); assertThat(commandLine).containsExactlyElementsIn(expectedCommandLine).inOrder();