From f3a2fd31f57013d95c4f0b175dcb0be90c83cf1f Mon Sep 17 00:00:00 2001 From: Petr Kudriavtsev Date: Thu, 21 Aug 2025 13:43:08 +0000 Subject: [PATCH 1/5] [databricks] added logging for the function calls Could be configured via env variables --- .../java/net/starlark/java/eval/Eval.java | 124 +++++++++++++++++- 1 file changed, 118 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/starlark/java/eval/Eval.java b/src/main/java/net/starlark/java/eval/Eval.java index 5b46ea2204963c..fd41f7109a8f46 100644 --- a/src/main/java/net/starlark/java/eval/Eval.java +++ b/src/main/java/net/starlark/java/eval/Eval.java @@ -17,11 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import net.starlark.java.spelling.SpellChecker; import net.starlark.java.syntax.Argument; import net.starlark.java.syntax.AssignmentStatement; @@ -587,6 +583,118 @@ private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot) } } + // databricks-extension { + private static final boolean DB_LOG_CALLS = System.getenv().containsKey("DB_LOG_STARLARK_CALLS"); + private static final boolean DB_LOG_CALL_PARAMS; + private static final boolean DB_LOG_CALL_CALLSTACK; + private static final boolean DB_LOG_CALL_RESULT; + + // (function name -> set of parameter groups). For a call to be logged, at least one group should be matched + // against the actual parameters in the call. + // + // An example: "(function name -> (p1 & p2) | (p3)" + // this means that the function will be logged if call parameters contain either 'p1' and 'p2', or 'p3' + // + // An example of a filter specified from the terminal: + // ``` + // export DB_LOG_STARLARK_CALLS='params;callstack;result' + // export DB_LOG_STARLARK_CALLS_FILTER='cc_toolchain;_is_enabled:module_maps|header_modules' + // ``` + // This means that + // a) All calls that are logged will be logged with the passed parameters, the callstack and the result + // b) The logging will be applied to + // * all `cc_toolchain` calls + // * `_is_enabled` calls which have "module_maps" or "header_modules" values among parameters + private static final Map>> DB_CALL_FILTER = new HashMap<>(); + + static { + final var logCallFeatures = new HashSet<>( + Arrays.asList(System.getenv() + .getOrDefault("DB_LOG_STARLARK_CALLS", "").split(";")) + ); + DB_LOG_CALL_PARAMS = logCallFeatures.contains("params"); + DB_LOG_CALL_CALLSTACK = logCallFeatures.contains("callstack"); + DB_LOG_CALL_RESULT = logCallFeatures.contains("result"); + String[] logCallFilters = System.getenv() + .getOrDefault("DB_LOG_STARLARK_CALLS_FILTER", "").split(";"); + for (String logCallFilter : logCallFilters) { + final var nameAndOrParamGroups = logCallFilter.split(":"); + final var name = nameAndOrParamGroups[0]; + final var orParamGroupsFilter = DB_CALL_FILTER.computeIfAbsent(name, key -> new HashSet<>()); + final var orParamGroups = nameAndOrParamGroups.length > 1 + ? nameAndOrParamGroups[1].split("\\|") + : new String[0]; + for (String orParamGroup : orParamGroups) { + final String[] andParams = orParamGroup.split("&"); + orParamGroupsFilter.add(new HashSet<>(List.of(andParams))); + } + } + } + + private static boolean dbIsAcceptableCall(Object fn, Object[] positionalArgs, Object[] namedArgs) { + if (DB_CALL_FILTER.isEmpty()) { + return true; + } + if (!(fn instanceof StarlarkFunction)) { + return false; + } + String functionName = ((StarlarkFunction) fn).getName(); + final var orParamGroups = DB_CALL_FILTER.get(functionName); + if (orParamGroups == null) { + return false; + } else if (orParamGroups.isEmpty()) { + return true; + } + for (var andParams : orParamGroups) { + int counter = 0; + for (var positional : positionalArgs) { + if (andParams.contains(positional.toString())) { + ++counter; + } + } + for (var named : namedArgs) { + if (andParams.contains(named.toString())) { + ++counter; + } + } + if (counter >= andParams.size()) { + return true; + } + } + return false; + } + + private static void dbLogCall(StarlarkThread.Frame fr, Object fn, Object[] positional, + Object[] named, Location loc, Object result) { + final var dbLogCalls = DB_LOG_CALLS && dbIsAcceptableCall(fn, positional, named); + if (dbLogCalls) { + StringBuilder sb = new StringBuilder(); + String threadId = String.format("%h", System.identityHashCode(fr.thread)); + sb.append(String.format("(%s): evaluating call %s at %s\n", threadId, fn, loc.toString())); + if (DB_LOG_CALL_PARAMS) { + sb.append(" positional args: ").append(Arrays.toString(positional)).append("\n"); + sb.append(" named args: ").append(Arrays.toString(named)).append("\n"); + } + if (DB_LOG_CALL_CALLSTACK) { + sb.append(" call stack:\n"); + for (var entry : fr.thread.getCallStack()) { + sb.append(" (") + .append(threadId) + .append(") ") + .append(entry.name) + .append("(") + .append(entry.location) + .append(")\n"); + } + } + if (DB_LOG_CALL_RESULT) { + sb.append(" result: ").append(result).append("\n"); + } + System.err.print(sb); + } + } + // databricks-extension } + private static Object evalCall(StarlarkThread.Frame fr, CallExpression call) throws EvalException, InterruptedException { fr.thread.checkInterrupt(); @@ -679,7 +787,11 @@ private static Object evalCall(StarlarkThread.Frame fr, CallExpression call) Location loc = call.getLparenLocation(); // (Location is prematerialized) fr.setLocation(loc); try { - return Starlark.fastcall(fr.thread, fn, positional, named); + // databricks-changed { + Object result = Starlark.fastcall(fr.thread, fn, positional, named); + dbLogCall(fr, fn, positional, named, loc, result); + return result; + // databricks-changed } } catch (EvalException ex) { fr.setErrorLocation(loc); throw ex; From 39c5b35c635f962cf84cf9de61245adf21739a7a Mon Sep 17 00:00:00 2001 From: PetrKudr Date: Mon, 1 Sep 2025 14:24:49 +0000 Subject: [PATCH 2/5] [databricks] allow function call filter to be blank --- src/main/java/net/starlark/java/eval/Eval.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/net/starlark/java/eval/Eval.java b/src/main/java/net/starlark/java/eval/Eval.java index fd41f7109a8f46..9974300ee21794 100644 --- a/src/main/java/net/starlark/java/eval/Eval.java +++ b/src/main/java/net/starlark/java/eval/Eval.java @@ -618,6 +618,9 @@ private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot) String[] logCallFilters = System.getenv() .getOrDefault("DB_LOG_STARLARK_CALLS_FILTER", "").split(";"); for (String logCallFilter : logCallFilters) { + if (logCallFilter.isBlank()) { + continue; + } final var nameAndOrParamGroups = logCallFilter.split(":"); final var name = nameAndOrParamGroups[0]; final var orParamGroupsFilter = DB_CALL_FILTER.computeIfAbsent(name, key -> new HashSet<>()); From 1208dfa26994d9524db2aa99afec6d63b6ca1032 Mon Sep 17 00:00:00 2001 From: PetrKudr Date: Mon, 1 Sep 2025 14:25:18 +0000 Subject: [PATCH 3/5] [databricks] logging for TC features --- .../devtools/build/lib/rules/cpp/CcToolchainFeatures.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java index ad7d993a227f0a..74f57fb9c2202e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java @@ -1576,6 +1576,11 @@ ImmutableSet getEnabledFeatureNames() { private PathFragment ccToolchainPath; + // databricks-extension { + private static final boolean DB_LOG_TC_FEATURES = System.getenv().containsKey("DB_LOG_TOOLCHAIN_FEATURES") && + !System.getenv().get("DB_LOG_TOOLCHAIN_FEATURES").isBlank(); + // databricks-extension } + /** * Constructs the feature configuration from a {@link CcToolchainConfigInfo}. * @@ -1599,6 +1604,9 @@ public CcToolchainFeatures( ImmutableList.Builder defaultSelectablesBuilder = ImmutableList.builder(); for (Feature feature : ccToolchainConfigInfo.getFeatures()) { + if (DB_LOG_TC_FEATURES) { + System.err.println("CC toolchain feature: " + feature.getName()); + } selectablesBuilder.add(feature); selectablesByName.put(feature.getName(), feature); if (feature.isEnabled()) { From cb038c5f3ea8c1fec3f564a475d66bd8556e7b7d Mon Sep 17 00:00:00 2001 From: PetrKudr Date: Mon, 1 Sep 2025 14:50:37 +0000 Subject: [PATCH 4/5] [databricks] added an easy way to disable logging --- src/main/java/net/starlark/java/eval/Eval.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/starlark/java/eval/Eval.java b/src/main/java/net/starlark/java/eval/Eval.java index 9974300ee21794..528ac09fef10fe 100644 --- a/src/main/java/net/starlark/java/eval/Eval.java +++ b/src/main/java/net/starlark/java/eval/Eval.java @@ -584,7 +584,9 @@ private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot) } // databricks-extension { - private static final boolean DB_LOG_CALLS = System.getenv().containsKey("DB_LOG_STARLARK_CALLS"); + private static final boolean DB_LOG_CALLS = !"no".contentEquals( + System.getenv().getOrDefault("DB_LOG_STARLARK_CALLS", "no") + ); private static final boolean DB_LOG_CALL_PARAMS; private static final boolean DB_LOG_CALL_CALLSTACK; private static final boolean DB_LOG_CALL_RESULT; From a0699189065d33beaa018c625a4347ac0fc0f771 Mon Sep 17 00:00:00 2001 From: PetrKudr Date: Tue, 2 Sep 2025 15:42:58 +0000 Subject: [PATCH 5/5] [databricks] logging of function calls moved down the call stack --- .../java/net/starlark/java/eval/Eval.java | 123 +--------------- .../java/net/starlark/java/eval/Starlark.java | 139 ++++++++++++++++-- 2 files changed, 127 insertions(+), 135 deletions(-) diff --git a/src/main/java/net/starlark/java/eval/Eval.java b/src/main/java/net/starlark/java/eval/Eval.java index 528ac09fef10fe..2ef344dfec76e1 100644 --- a/src/main/java/net/starlark/java/eval/Eval.java +++ b/src/main/java/net/starlark/java/eval/Eval.java @@ -583,123 +583,6 @@ private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot) } } - // databricks-extension { - private static final boolean DB_LOG_CALLS = !"no".contentEquals( - System.getenv().getOrDefault("DB_LOG_STARLARK_CALLS", "no") - ); - private static final boolean DB_LOG_CALL_PARAMS; - private static final boolean DB_LOG_CALL_CALLSTACK; - private static final boolean DB_LOG_CALL_RESULT; - - // (function name -> set of parameter groups). For a call to be logged, at least one group should be matched - // against the actual parameters in the call. - // - // An example: "(function name -> (p1 & p2) | (p3)" - // this means that the function will be logged if call parameters contain either 'p1' and 'p2', or 'p3' - // - // An example of a filter specified from the terminal: - // ``` - // export DB_LOG_STARLARK_CALLS='params;callstack;result' - // export DB_LOG_STARLARK_CALLS_FILTER='cc_toolchain;_is_enabled:module_maps|header_modules' - // ``` - // This means that - // a) All calls that are logged will be logged with the passed parameters, the callstack and the result - // b) The logging will be applied to - // * all `cc_toolchain` calls - // * `_is_enabled` calls which have "module_maps" or "header_modules" values among parameters - private static final Map>> DB_CALL_FILTER = new HashMap<>(); - - static { - final var logCallFeatures = new HashSet<>( - Arrays.asList(System.getenv() - .getOrDefault("DB_LOG_STARLARK_CALLS", "").split(";")) - ); - DB_LOG_CALL_PARAMS = logCallFeatures.contains("params"); - DB_LOG_CALL_CALLSTACK = logCallFeatures.contains("callstack"); - DB_LOG_CALL_RESULT = logCallFeatures.contains("result"); - String[] logCallFilters = System.getenv() - .getOrDefault("DB_LOG_STARLARK_CALLS_FILTER", "").split(";"); - for (String logCallFilter : logCallFilters) { - if (logCallFilter.isBlank()) { - continue; - } - final var nameAndOrParamGroups = logCallFilter.split(":"); - final var name = nameAndOrParamGroups[0]; - final var orParamGroupsFilter = DB_CALL_FILTER.computeIfAbsent(name, key -> new HashSet<>()); - final var orParamGroups = nameAndOrParamGroups.length > 1 - ? nameAndOrParamGroups[1].split("\\|") - : new String[0]; - for (String orParamGroup : orParamGroups) { - final String[] andParams = orParamGroup.split("&"); - orParamGroupsFilter.add(new HashSet<>(List.of(andParams))); - } - } - } - - private static boolean dbIsAcceptableCall(Object fn, Object[] positionalArgs, Object[] namedArgs) { - if (DB_CALL_FILTER.isEmpty()) { - return true; - } - if (!(fn instanceof StarlarkFunction)) { - return false; - } - String functionName = ((StarlarkFunction) fn).getName(); - final var orParamGroups = DB_CALL_FILTER.get(functionName); - if (orParamGroups == null) { - return false; - } else if (orParamGroups.isEmpty()) { - return true; - } - for (var andParams : orParamGroups) { - int counter = 0; - for (var positional : positionalArgs) { - if (andParams.contains(positional.toString())) { - ++counter; - } - } - for (var named : namedArgs) { - if (andParams.contains(named.toString())) { - ++counter; - } - } - if (counter >= andParams.size()) { - return true; - } - } - return false; - } - - private static void dbLogCall(StarlarkThread.Frame fr, Object fn, Object[] positional, - Object[] named, Location loc, Object result) { - final var dbLogCalls = DB_LOG_CALLS && dbIsAcceptableCall(fn, positional, named); - if (dbLogCalls) { - StringBuilder sb = new StringBuilder(); - String threadId = String.format("%h", System.identityHashCode(fr.thread)); - sb.append(String.format("(%s): evaluating call %s at %s\n", threadId, fn, loc.toString())); - if (DB_LOG_CALL_PARAMS) { - sb.append(" positional args: ").append(Arrays.toString(positional)).append("\n"); - sb.append(" named args: ").append(Arrays.toString(named)).append("\n"); - } - if (DB_LOG_CALL_CALLSTACK) { - sb.append(" call stack:\n"); - for (var entry : fr.thread.getCallStack()) { - sb.append(" (") - .append(threadId) - .append(") ") - .append(entry.name) - .append("(") - .append(entry.location) - .append(")\n"); - } - } - if (DB_LOG_CALL_RESULT) { - sb.append(" result: ").append(result).append("\n"); - } - System.err.print(sb); - } - } - // databricks-extension } - private static Object evalCall(StarlarkThread.Frame fr, CallExpression call) throws EvalException, InterruptedException { fr.thread.checkInterrupt(); @@ -792,11 +675,7 @@ private static Object evalCall(StarlarkThread.Frame fr, CallExpression call) Location loc = call.getLparenLocation(); // (Location is prematerialized) fr.setLocation(loc); try { - // databricks-changed { - Object result = Starlark.fastcall(fr.thread, fn, positional, named); - dbLogCall(fr, fn, positional, named, loc, result); - return result; - // databricks-changed } + return Starlark.fastcall(fr.thread, fn, positional, named); } catch (EvalException ex) { fr.setErrorLocation(loc); throw ex; diff --git a/src/main/java/net/starlark/java/eval/Starlark.java b/src/main/java/net/starlark/java/eval/Starlark.java index c8ee62bc264d7d..0fd98620f41df8 100644 --- a/src/main/java/net/starlark/java/eval/Starlark.java +++ b/src/main/java/net/starlark/java/eval/Starlark.java @@ -30,24 +30,14 @@ import java.lang.reflect.Method; import java.math.BigInteger; import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import net.starlark.java.annot.StarlarkAnnotations; import net.starlark.java.annot.StarlarkBuiltin; import net.starlark.java.annot.StarlarkMethod; import net.starlark.java.spelling.SpellChecker; -import net.starlark.java.syntax.Expression; -import net.starlark.java.syntax.FileOptions; -import net.starlark.java.syntax.ParserInput; -import net.starlark.java.syntax.Program; -import net.starlark.java.syntax.Resolver; -import net.starlark.java.syntax.StarlarkFile; -import net.starlark.java.syntax.SyntaxError; +import net.starlark.java.syntax.*; /** * The Starlark class defines the most important entry points, constants, and functions needed by @@ -771,6 +761,125 @@ public static Object call( } return fastcall(thread, fn, args.toArray(), named); } + +// databricks-extension { + private static final boolean DB_LOG_CALLS = !"no".contentEquals( + System.getenv().getOrDefault("DB_LOG_STARLARK_CALLS", "no") + ); + private static final boolean DB_LOG_CALL_PARAMS; + private static final boolean DB_LOG_CALL_CALLSTACK; + private static final boolean DB_LOG_CALL_RESULT; + + // (function name -> set of parameter groups). For a call to be logged, at least one group should be matched + // against the actual parameters in the call. + // + // An example: "(function name -> (p1 & p2) | (p3)" + // this means that the function will be logged if call parameters contain either 'p1' and 'p2', or 'p3' + // + // An example of a filter specified from the terminal: + // ``` + // export DB_LOG_STARLARK_CALLS='params;callstack;result' + // export DB_LOG_STARLARK_CALLS_FILTER='cc_toolchain;_is_enabled:module_maps|header_modules' + // ``` + // This means that + // a) All calls that are logged will be logged with the passed parameters, the callstack and the result + // b) The logging will be applied to + // * all `cc_toolchain` calls + // * `_is_enabled` calls which have "module_maps" or "header_modules" values among parameters + private static final Map>> DB_CALL_FILTER = new HashMap<>(); + + static { + final var logCallFeatures = new HashSet<>( + Arrays.asList(System.getenv() + .getOrDefault("DB_LOG_STARLARK_CALLS", "").split(";")) + ); + DB_LOG_CALL_PARAMS = logCallFeatures.contains("params"); + DB_LOG_CALL_CALLSTACK = logCallFeatures.contains("callstack"); + DB_LOG_CALL_RESULT = logCallFeatures.contains("result"); + String[] logCallFilters = System.getenv() + .getOrDefault("DB_LOG_STARLARK_CALLS_FILTER", "").split(";"); + for (String logCallFilter : logCallFilters) { + if (logCallFilter.isBlank()) { + continue; + } + final var nameAndOrParamGroups = logCallFilter.split(":"); + final var name = nameAndOrParamGroups[0]; + final var orParamGroupsFilter = DB_CALL_FILTER.computeIfAbsent(name, key -> new HashSet<>()); + final var orParamGroups = nameAndOrParamGroups.length > 1 + ? nameAndOrParamGroups[1].split("\\|") + : new String[0]; + for (String orParamGroup : orParamGroups) { + final String[] andParams = orParamGroup.split("&"); + orParamGroupsFilter.add(new HashSet<>(List.of(andParams))); + } + } + } + + private static boolean dbIsAcceptableCall(Object fn, Object[] positionalArgs, Object[] namedArgs) { + if (DB_CALL_FILTER.isEmpty()) { + return true; + } + if (!(fn instanceof StarlarkFunction)) { + return false; + } + String functionName = ((StarlarkFunction) fn).getName(); + final var orParamGroups = DB_CALL_FILTER.get(functionName); + if (orParamGroups == null) { + return false; + } else if (orParamGroups.isEmpty()) { + return true; + } + for (var andParams : orParamGroups) { + int counter = 0; + for (var positional : positionalArgs) { + if (andParams.contains(positional.toString())) { + ++counter; + } + } + for (var named : namedArgs) { + if (andParams.contains(named.toString())) { + ++counter; + } + } + if (counter >= andParams.size()) { + return true; + } + } + return false; + } + + private static void dbLogCall(StarlarkThread thread, Object fn, Object[] positional, + Object[] named, Object result) { + final var dbLogCalls = DB_LOG_CALLS && dbIsAcceptableCall(fn, positional, named); + if (dbLogCalls) { + final var fr = thread.frame(0); + final var loc = fr.getLocation(); + StringBuilder sb = new StringBuilder(); + String threadId = String.format("%h", System.identityHashCode(thread)); + sb.append(String.format("(%s): evaluating call %s at %s\n", threadId, fn, loc.toString())); + if (DB_LOG_CALL_PARAMS) { + sb.append(" positional args: ").append(Arrays.toString(positional)).append("\n"); + sb.append(" named args: ").append(Arrays.toString(named)).append("\n"); + } + if (DB_LOG_CALL_CALLSTACK) { + sb.append(" call stack:\n"); + for (var entry : thread.getCallStack()) { + sb.append(" (") + .append(threadId) + .append(") ") + .append(entry.name) + .append("(") + .append(entry.location) + .append(")\n"); + } + } + if (DB_LOG_CALL_RESULT) { + sb.append(" result: ").append(result).append("\n"); + } + System.err.print(sb); + } + } + // databricks-extension } /** * Calls the function-like value {@code fn} in the specified thread, passing it the given @@ -803,7 +912,11 @@ public static Object fastcall( thread.push(callable); try { - return callable.fastcall(thread, positional, named); + // databricks-changed { + Object result = callable.fastcall(thread, positional, named); + dbLogCall(thread, callable, positional, named, result); + return result; + // databricks-changed } } catch (UncheckedEvalException | UncheckedEvalError ex) { throw ex; // already wrapped } catch (RuntimeException ex) {