diff --git a/dd-openfeature/build.gradle.kts b/dd-openfeature/build.gradle.kts index ecd5d79228c..65835411e02 100644 --- a/dd-openfeature/build.gradle.kts +++ b/dd-openfeature/build.gradle.kts @@ -160,3 +160,12 @@ tasks.withType().configureEach { ) ).withPathSensitivity(PathSensitivity.RELATIVE) } + +// The scale benchmarks read the fixtures from disk at run time, so a fixture change reruns them. +tasks.named("jmh") { + inputs.dir( + rootProject.layout.projectDirectory.dir( + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites" + ) + ).withPathSensitivity(PathSensitivity.RELATIVE) +} diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java new file mode 100644 index 00000000000..d3f591b3362 --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteBenchmark.java @@ -0,0 +1,143 @@ +package com.datadog.openfeature; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import de.thetaphi.forbiddenapis.SuppressForbidden; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.LayeredEvaluationContext; +import dev.openfeature.sdk.ProviderEvaluation; +import java.io.IOException; +import java.util.List; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Evaluation cost on the opt-in scale suites of the shared FFE fixtures ({@code + * evaluation-suites/scale} and {@code evaluation-suites/scale-json}): large lists, regex, rules, + * splits and shards, time windows, many variations and flags, and JSON values up to 4 MB. The + * dimensions are listed in {@code evaluation-suites/scale/AGENTS.md} of the fixtures. + * + *

Each operation evaluates a case of the selected case file through the provider. With {@code + * index=all}, the operations go through the cases of the file in turn, so a row is the unweighted + * mean over every case of the file, and its slowest case dominates it. Select one case to measure + * one shape: the setup prints the index of each case. + * + *

The setup evaluates every case once and the teardown again after the measurement. Both fail if + * a value or reason differs from the expected one, or if an evaluation reports an error, so a run + * that resolves a case wrongly fails instead of reporting a number. JSON values are compared by + * numeric value: a JSON integer can come back as an integer or as a double. This check guards the + * measurement and is not the conformance check: {@code CanonicalFixtureTest} checks the types. + * + *

The provider is called directly: the OpenFeature client, its hooks and exposure logging are + * not measured. {@code evaluate} passes a reused context. {@code evaluateLayered} passes a fresh + * {@code LayeredEvaluationContext} around it on each call, as the client does, so it also shows the + * cost of reading attributes through the client's context, such as a rule on {@code id} when the + * context has no {@code id}. Select one method with {@code + * -Pjmh.includes=ScaleSuiteBenchmark.evaluate$}. + * + *

The cases and their check are shared with {@code ScaleSuiteTest}, which runs them in the unit + * tests. One fork is enough for allocation and for large time differences. For a time difference + * under about 15%, run three forks: {@code -Pjmh.forks=3} or {@code -f 3}. + * + *

Run all case files: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteBenchmark + * -Pjmh.profilers=gc}. Run one case: {@code ./gradlew :dd-openfeature:jmhJar}, then {@code java + * -jar dd-openfeature/build/libs/*-jmh.jar ScaleSuiteBenchmark -p cases=scale/splits -p index=0 + * -prof gc}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class ScaleSuiteBenchmark { + + /** + * A case file, as {@code /} without the {@code .json} extension. {@code + * ScaleSuiteTest} fails when this list misses a case file of the fixtures. + */ + @Param({ + "scale/flags", + "scale/numeric", + "scale/one-of", + "scale/one-of-many", + "scale/one-of-small-numbers", + "scale/regex", + "scale/regex-many", + "scale/rules", + "scale/rules-sweep", + "scale/semver", + "scale/splits", + "scale/time-windows", + "scale/variations", + "scale/wide-context", + "scale-json/json-shapes", + "scale-json/json-sizes", + "scale-json/json-values" + }) + public String cases; + + /** The index of one case in the file, or {@code all} to go through every case in turn. */ + @Param({"all"}) + public String index; + + private Provider provider; + private ScaleCase[] scaleCases; + private int next; + + @Setup(Level.Trial) + @SuppressForbidden + public void setUp() throws IOException { + final int slash = cases.indexOf('/'); + final ScaleSuites.Suite suite = ScaleSuites.load(cases.substring(0, slash)); + provider = suite.provider(); + final List loaded = suite.cases(cases.substring(slash + 1)); + for (int i = 0; i < loaded.size(); i++) { + final ScaleCase scaleCase = loaded.get(i); + scaleCase.check(scaleCase.evaluate(provider)); + System.out.println("[" + cases + "] case " + i + ": " + scaleCase.describe()); + } + scaleCases = + "all".equals(index) + ? loaded.toArray(new ScaleCase[0]) + : new ScaleCase[] {loaded.get(Integer.parseInt(index))}; + } + + @TearDown(Level.Trial) + public void tearDown() { + for (final ScaleCase scaleCase : scaleCases) { + scaleCase.check(scaleCase.evaluate(provider)); + } + provider.shutdown(); + } + + @Benchmark + public ProviderEvaluation evaluate() { + final ScaleCase scaleCase = scaleCases[next]; + next = next + 1 == scaleCases.length ? 0 : next + 1; + return scaleCase.evaluate(provider); + } + + /** As {@link #evaluate}, with the fresh layered context that the OpenFeature client builds. */ + @Benchmark + public ProviderEvaluation evaluateLayered() { + final ScaleCase scaleCase = scaleCases[next]; + next = next + 1 == scaleCases.length ? 0 : next + 1; + return scaleCase.evaluate( + provider, + new LayeredEvaluationContext( + ImmutableContext.EMPTY, ImmutableContext.EMPTY, null, scaleCase.context())); + } +} diff --git a/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java new file mode 100644 index 00000000000..6d047098cae --- /dev/null +++ b/dd-openfeature/src/jmh/java/com/datadog/openfeature/ScaleSuiteLoadBenchmark.java @@ -0,0 +1,90 @@ +package com.datadog.openfeature; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.datadog.openfeature.internal.ufc.ServerConfiguration; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import de.thetaphi.forbiddenapis.SuppressForbidden; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.ref.Reference; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Configuration load cost of the opt-in scale suites: the time to parse each suite's {@code + * ufc-config.json}. Work that moves into the parser, such as indexes, compiled patterns or prebuilt + * JSON values, shows here. + * + *

The setup also prints the heap that a parsed configuration retains. It is the used heap after + * a full collection, with and without the parsed configuration, as the smallest of three + * measurements, so it is approximate. + * + *

Run: {@code ./gradlew :dd-openfeature:jmh -Pjmh.includes=ScaleSuiteLoadBenchmark + * -Pjmh.profilers=gc}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 2, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(MILLISECONDS) +@Fork(value = 1) +public class ScaleSuiteLoadBenchmark { + + @Param({"scale", "scale-json"}) + public String suite; + + private byte[] config; + + @Setup(Level.Trial) + @SuppressForbidden + public void setUp() throws IOException { + config = ScaleSuites.load(suite).config(); + // The first parse initializes the parser, so it is left out of the measurement. + ServerConfiguration retained = UniversalFlagConfigParser.parse(config); + long retainedBytes = Long.MAX_VALUE; + for (int i = 0; i < 3; i++) { + retained = null; + final long before = usedHeapAfterCollection(); + retained = UniversalFlagConfigParser.parse(config); + final long after = usedHeapAfterCollection(); + Reference.reachabilityFence(retained); + retainedBytes = Math.min(retainedBytes, after - before); + } + System.out.println( + "[" + + suite + + "] config " + + config.length / 1024 + + " KB, " + + retained.flags.size() + + " flags, retained heap about " + + retainedBytes / 1024 + + " KB"); + } + + @Benchmark + public ServerConfiguration parse() throws IOException { + return UniversalFlagConfigParser.parse(config); + } + + private static long usedHeapAfterCollection() { + final MemoryMXBean memory = ManagementFactory.getMemoryMXBean(); + for (int i = 0; i < 3; i++) { + System.gc(); + } + return memory.getHeapMemoryUsage().getUsed(); + } +} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java new file mode 100644 index 00000000000..61d317c75b8 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleCase.java @@ -0,0 +1,177 @@ +package com.datadog.openfeature; + +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.Value; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * One case of a scale suite of the shared FFE fixtures, with its expected value and reason. + * + *

JSON values are compared by numeric value: a JSON integer can come back as an integer or as a + * double, so that one check runs on every evaluator. {@code CanonicalFixtureTest} checks the types. + */ +final class ScaleCase { + private final String flag; + private final String variationType; + private final Object defaultValue; + private final Object expectedValue; + private final String expectedReason; + private final String targetingKey; + private final EvaluationContext context; + + ScaleCase(final Map json) { + this.flag = (String) json.get("flag"); + this.variationType = (String) json.get("variationType"); + final Map result = (Map) json.get("result"); + if (!Arrays.asList("value", "reason").containsAll(result.keySet())) { + throw new IllegalStateException( + "Unhandled expectations for " + json.get("flag") + ": " + result.keySet()); + } + this.defaultValue = typed(json.get("defaultValue")); + this.expectedValue = result.get("value"); + this.expectedReason = (String) result.get("reason"); + this.targetingKey = (String) json.get("targetingKey"); + final MutableContext context = + new MutableContext(Value.objectToValue(json.get("attributes")).asStructure().asMap()); + context.setTargetingKey(targetingKey); + this.context = context; + } + + ProviderEvaluation evaluate(final Provider provider) { + return evaluate(provider, context); + } + + /** Evaluates this case with another context, such as a layered one wrapping {@link #context}. */ + ProviderEvaluation evaluate(final Provider provider, final EvaluationContext context) { + switch (variationType) { + case "BOOLEAN": + return provider.getBooleanEvaluation(flag, (Boolean) defaultValue, context); + case "INTEGER": + return provider.getIntegerEvaluation(flag, (Integer) defaultValue, context); + case "NUMERIC": + return provider.getDoubleEvaluation(flag, (Double) defaultValue, context); + case "STRING": + return provider.getStringEvaluation(flag, (String) defaultValue, context); + case "JSON": + return provider.getObjectEvaluation(flag, (Value) defaultValue, context); + default: + throw new IllegalArgumentException("Unsupported variationType: " + variationType); + } + } + + void check(final ProviderEvaluation details) { + final Object actual = + details.getValue() instanceof Value + ? plain((Value) details.getValue()) + : details.getValue(); + if (!sameValue(expectedValue, actual) + || !Objects.equals(expectedReason, details.getReason()) + || details.getErrorCode() != null) { + throw new IllegalStateException( + "Unexpected result for " + + flag + + " (" + + targetingKey + + "): expected " + + expectedReason + + " " + + abbreviate(expectedValue) + + ", got " + + details.getReason() + + " " + + abbreviate(actual)); + } + } + + EvaluationContext context() { + return context; + } + + String describe() { + return flag + " (" + targetingKey + ") -> " + expectedReason; + } + + /** Converts a fixture value to the type that the provider takes for this variation type. */ + private Object typed(final Object value) { + if (value == null) { + return null; + } + switch (variationType) { + case "INTEGER": + return ((Number) value).intValue(); + case "NUMERIC": + return ((Number) value).doubleValue(); + case "JSON": + return Value.objectToValue(value); + default: + return value; + } + } + + /** Converts a returned value to maps, lists, strings, numbers, booleans and null. */ + private static Object plain(final Value value) { + if (value == null || value.isNull()) { + return null; + } + if (value.isStructure()) { + final Map map = new LinkedHashMap<>(); + value.asStructure().asMap().forEach((k, v) -> map.put(k, plain(v))); + return map; + } + if (value.isList()) { + final List list = new ArrayList<>(); + for (final Value item : value.asList()) { + list.add(plain(item)); + } + return list; + } + return value.asObject(); + } + + /** Equality that compares numbers by value, so 5 and 5.0 are the same. */ + private static boolean sameValue(final Object expected, final Object actual) { + if (expected instanceof Number && actual instanceof Number) { + return new BigDecimal(expected.toString()).compareTo(new BigDecimal(actual.toString())) == 0; + } + if (expected instanceof Map && actual instanceof Map) { + final Map e = (Map) expected; + final Map a = (Map) actual; + if (e.size() != a.size()) { + return false; + } + for (final Map.Entry entry : e.entrySet()) { + if (!a.containsKey(entry.getKey()) || !sameValue(entry.getValue(), a.get(entry.getKey()))) { + return false; + } + } + return true; + } + if (expected instanceof List && actual instanceof List) { + final List e = (List) expected; + final List a = (List) actual; + if (e.size() != a.size()) { + return false; + } + for (int i = 0; i < e.size(); i++) { + if (!sameValue(e.get(i), a.get(i))) { + return false; + } + } + return true; + } + return Objects.equals(expected, actual); + } + + private static String abbreviate(final Object value) { + final String text = String.valueOf(value); + return text.length() <= 200 ? text : text.substring(0, 200) + "... (" + text.length() + ")"; + } +} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java new file mode 100644 index 00000000000..274bd507764 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuiteTest.java @@ -0,0 +1,76 @@ +package com.datadog.openfeature; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Evaluates every case of the opt-in scale suites of the shared FFE fixtures, as {@code + * CanonicalFixtureTest} does for the core cases. The scale benchmarks run the same check, so a + * change that resolves a large configuration wrongly fails here instead of only in a manual run. + */ +class ScaleSuiteTest { + private static final List SUITES = Arrays.asList("scale", "scale-json"); + + /** The case files that {@code ScaleSuiteBenchmark} lists in its {@code cases} parameter. */ + private static final List BENCHMARKED_CASE_FILES = + Arrays.asList( + "scale/flags", + "scale/numeric", + "scale/one-of", + "scale/one-of-many", + "scale/one-of-small-numbers", + "scale/regex", + "scale/regex-many", + "scale/rules", + "scale/rules-sweep", + "scale/semver", + "scale/splits", + "scale/time-windows", + "scale/variations", + "scale/wide-context", + "scale-json/json-shapes", + "scale-json/json-sizes", + "scale-json/json-values"); + + static Stream caseFiles() throws IOException { + final List caseFiles = new ArrayList<>(); + for (final String suite : SUITES) { + for (final String file : ScaleSuites.load(suite).caseFiles()) { + caseFiles.add(suite + "/" + file); + } + } + return caseFiles.stream(); + } + + @Test + void benchmarkListsEveryCaseFile() throws IOException { + final List found = new ArrayList<>(); + caseFiles().forEach(found::add); + assertEquals(BENCHMARKED_CASE_FILES, found); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("caseFiles") + void evaluatesEveryCase(final String caseFile) throws IOException { + final int slash = caseFile.indexOf('/'); + final ScaleSuites.Suite suite = ScaleSuites.load(caseFile.substring(0, slash)); + final Provider provider = suite.provider(); + try { + for (final ScaleCase scaleCase : suite.cases(caseFile.substring(slash + 1))) { + // Twice: a cache or index built on the first evaluation must give the same result. + scaleCase.check(scaleCase.evaluate(provider)); + scaleCase.check(scaleCase.evaluate(provider)); + } + } finally { + provider.shutdown(); + } + } +} diff --git a/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java new file mode 100644 index 00000000000..3a454e733d6 --- /dev/null +++ b/dd-openfeature/src/test/java/com/datadog/openfeature/ScaleSuites.java @@ -0,0 +1,93 @@ +package com.datadog.openfeature; + +import com.datadog.openfeature.internal.JsonReading; +import com.datadog.openfeature.internal.bridge.v1.AgentBridge; +import com.datadog.openfeature.internal.config.Settings; +import com.datadog.openfeature.internal.ufc.UniversalFlagConfigParser; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Reads the opt-in scale suites of the ffe-system-test-data submodule, for {@code ScaleSuiteTest} + * and the scale benchmarks. + */ +final class ScaleSuites { + private static final String SUITES_PATH = + "dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data/evaluation-suites"; + + private ScaleSuites() {} + + /** One suite directory: its configuration and its case files. */ + static final class Suite { + private final Path directory; + + private Suite(final Path directory) { + this.directory = directory; + } + + byte[] config() throws IOException { + return Files.readAllBytes(directory.resolve("ufc-config.json")); + } + + /** A provider that evaluates against this suite's configuration. */ + Provider provider() throws IOException { + final DDEvaluator evaluator = + new DDEvaluator( + () -> {}, + AgentBridge.NONE, + Settings.load(AgentBridge.NONE), + ExperimentationOptOutMode.DISABLED); + evaluator.accept(UniversalFlagConfigParser.parse(config())); + return new Provider(new Provider.Options(), evaluator, false); + } + + /** The names of the case files, without the {@code .json} extension, in sorted order. */ + List caseFiles() throws IOException { + final List names = new ArrayList<>(); + try (DirectoryStream files = + Files.newDirectoryStream(directory.resolve("cases"), "*.json")) { + for (final Path file : files) { + final String name = file.getFileName().toString(); + names.add(name.substring(0, name.length() - ".json".length())); + } + } + Collections.sort(names); + return names; + } + + /** The cases of {@code cases/.json}. */ + List cases(final String name) throws IOException { + final List cases = new ArrayList<>(); + for (final Object json : + (List) + JsonReading.read( + Files.readAllBytes(directory.resolve("cases").resolve(name + ".json")))) { + cases.add(new ScaleCase((Map) json)); + } + if (cases.isEmpty()) { + throw new IllegalStateException("The case file " + name + " has no cases"); + } + return cases; + } + } + + static Suite load(final String name) { + Path directory = Paths.get("").toAbsolutePath(); + while (directory != null) { + final Path candidate = directory.resolve(SUITES_PATH).resolve(name); + if (Files.exists(candidate.resolve("ufc-config.json"))) { + return new Suite(candidate); + } + directory = directory.getParent(); + } + throw new IllegalStateException( + "Unable to find the FFE suite " + name + ": initialize the ffe-system-test-data submodule"); + } +} diff --git a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data index 6df74e605f7..ec0ec0591f0 160000 --- a/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data +++ b/dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data @@ -1 +1 @@ -Subproject commit 6df74e605f7c1eed8b397dc9aa4c002691475939 +Subproject commit ec0ec0591f0b6b1bce35aa4cdc195f9652a0a61d