Skip to content

Commit eed12d1

Browse files
szybiaszybia
committed
Merge remote-tracking branch 'upstream/main' into security-stats-with-dls
* upstream/main: (100 commits) ES|QL: Add FUSE operator tests (elastic#135307) [D0CS] Revise connector setup steps in documentation (elastic#135426) Fix DiscoveryDisruptionIT.testElectMasterWithLatestVersion (elastic#135396) [DOCS] Marks the change point agg as GA (elastic#134898) Rework ShardSearchContextId to explain use of searcher id better (elastic#135233) [CI] Handle caching bwc dependencies more gracefully (elastic#135417) Mute org.elasticsearch.gradle.TestClustersPluginFuncTest override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities elastic#135413 [Build] update eclipse formatter used by spotless (elastic#135382) [Test] Fix typo in build tool tests (elastic#135405) Fixes testSnapshotShutdownProgressTracker (elastic#134926) Mute org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT testLogsIndexing {upgradedNodes=1} elastic#135313 OTLP: remove feature flag (elastic#135401) [Docs] Convert asciidoc lifecycle markers into Docs V3 syntax (elastic#135347) Mute org.elasticsearch.upgrades.QueryableBuiltInRolesUpgradeIT testBuiltInRolesSyncedOnClusterUpgrade elastic#135194 Mute org.elasticsearch.upgrades.IndexingIT testIndexing elastic#135407 Mute org.elasticsearch.upgrades.DataStreamsUpgradeIT testDataStreamValidationDoesNotBreakUpgrade elastic#135406 [CI] Handle git snapshot BWC versions correctly when calculating jdk fallback (elastic#135399) [Build] Update checkstyle from 10.3 to 11.0.1 (elastic#135381) Mute org.elasticsearch.upgrades.TextRollingUpgradeIT testIndexing {upgradedNodes=2} elastic#135238 Mute org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT testIndexing {upgradedNodes=2} elastic#135325 ...
2 parents 61d6bee + c94f366 commit eed12d1

File tree

393 files changed

+12007
-3947
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

393 files changed

+12007
-3947
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.bytes;
11+
12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
14+
import org.elasticsearch.common.recycler.Recycler;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
26+
import java.io.IOException;
27+
import java.util.concurrent.ThreadLocalRandom;
28+
import java.util.concurrent.TimeUnit;
29+
import java.util.concurrent.atomic.AtomicReference;
30+
31+
@Warmup(iterations = 3)
32+
@Measurement(iterations = 3)
33+
@BenchmarkMode(Mode.AverageTime)
34+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
35+
@State(Scope.Thread)
36+
@Fork(value = 1)
37+
public class RecyclerBytesStreamOutputBenchmark {
38+
39+
private final AtomicReference<BytesRef> bytesRef = new AtomicReference<>(new BytesRef(16384));
40+
private RecyclerBytesStreamOutput streamOutput;
41+
private String shortString;
42+
private String longString;
43+
private String nonAsciiString;
44+
private String veryLongString;
45+
private byte[] bytes1;
46+
private byte[] bytes2;
47+
private byte[] bytes3;
48+
private byte[] multiPageBytes;
49+
private int[] vints;
50+
51+
@Setup
52+
public void initResults() throws IOException {
53+
streamOutput = new RecyclerBytesStreamOutput(new BenchmarkRecycler(bytesRef));
54+
ThreadLocalRandom random = ThreadLocalRandom.current();
55+
56+
bytes1 = new byte[327];
57+
bytes2 = new byte[712];
58+
bytes3 = new byte[1678];
59+
multiPageBytes = new byte[16387 * 4];
60+
random.nextBytes(bytes1);
61+
random.nextBytes(bytes2);
62+
random.nextBytes(bytes3);
63+
random.nextBytes(multiPageBytes);
64+
65+
// We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
66+
// impact direct comparisons run-to-run
67+
68+
shortString = generateAsciiString(20);
69+
longString = generateAsciiString(100);
70+
nonAsciiString = generateUtf8String(200);
71+
veryLongString = generateAsciiString(800);
72+
// vint values for benchmarking
73+
vints = new int[1000];
74+
for (int i = 0; i < vints.length; i++) {
75+
if (random.nextBoolean()) {
76+
// 1-byte 50% of the time
77+
vints[i] = random.nextInt(128);
78+
} else if (random.nextBoolean()) {
79+
// 2-byte 25% of the time
80+
vints[i] = random.nextInt(128, 16384);
81+
} else {
82+
if (random.nextBoolean()) {
83+
// 3-byte vints
84+
vints[i] = random.nextInt(16384, 2097152);
85+
} else {
86+
// All vint variants
87+
vints[i] = random.nextInt();
88+
}
89+
}
90+
}
91+
}
92+
93+
@Benchmark
94+
public void writeByte() throws IOException {
95+
streamOutput.seek(1);
96+
for (byte item : bytes1) {
97+
streamOutput.writeByte(item);
98+
}
99+
for (byte item : bytes2) {
100+
streamOutput.writeByte(item);
101+
}
102+
for (byte item : bytes3) {
103+
streamOutput.writeByte(item);
104+
}
105+
}
106+
107+
@Benchmark
108+
public void writeBytes() throws IOException {
109+
streamOutput.seek(1);
110+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
111+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
112+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
113+
}
114+
115+
@Benchmark
116+
public void writeBytesAcrossPageBoundary() throws IOException {
117+
streamOutput.seek(16384 - 1000);
118+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
119+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
120+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
121+
}
122+
123+
@Benchmark
124+
public void writeBytesMultiPage() throws IOException {
125+
streamOutput.seek(16384 - 1000);
126+
streamOutput.writeBytes(multiPageBytes, 0, multiPageBytes.length);
127+
}
128+
129+
@Benchmark
130+
public void writeString() throws IOException {
131+
streamOutput.seek(1);
132+
streamOutput.writeString(shortString);
133+
streamOutput.writeString(longString);
134+
streamOutput.writeString(nonAsciiString);
135+
streamOutput.writeString(veryLongString);
136+
}
137+
138+
@Benchmark
139+
public void writeVInt() throws IOException {
140+
streamOutput.seek(1);
141+
for (int vint : vints) {
142+
streamOutput.writeVInt(vint);
143+
}
144+
}
145+
146+
public static String generateAsciiString(int n) {
147+
ThreadLocalRandom random = ThreadLocalRandom.current();
148+
StringBuilder sb = new StringBuilder(n);
149+
150+
for (int i = 0; i < n; i++) {
151+
int ascii = random.nextInt(128);
152+
sb.append((char) ascii);
153+
}
154+
155+
return sb.toString();
156+
}
157+
158+
public static String generateUtf8String(int n) {
159+
ThreadLocalRandom random = ThreadLocalRandom.current();
160+
StringBuilder sb = new StringBuilder(n);
161+
162+
for (int i = 0; i < n; i++) {
163+
int codePoint;
164+
int probability = random.nextInt(100);
165+
166+
if (probability < 85) {
167+
// 1-byte UTF-8 (ASCII range)
168+
// 0x0000 to 0x007F
169+
codePoint = random.nextInt(0x0080);
170+
} else if (probability < 95) {
171+
// 2-byte UTF-8
172+
// 0x0080 to 0x07FF
173+
codePoint = random.nextInt(0x0080, 0x0800);
174+
} else {
175+
// 3-byte UTF-8
176+
// 0x0800 to 0xFFFF
177+
do {
178+
codePoint = random.nextInt(0x0800, 0x10000);
179+
// Skip surrogate pairs (0xD800-0xDFFF)
180+
} while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
181+
}
182+
183+
sb.appendCodePoint(codePoint);
184+
}
185+
186+
return sb.toString();
187+
}
188+
189+
private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
190+
191+
@Override
192+
public V<BytesRef> obtain() {
193+
BytesRef recycledBytesRef = bytesRef.getAndSet(null);
194+
final BytesRef localBytesRef;
195+
final boolean recycled;
196+
if (recycledBytesRef != null) {
197+
recycled = true;
198+
localBytesRef = recycledBytesRef;
199+
} else {
200+
recycled = false;
201+
localBytesRef = new BytesRef(16384);
202+
}
203+
return new V<>() {
204+
@Override
205+
public BytesRef v() {
206+
return localBytesRef;
207+
}
208+
209+
@Override
210+
public boolean isRecycled() {
211+
return recycled;
212+
}
213+
214+
@Override
215+
public void close() {
216+
if (recycled) {
217+
bytesRef.set(localBytesRef);
218+
}
219+
}
220+
};
221+
}
222+
223+
@Override
224+
public int pageSize() {
225+
return 16384;
226+
}
227+
}
228+
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,23 @@ public void apply(Project project) {
5252
// Spotless resolves required dependencies from project repositories, so we need maven central
5353
project.getRepositories().mavenCentral();
5454

55+
// we cannot update to a newer spotless plugin version yet but we want to use the latest eclipse formatter to be compatible
56+
// with latest java versions
57+
project.getConfigurations().matching(it -> it.getName().startsWith("spotless")).configureEach(conf -> {
58+
project.getDependencies().constraints(constraints -> {
59+
constraints.add(conf.getName(), "org.eclipse.jdt:org.eclipse.jdt.core:3.42.0", dependencyConstraint -> {
60+
dependencyConstraint.because(
61+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
62+
);
63+
});
64+
constraints.add(conf.getName(), "org.eclipse.jdt:ecj:3.42.0", dependencyConstraint -> {
65+
dependencyConstraint.because(
66+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
67+
);
68+
});
69+
});
70+
});
71+
5572
project.getExtensions().getByType(SpotlessExtension.class).java(java -> {
5673
File elasticsearchWorkspace = Util.locateElasticsearchWorkspace(project.getGradle());
5774
String importOrderPath = "build-conventions/elastic.importorder";
@@ -74,7 +91,7 @@ public void apply(Project project) {
7491
// When running build benchmarks we alter the source in some scenarios.
7592
// The gradle-profiler unfortunately does not generate compliant formatted
7693
// sources so we ignore that altered file when running build benchmarks
77-
if(Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
94+
if (Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
7895
java.targetExclude("src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java");
7996
}
8097
});

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
9090
assertUpperBound("9.2", "new_tv,8124000")
9191
}
9292

93+
/*
94+
temporarily muted, see https://github.com/elastic/elasticsearch/pull/135226
95+
9396
def "invalid changes to a upper bounds should be reverted"() {
9497
given:
9598
transportVersionUpperBound("9.2", "modification", "9000000")
@@ -144,7 +147,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
144147
assertReferableDefinitionDoesNotExist("test_tv")
145148
assertUpperBound("9.2", "existing_92,8123000")
146149
assertUpperBound("9.1", "existing_92,8012001")
147-
}
150+
}*/
148151

149152
def "a reference can be renamed"() {
150153
given:
@@ -242,8 +245,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
242245
def "unreferenced definitions are removed"() {
243246
given:
244247
referableTransportVersion("test_tv", "8124000,8012002")
248+
/*
249+
TODO: reset of upper bounds
245250
transportVersionUpperBound("9.2", "test_tv", "8124000")
246251
transportVersionUpperBound("9.1", "test_tv", "8012002")
252+
*/
247253

248254
when:
249255
def result = runGenerateAndValidateTask().build()
@@ -406,6 +412,8 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
406412
assertUpperBound("9.2", "new_tv,8124000")
407413
}
408414

415+
/*
416+
TODO: reset of upper bounds
409417
def "deleted upper bounds files are restored"() {
410418
given:
411419
file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete()
@@ -416,7 +424,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
416424
then:
417425
assertGenerateAndValidateSuccess(result)
418426
assertUpperBound("9.2", "existing_92,8123000")
419-
}
427+
}*/
420428

421429
def "upper bounds files must exist for backport branches"() {
422430
when:

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
113113
String versionProperty = System.getProperty("tests.bwc.main.version");
114114
// We use this phony version as a placeholder for the real version
115115
if (distribution.getVersion().equals("0.0.0")) {
116+
if (versionProperty == null) {
117+
throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
118+
}
116119
BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
117120
Version.fromString(versionProperty),
118121
"main",

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,14 @@
4444
import org.gradle.api.file.FileCollection;
4545
import org.gradle.api.file.FileTree;
4646
import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
47+
import org.gradle.api.plugins.JvmToolchainsPlugin;
4748
import org.gradle.api.provider.ProviderFactory;
4849
import org.gradle.api.tasks.ClasspathNormalizer;
4950
import org.gradle.api.tasks.PathSensitivity;
5051
import org.gradle.api.tasks.util.PatternFilterable;
52+
import org.gradle.jvm.toolchain.JavaLanguageVersion;
53+
import org.gradle.jvm.toolchain.JavaToolchainService;
54+
import org.gradle.jvm.toolchain.JvmVendorSpec;
5155

5256
import java.util.Collection;
5357
import java.util.Iterator;
@@ -59,6 +63,7 @@
5963
import javax.inject.Inject;
6064

6165
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
66+
import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
6267

6368
/**
6469
* Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -94,6 +99,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) {
9499
public void apply(Project project) {
95100
project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
96101
project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
102+
project.getPluginManager().apply(JvmToolchainsPlugin.class);
97103
var bwcVersions = loadBuildParams(project).get().getBwcVersions();
98104

99105
// Register integ-test and default distributions
@@ -236,6 +242,17 @@ public Void call(Object... args) {
236242
String versionString = version.toString();
237243
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
238244

245+
if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
246+
var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
247+
var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
248+
spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
249+
spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
250+
});
251+
task.environment(
252+
"ES_FALLBACK_JAVA_HOME",
253+
fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
254+
);
255+
}
239256
task.dependsOn(bwcDistro);
240257
registerDistributionInputs(task, bwcDistro);
241258

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ public void run() throws IOException {
103103

104104
getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
105105
if (targetDefinitionName.isEmpty()) {
106-
resetAllUpperBounds(resources);
106+
// TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
107+
// resetAllUpperBounds(resources);
107108
} else {
108109
List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
109110
// (Re)write the definition file.

0 commit comments

Comments
 (0)