[DRAFT PR]: [No Need to review]: S3 sse support final

.ci/bwcVersions

@@ -46,6 +46,7 @@ BWC_VERSION:
   - "2.19.1"
   - "2.19.2"
   - "2.19.3"
+  - "2.19.4"
   - "3.0.0"
   - "3.1.0"
   - "3.2.0"

.gitattributes

@@ -10,5 +10,6 @@
 *.bcfks binary
 *.crt binary
 *.p12 binary
+*.ttf binary
 *.txt text=auto
 CHANGELOG.md merge=union

.github/workflows/delete_backport_branch.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-    if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/')
+    if: github.repository == 'opensearch-project/OpenSearch' && (startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/'))
     steps:
     - name: Delete merged branch
       uses: actions/github-script@v7

CHANGELOG.md

@@ -7,15 +7,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Expand fetch phase profiling to support inner hits and top hits aggregation phases  ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936))
 - Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920))
+- Implement Query Rewriting Infrastructure ([#19060](https://github.com/opensearch-project/OpenSearch/pull/19060))
 - The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065))
 - Add a toBuilder method in EngineConfig to support easy modification of configs([#19054](https://github.com/opensearch-project/OpenSearch/pull/19054))
 - Add StoreFactory plugin interface for custom Store implementations([#19091](https://github.com/opensearch-project/OpenSearch/pull/19091))
+- Use S3CrtClient for higher throughput while uploading files to S3 ([#18800](https://github.com/opensearch-project/OpenSearch/pull/18800))
 - Add a dynamic setting to change skip_cache_factor and min_frequency for querycache ([#18351](https://github.com/opensearch-project/OpenSearch/issues/18351))
+- Add overload constructor for Translog to accept Channel Factory as a parameter ([#18918](https://github.com/opensearch-project/OpenSearch/pull/18918))
 
 ### Changed
 - Add CompletionStage variants to methods in the Client Interface and default to ActionListener impl ([#18998](https://github.com/opensearch-project/OpenSearch/pull/18998))
 - IllegalArgumentException when scroll ID references a node not found in Cluster ([#19031](https://github.com/opensearch-project/OpenSearch/pull/19031))
 - Adding ScriptedAvg class to painless spi to allowlist usage from plugins ([#19006](https://github.com/opensearch-project/OpenSearch/pull/19006))
+- Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
+- Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
+- Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))
@@ -25,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix flaky tests in CloseIndexIT by addressing cluster state synchronization issues ([#18878](https://github.com/opensearch-project/OpenSearch/issues/18878))
 - [Tiered Caching] Handle  query execution exception ([#19000](https://github.com/opensearch-project/OpenSearch/issues/19000))
 - Grant access to testclusters dir for tests ([#19085](https://github.com/opensearch-project/OpenSearch/issues/19085))
+- Fix assertion error when collapsing search results with concurrent segment search enabled ([#19053](https://github.com/opensearch-project/OpenSearch/pull/19053))
 - Fix skip_unavailable setting changing to default during node drop issue ([#18766](https://github.com/opensearch-project/OpenSearch/pull/18766))
 
 ### Dependencies
@@ -33,12 +40,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-cli:commons-cli` from 1.9.0 to 1.10.0 ([#19021](https://github.com/opensearch-project/OpenSearch/pull/19021))
 - Bump `org.jline:jline` from 3.30.4 to 3.30.5 ([#19013](https://github.com/opensearch-project/OpenSearch/pull/19013))
 - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.3 to 4.9.4 ([#19015](https://github.com/opensearch-project/OpenSearch/pull/19015))
-- Bump `com.azure:azure-storage-common` from 12.29.1 to 12.30.1 ([#19016](https://github.com/opensearch-project/OpenSearch/pull/19016))
+- Bump `com.azure:azure-storage-common` from 12.29.1 to 12.30.2 ([#19016](https://github.com/opensearch-project/OpenSearch/pull/19016), [#19145](https://github.com/opensearch-project/OpenSearch/pull/19145))
 - Update OpenTelemetry to 1.53.0 and OpenTelemetry SemConv to 1.34.0 ([#19068](https://github.com/opensearch-project/OpenSearch/pull/19068))
 - Bump `1password/load-secrets-action` from 2 to 3 ([#19100](https://github.com/opensearch-project/OpenSearch/pull/19100))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 10.3 to 10.4.2 ([#19099](https://github.com/opensearch-project/OpenSearch/pull/19099), [#19101](https://github.com/opensearch-project/OpenSearch/pull/19101))
 - Bump netty from 4.1.121.Final to 4.1.124.Final ([#19103](https://github.com/opensearch-project/OpenSearch/pull/19103))
-- Bump google cloud storage from 1.113.1 to 2.55.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
+- Bump Google Cloud Storage SDK from 1.113.1 to 2.55.0 ([#18922](https://github.com/opensearch-project/OpenSearch/pull/18922))
+- Bump `com.google.auth:google-auth-library-oauth2-http` from 1.37.1 to 1.38.0 ([#19144](https://github.com/opensearch-project/OpenSearch/pull/19144))
+- Bump `com.squareup.okio:okio` from 3.15.0 to 3.16.0 ([#19146](https://github.com/opensearch-project/OpenSearch/pull/19146))
+- Bump Slf4j from 1.7.36 to 2.0.17 ([#19136](https://github.com/opensearch-project/OpenSearch/pull/19136))
+- Bump `org.apache.tika` from 2.9.2 to 3.2.2 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125))
+- Bump `org.apache.commons:commons-compress` from 1.26.1 to 1.28.0 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125))
 
 ### Deprecated
 
@@ -47,4 +59,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Security
 
-[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.1...main
+[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.2...main

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.8'
+  api 'de.thetaphi:forbiddenapis:3.9'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'

buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java

@@ -36,7 +36,7 @@
  * This class models the different Docker base images that are used to build Docker distributions of OpenSearch.
  */
 public enum DockerBase {
-    CENTOS("centos:8");
+    ALMALINUX("almalinux:8");
 
     private final String image;
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java

@@ -40,7 +40,6 @@
 import org.opensearch.gradle.ExportOpenSearchBuildResourcesTask;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.util.GradleUtils;
-import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
@@ -53,6 +52,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin {
     @Override
@@ -89,10 +89,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.setClasspath(project.files(sourceSet.getRuntimeClasspath()).plus(sourceSet.getCompileClasspath()));
 
             t.setTargetCompatibility(BuildParams.getRuntimeJavaVersion().getMajorVersion());
-            if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) > 0) {
-                // TODO: forbidden apis does not yet support java 15, rethink using runtime version
-                t.setTargetCompatibility(JavaVersion.VERSION_14.getMajorVersion());
-            }
             t.setBundledSignatures(new HashSet<>(Arrays.asList("jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out")));
             t.setSignaturesFiles(
                 project.files(
@@ -140,6 +136,18 @@ public Void call(Object... names) {
                     return null;
                 }
             });
+            // Use of the deprecated security manager APIs are pervasive so set them to warn
+            // globally for all projects. Replacements for (most of) these APIs are available
+            // so usages can move to the non-deprecated variants to avoid the warnings.
+            t.setSignaturesWithSeverityWarn(
+                Set.of(
+                    "java.security.AccessController",
+                    "java.security.AccessControlContext",
+                    "java.lang.System#getSecurityManager()",
+                    "java.lang.SecurityManager",
+                    "java.security.Policy"
+                )
+            );
         });
         TaskProvider<Task> forbiddenApis = project.getTasks().named("forbiddenApis");
         forbiddenApis.configure(t -> t.setGroup(""));

client/rest/build.gradle

@@ -105,9 +105,6 @@ testingConventions {
 thirdPartyAudit {
   ignoreMissingClasses(
       'org.conscrypt.Conscrypt',
-      'org.slf4j.impl.StaticLoggerBinder',
-      'org.slf4j.impl.StaticMDCBinder',
-      'org.slf4j.impl.StaticMarkerBinder',
       //commons-logging optional dependencies
       'org.apache.avalon.framework.logger.Logger',
       'org.apache.log.Hierarchy',

client/rest/licenses/slf4j-api-2.0.17.jar.sha1

@@ -0,0 +1 @@
+d9e58ac9c7779ba3bf8142aff6c830617a7fe60f

distribution/docker/build.gradle

@@ -132,8 +132,8 @@ project.ext {
 }
 
 void addCopyDockerContextTask(Architecture architecture, DockerBase base) {
-  if (base != DockerBase.CENTOS) {
-    throw new GradleException("The only allowed docker base image for builds is CENTOS")
+  if (base != DockerBase.ALMALINUX) {
+    throw new GradleException("The only allowed docker base image for builds is ALMALINUX")
   }
 
   tasks.register(taskName("copy", architecture, base, "DockerContext"), Sync) {
@@ -181,8 +181,8 @@ opensearch_distributions {
 tasks.named("preProcessFixture").configure {
   dependsOn opensearch_distributions.docker
   // always run the task, otherwise the folders won't be created
-  outputs.upToDateWhen { 
-    false 
+  outputs.upToDateWhen {
+    false
   }
   doLast {
     // tests expect to have an empty repo
@@ -208,8 +208,8 @@ tasks.named("check").configure {
 }
 
 void addBuildDockerImage(Architecture architecture, DockerBase base) {
-  if (base != DockerBase.CENTOS) {
-    throw new GradleException("The only allowed docker base image for builds is CENTOS")
+  if (base != DockerBase.ALMALINUX) {
+    throw new GradleException("The only allowed docker base image for builds is ALMALINUX")
   }
 
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
@@ -232,9 +232,9 @@ void addBuildDockerImage(Architecture architecture, DockerBase base) {
 }
 
 for (final Architecture architecture : Architecture.values()) {
-  // We only create Docker images for the distribution on CentOS.
+  // We only create Docker images for the distribution on AlmaLinux.
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CENTOS) {
+    if (base == DockerBase.ALMALINUX) {
       addCopyDockerContextTask(architecture, base)
       addBuildDockerImage(architecture, base)
     }
@@ -257,7 +257,7 @@ subprojects { Project subProject ->
     apply plugin: 'distribution'
 
     final Architecture architecture = subProject.name.contains('arm64-') ? Architecture.ARM64 : Architecture.X64
-    final DockerBase base = DockerBase.CENTOS
+    final DockerBase base = DockerBase.ALMALINUX
 
     final String arch = architecture == Architecture.ARM64 ? '-arm64' : ''
     final String extension = 'docker.tar'

distribution/docker/docker-build-context/build.gradle

@@ -19,7 +19,7 @@ tasks.register("buildDockerBuildContext", Tar) {
   archiveClassifier = "docker-build-context"
   archiveBaseName = "opensearch"
   // Non-local builds don't need to specify an architecture.
-  with dockerBuildContext(null, DockerBase.CENTOS, false)
+  with dockerBuildContext(null, DockerBase.ALMALINUX, false)
 }
 
 tasks.named("assemble").configure { dependsOn "buildDockerBuildContext" }

distribution/docker/src/docker/Dockerfile

@@ -63,16 +63,12 @@ FROM ${base_image}
 
 ENV OPENSEARCH_CONTAINER true
 
-RUN  sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \\
-     sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* && \\
-     for iter in {1..10}; do \\
-      ${package_manager} update --setopt=tsflags=nodocs -y && \\
-      ${package_manager} install --setopt=tsflags=nodocs -y \\
-        nc shadow-utils zip unzip && \\
-      ${package_manager} clean all && exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && \\
-      sleep 10; \\
-    done; \\
-    (exit \$exit_code)
+RUN set -e \\
+     && dnf -y update \\
+     && dnf -y install --setopt=tsflags=nodocs \\
+          nmap-ncat shadow-utils zip unzip \\
+     && dnf clean all \\
+     && rm -rf /var/cache/dnf
 
 RUN groupadd -g 1000 opensearch && \\
     adduser -u 1000 -g 1000 -G 0 -d /usr/share/opensearch opensearch && \\

distribution/tools/plugin-cli/build.gradle

@@ -81,5 +81,10 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.tukaani.xz.XZOutputStream',
   'org.apache.commons.codec.digest.PureJavaCrc32C',
   'org.apache.commons.codec.digest.XXHash32',
-  'org.apache.commons.lang3.reflect.FieldUtils'
+  'org.apache.commons.lang3.reflect.FieldUtils',
+  'org.apache.commons.lang3.ArrayFill',
+  'org.apache.commons.lang3.ArrayUtils',
+  'org.apache.commons.lang3.StringUtils',
+  'org.apache.commons.lang3.SystemProperties',
+  'org.apache.commons.lang3.function.Suppliers'
 )

distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1

@@ -0,0 +1 @@
+e482f2c7a88dac3c497e96aa420b6a769f59c8d7

distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java

@@ -399,7 +399,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf
     @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
     boolean urlExists(Terminal terminal, String urlString) throws IOException {
         terminal.println(VERBOSE, "Checking if url exists: " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         assert "https".equals(url.getProtocol()) : "Use of https protocol is required";
         HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -427,7 +427,7 @@ private List<String> checkMisspelledPlugin(String pluginId) {
     @SuppressForbidden(reason = "We use getInputStream to download plugins")
     Path downloadZip(Terminal terminal, String urlString, Path tmpDir, boolean isBatch) throws IOException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
         URLConnection urlConnection = url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -684,7 +684,7 @@ InputStream getPublicKey() {
      */
     // pkg private for tests
     URL openUrl(String urlString) throws IOException {
-        URL checksumUrl = new URL(urlString);
+        URL checksumUrl = URI.create(urlString).toURL();
         HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection();
         if (connection.getResponseCode() == 404) {
             return null;

distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java

@@ -526,7 +526,7 @@ public void testSpaceInUrl() throws Exception {
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPluginUrl("fake", pluginDir);
         Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
-        try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip))) {
+        try (InputStream in = FileSystemUtils.openFileURLStream(URI.create(pluginZip).toURL())) {
             Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
         }
         installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
@@ -536,8 +536,8 @@ public void testSpaceInUrl() throws Exception {
     public void testMalformedUrlNotMaven() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         // has two colons, so it appears similar to maven coordinates
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> installPlugin("://host:1234", env.v1()));
+        assertThat(e.getMessage(), startsWith("Expected scheme name"));
     }
 
     public void testFileNotMaven() throws Exception {

gradle/libs.versions.toml

@@ -14,7 +14,7 @@ snakeyaml         = "2.1"
 icu4j             = "77.1"
 supercsv          = "2.4.0"
 log4j             = "2.21.0"
-slf4j             = "1.7.36"
+slf4j             = "2.0.17"
 asm               = "9.7"
 jettison          = "1.5.4"
 woodstox          = "6.4.0"
@@ -50,7 +50,7 @@ httpasyncclient   = "4.1.5"
 commonslogging    = "1.2"
 commonscodec      = "1.18.0"
 commonslang       = "3.18.0"
-commonscompress   = "1.26.1"
+commonscompress   = "1.28.0"
 commonsio         = "2.16.0"
 # plugin dependencies
 aws               = "2.30.31"

libs/core/src/main/java/org/opensearch/Version.java

@@ -117,6 +117,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_19_1 = new Version(2190199, org.apache.lucene.util.Version.LUCENE_9_12_1);
     public static final Version V_2_19_2 = new Version(2190299, org.apache.lucene.util.Version.LUCENE_9_12_1);
     public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_2);
+    public static final Version V_2_19_4 = new Version(2190499, org.apache.lucene.util.Version.LUCENE_9_12_2);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0);
     public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1);
     public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2);