From 68787170d9c818dafb8509d1ab83e9e61ee0f989 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 16 Oct 2025 14:38:08 -0400 Subject: [PATCH] [Backport 2.19] Onboarding new maven snapshots publishing to s3 (sql) Signed-off-by: Peter Zhu --- .github/workflows/maven-publish.yml | 13 +++++++++---- .../sql/spark/data/constants/SparkConstants.java | 2 +- .../spark/dispatcher/SparkQueryDispatcherTest.java | 2 +- build.gradle | 8 ++++---- integ-test/build.gradle | 2 +- plugin/build.gradle | 9 +++++---- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index d75ed3c89c4..c528958f6d1 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -5,8 +5,8 @@ on: push: branches: - main - - 1.* - - 2.* + - '[0-9]+.[0-9]+' + - '[0-9]+.x' jobs: build-and-publish-snapshots: @@ -32,8 +32,13 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username - SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password + MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo + MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 - name: publish snapshots to maven run: | ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index 43815a99041..6f43a59f473 100644 --- a/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -72,7 +72,7 @@ public class SparkConstants { public static final String PPL_STANDALONE_PACKAGE = "org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT"; public static final String AWS_SNAPSHOT_REPOSITORY = - "https://aws.oss.sonatype.org/content/repositories/snapshots"; + "https://ci.opensearch.org/ci/dbc/snapshots/maven/"; public static final String GLUE_HIVE_CATALOG_FACTORY_CLASS = "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory"; public static final String FLINT_DELEGATE_CATALOG = diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index 61d72773d98..01345aa7eff 100644 --- a/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -981,7 +981,7 @@ private String constructExpectedSparkSubmitParameterString( "spark.hadoop.fs.s3.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider", "spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory", "spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT", - "spark.jars.repositories=https://aws.oss.sonatype.org/content/repositories/snapshots", + "spark.jars.repositories=https://ci.opensearch.org/ci/dbc/snapshots/maven/", "spark.emr-serverless.driverEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/", "spark.executorEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/", "spark.emr-serverless.driverEnv.FLINT_CLUSTER_NAME=TEST_CLUSTER", diff --git a/build.gradle b/build.gradle index 53de0c42a4a..6f0ba155726 100644 --- a/build.gradle +++ b/build.gradle @@ -55,8 +55,8 @@ buildscript { repositories { mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() + maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } } dependencies { @@ -79,8 +79,8 @@ apply plugin: 'opensearch.java' // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available + maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } maven { url 'https://jitpack.io' } } @@ -128,9 +128,9 @@ allprojects { subprojects { repositories { mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() - maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } + maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } + maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } maven { url 'https://jitpack.io' } } } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 42bfef75a88..12ff4f5bc21 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -63,7 +63,7 @@ ext { noticeFile = rootProject.file('NOTICE') getSecurityPluginDownloadLink = { -> - var repo = "https://central.sonatype.com/repository/maven-snapshots/org/opensearch/plugin/" + + var repo = "https://ci.opensearch.org/ci/dbc/snapshots/maven/org/opensearch/plugin/" + "opensearch-security/$opensearch_build_snapshot/" var metadataFile = Paths.get(projectDir.toString(), "build", "maven-metadata.xml").toAbsolutePath().toFile() download.run { diff --git a/plugin/build.gradle b/plugin/build.gradle index bbabf971847..b5a5e9013f3 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -80,10 +80,11 @@ publishing { repositories { maven { name = "Snapshots" // optional target repository name - url = "https://central.sonatype.com/repository/maven-snapshots/" - credentials { - username "$System.env.SONATYPE_USERNAME" - password "$System.env.SONATYPE_PASSWORD" + url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO") + credentials(AwsCredentials) { + accessKey = System.getenv("AWS_ACCESS_KEY_ID") + secretKey = System.getenv("AWS_SECRET_ACCESS_KEY") + sessionToken = System.getenv("AWS_SESSION_TOKEN") } } }