From 17b403864a8f9647d790f6e317394c583a959ef6 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 18 Jul 2025 09:04:14 -0400 Subject: [PATCH 01/18] Run GeoIp yaml tests in multi-project cluster --- .../qa/multi-project/build.gradle | 17 ++ .../ingest}/geoip/GeoIpMultiProjectIT.java | 4 +- ...eoIpClientMultiProjectYamlTestSuiteIT.java | 136 ++++++++++++++ .../ingest/geoip/DatabaseNodeService.java | 174 +++++++++--------- 4 files changed, 243 insertions(+), 88 deletions(-) rename modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/{ => org/elasticsearch/ingest}/geoip/GeoIpMultiProjectIT.java (97%) create mode 100644 modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index bcfe4f22599ac..2514b2c641897 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -8,15 +8,25 @@ */ apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' dependencies { javaRestTestImplementation project(':modules:ingest-geoip') javaRestTestImplementation project(':test:external-modules:test-multi-project') javaRestTestImplementation project(':test:fixtures:geoip-fixture') + yamlRestTestImplementation project(':modules:ingest-geoip') + yamlRestTestImplementation project(':test:external-modules:test-multi-project') + yamlRestTestImplementation project(':test:fixtures:geoip-fixture') + yamlRestTestImplementation project(':x-pack:qa:multi-project:yaml-test-framework') + clusterModules project(':modules:ingest-geoip') clusterModules project(':modules:reindex') // needed for database cleanup clusterModules project(':test:external-modules:test-multi-project') + + // includes yaml rest test artifacts from ingest-geoip module + restTestConfig project(path: ':modules:ingest-geoip', configuration: "restTests") } tasks.withType(Test).configureEach { @@ -27,3 +37,10 @@ tasks.withType(Test).configureEach { tasks.named { it == "javaRestTest" || it == "yamlRestTest" }.configureEach { it.onlyIf("snapshot build") { buildParams.snapshotBuild } } + +restResources { + restTests { + /// includes yaml rest test from ingest_geoip folder + includeCore 'ingest_geoip' + } +} diff --git a/modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/geoip/GeoIpMultiProjectIT.java b/modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpMultiProjectIT.java similarity index 97% rename from modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/geoip/GeoIpMultiProjectIT.java rename to modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpMultiProjectIT.java index 1ed4c9e2172de..2472a0e671d6b 100644 --- a/modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/geoip/GeoIpMultiProjectIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/javaRestTest/java/org/elasticsearch/ingest/geoip/GeoIpMultiProjectIT.java @@ -7,15 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package geoip; +package org.elasticsearch.ingest.geoip; import fixture.geoip.GeoIpHttpFixture; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.core.Booleans; -import org.elasticsearch.ingest.geoip.GeoIpDownloader; -import org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java new file mode 100644 index 0000000000000..703a0e6968976 --- /dev/null +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import fixture.geoip.GeoIpHttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.multiproject.test.MultipleProjectsClientYamlSuiteTestCase; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { + + private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false; + + private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("reindex") + .module("ingest-geoip") + .systemProperty("ingest.geoip.downloader.enabled.default", "true") + // sets the plain (geoip.elastic.co) downloader endpoint, which is used in these tests + .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + // also sets the enterprise downloader maxmind endpoint, to make sure we do not accidentally hit the real endpoint from tests + // note: it's not important that the downloading actually work at this point -- the rest tests (so far) don't exercise + // the downloading code because of license reasons -- but if they did, then it would be important that we're hitting a fixture + .systemProperty("ingest.geoip.downloader.maxmind.endpoint.default", () -> fixture.getAddress(), s -> useFixture) + .setting("test.multi_project.enabled", "true") + .setting("xpack.license.self_generated.type", "trial") + .user(USER, PASS) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public IngestGeoIpClientMultiProjectYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Before + public void waitForDatabases() throws Exception { + putGeoipPipeline(); + assertBusy(() -> { + Request request = new Request("GET", "/_ingest/geoip/stats"); + Map response = entityAsMap(client().performRequest(request)); + // assert databases are downloaded + Map downloadStats = (Map) response.get("stats"); + assertThat(downloadStats.get("databases_count"), equalTo(4)); + // assert databases are loaded to node + Map nodes = (Map) response.get("nodes"); + assertThat(nodes.size(), equalTo(1)); + Map node = (Map) nodes.values().iterator().next(); + List databases = ((List) node.get("databases")); + assertThat(databases, notNullValue()); + List databaseNames = databases.stream().map(o -> (String) ((Map) o).get("name")).toList(); + assertThat( + databaseNames, + containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb") + ); + }, 10, TimeUnit.SECONDS); + } + + /** + * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. + * @throws IOException + */ + private void putGeoipPipeline() throws IOException { + final BytesReference bytes; + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + { + builder.startArray("processors"); + { + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", "ip"); + builder.field("target_field", "ip-city"); + builder.field("database_file", "GeoLite2-City.mmdb"); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + bytes = BytesReference.bytes(builder); + } + Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); + putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); + client().performRequest(putPipelineRequest); + } + +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 6de92f373e648..dd5c01bbeaf37 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -289,103 +290,106 @@ void checkDatabases(ClusterState state) { // Optimization: only load the .geoip_databases for projects that are allocated to this node for (ProjectMetadata projectMetadata : state.metadata().projects().values()) { - ProjectId projectId = projectMetadata.id(); + checkDatabases(state, projectMetadata); + } + } - PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); - if (persistentTasks == null) { - logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); - continue; - } + void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { + ProjectId projectId = projectMetadata.id(); + PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); + if (persistentTasks == null) { + logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); + return; + } - IndexAbstraction databasesAbstraction = projectMetadata.getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX); - if (databasesAbstraction == null) { - logger.trace("Not checking databases because geoip databases index does not exist for project [{}]", projectId); + IndexAbstraction databasesAbstraction = projectMetadata.getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX); + if (databasesAbstraction == null) { + logger.trace("Not checking databases because geoip databases index does not exist for project [{}]", projectId); + return; + } else { + // regardless of whether DATABASES_INDEX is an alias, resolve it to a concrete index + Index databasesIndex = databasesAbstraction.getWriteIndex(); + IndexRoutingTable databasesIndexRT = state.routingTable(projectId).index(databasesIndex); + if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { + logger.trace( + "Not checking databases because geoip databases index does not have all active primary shards for" + + " project [{}]", + projectId + ); return; - } else { - // regardless of whether DATABASES_INDEX is an alias, resolve it to a concrete index - Index databasesIndex = databasesAbstraction.getWriteIndex(); - IndexRoutingTable databasesIndexRT = state.routingTable(projectId).index(databasesIndex); - if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { - logger.trace( - "Not checking databases because geoip databases index does not have all active primary shards for" - + " project [{}]", - projectId - ); - return; - } } + } - // we'll consult each of the geoip downloaders to build up a list of database metadatas to work with - List> validMetadatas = new ArrayList<>(); + // we'll consult each of the geoip downloaders to build up a list of database metadatas to work with + List> validMetadatas = new ArrayList<>(); - // process the geoip task state for the (ordinary) geoip downloader - { - GeoIpTaskState taskState = getGeoIpTaskState( - projectMetadata, - getTaskId(projectId, projectResolver.supportsMultipleProjects()) - ); - if (taskState == null) { - // Note: an empty state will purge stale entries in databases map - taskState = GeoIpTaskState.EMPTY; - } - validMetadatas.addAll( - taskState.getDatabases() - .entrySet() - .stream() - .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) - .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) - .toList() - ); + // process the geoip task state for the (ordinary) geoip downloader + { + GeoIpTaskState taskState = getGeoIpTaskState( + projectMetadata, + getTaskId(projectId, projectResolver.supportsMultipleProjects()) + ); + if (taskState == null) { + // Note: an empty state will purge stale entries in databases map + taskState = GeoIpTaskState.EMPTY; } + validMetadatas.addAll( + taskState.getDatabases() + .entrySet() + .stream() + .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList() + ); + } - // process the geoip task state for the enterprise geoip downloader - { - EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(state); - if (taskState == null) { - // Note: an empty state will purge stale entries in databases map - taskState = EnterpriseGeoIpTaskState.EMPTY; - } - validMetadatas.addAll( - taskState.getDatabases() - .entrySet() - .stream() - .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) - .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) - .toList() - ); + // process the geoip task state for the enterprise geoip downloader + { + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(state); + if (taskState == null) { + // Note: an empty state will purge stale entries in databases map + taskState = EnterpriseGeoIpTaskState.EMPTY; } + validMetadatas.addAll( + taskState.getDatabases() + .entrySet() + .stream() + .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList() + ); + } - // run through all the valid metadatas, regardless of source, and retrieve them if the persistent downloader task - // has downloaded a new version of the databases - validMetadatas.forEach(e -> { - String name = e.v1(); - GeoIpTaskState.Metadata metadata = e.v2(); - DatabaseReaderLazyLoader reference = getProjectLazyLoader(projectId, name); - String remoteMd5 = metadata.md5(); - String localMd5 = reference != null ? reference.getMd5() : null; - if (Objects.equals(localMd5, remoteMd5)) { - logger.debug("[{}] is up to date [{}] with cluster state [{}]", name, localMd5, remoteMd5); - return; - } + // run through all the valid metadatas, regardless of source, and retrieve them if the persistent downloader task + // has downloaded a new version of the databases + validMetadatas.forEach(e -> { + String name = e.v1(); + GeoIpTaskState.Metadata metadata = e.v2(); + DatabaseReaderLazyLoader reference = getProjectLazyLoader(projectId, name); + String remoteMd5 = metadata.md5(); + String localMd5 = reference != null ? reference.getMd5() : null; + if (Objects.equals(localMd5, remoteMd5)) { + logger.debug("[{}] is up to date [{}] with cluster state [{}]", name, localMd5, remoteMd5); + return; + } - try { - retrieveAndUpdateDatabase(projectId, name, metadata); - } catch (Exception ex) { - logger.error(() -> "failed to retrieve database [" + name + "]", ex); - } - }); - - // TODO perhaps we need to handle the license flap persistent task state better than we do - // i think the ideal end state is that we *do not* drop the files that the enterprise downloader - // handled if they fall out -- which means we need to track that in the databases map itself - - // start with the list of all databases we currently know about in this service, - // then drop the ones that didn't check out as valid from the task states - if (databases.containsKey(projectId)) { - Set staleDatabases = new HashSet<>(databases.get(projectId).keySet()); - staleDatabases.removeAll(validMetadatas.stream().map(Tuple::v1).collect(Collectors.toSet())); - removeStaleEntries(projectId, staleDatabases); + try { + retrieveAndUpdateDatabase(projectId, name, metadata); + } catch (Exception ex) { + logger.error(() -> "failed to retrieve database [" + name + "]", ex); } + }); + + // TODO perhaps we need to handle the license flap persistent task state better than we do + // i think the ideal end state is that we *do not* drop the files that the enterprise downloader + // handled if they fall out -- which means we need to track that in the databases map itself + + // start with the list of all databases we currently know about in this service, + // then drop the ones that didn't check out as valid from the task states + if (databases.containsKey(projectId)) { + Set staleDatabases = new HashSet<>(databases.get(projectId).keySet()); + staleDatabases.removeAll(validMetadatas.stream().map(Tuple::v1).collect(Collectors.toSet())); + removeStaleEntries(projectId, staleDatabases); } } From 1f9ab3c4ca5b20146815d4581e25193d6ad361e8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 18 Jul 2025 13:18:16 +0000 Subject: [PATCH 02/18] [CI] Auto commit changes from spotless --- .../elasticsearch/ingest/geoip/DatabaseNodeService.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index dd5c01bbeaf37..9e67d4809eeb5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -16,7 +16,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -312,8 +311,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { IndexRoutingTable databasesIndexRT = state.routingTable(projectId).index(databasesIndex); if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { logger.trace( - "Not checking databases because geoip databases index does not have all active primary shards for" - + " project [{}]", + "Not checking databases because geoip databases index does not have all active primary shards for" + " project [{}]", projectId ); return; @@ -325,10 +323,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { // process the geoip task state for the (ordinary) geoip downloader { - GeoIpTaskState taskState = getGeoIpTaskState( - projectMetadata, - getTaskId(projectId, projectResolver.supportsMultipleProjects()) - ); + GeoIpTaskState taskState = getGeoIpTaskState(projectMetadata, getTaskId(projectId, projectResolver.supportsMultipleProjects())); if (taskState == null) { // Note: an empty state will purge stale entries in databases map taskState = GeoIpTaskState.EMPTY; From e3a9314fcca25c2618f04dc193b7ce39e7d98061 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 18 Jul 2025 16:45:24 -0400 Subject: [PATCH 03/18] minor --- modules/ingest-geoip/qa/multi-project/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index 2514b2c641897..25b5ba72f4dd4 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -40,7 +40,7 @@ tasks.named { it == "javaRestTest" || it == "yamlRestTest" }.configureEach { restResources { restTests { - /// includes yaml rest test from ingest_geoip folder + // includes yaml rest test from ingest_geoip folder includeCore 'ingest_geoip' } } From 4ed662ecf776e964b326e0816096148c6bcb3fb9 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 25 Jul 2025 12:14:45 -0400 Subject: [PATCH 04/18] Use ProjectState --- .../ingest/geoip/DatabaseNodeService.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 9e67d4809eeb5..ac0f4152b662c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -288,13 +289,15 @@ void checkDatabases(ClusterState state) { } // Optimization: only load the .geoip_databases for projects that are allocated to this node - for (ProjectMetadata projectMetadata : state.metadata().projects().values()) { - checkDatabases(state, projectMetadata); + for (ProjectId projectId : state.metadata().projects().keySet()) { + checkDatabases(state.projectState(projectId)); } } - void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { - ProjectId projectId = projectMetadata.id(); + void checkDatabases(ProjectState projectState) { + ProjectId projectId = projectState.projectId(); + ProjectMetadata projectMetadata = projectState.metadata(); + ClusterState clusterState = clusterService.state(); PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks == null) { logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); @@ -308,7 +311,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { } else { // regardless of whether DATABASES_INDEX is an alias, resolve it to a concrete index Index databasesIndex = databasesAbstraction.getWriteIndex(); - IndexRoutingTable databasesIndexRT = state.routingTable(projectId).index(databasesIndex); + IndexRoutingTable databasesIndexRT = clusterState.routingTable(projectId).index(databasesIndex); if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { logger.trace( "Not checking databases because geoip databases index does not have all active primary shards for" + " project [{}]", @@ -332,7 +335,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .filter(e -> e.getValue().isNewEnough(clusterState.getMetadata().settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); @@ -340,7 +343,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { // process the geoip task state for the enterprise geoip downloader { - EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(state); + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(clusterState); if (taskState == null) { // Note: an empty state will purge stale entries in databases map taskState = EnterpriseGeoIpTaskState.EMPTY; @@ -349,7 +352,7 @@ void checkDatabases(ClusterState state, ProjectMetadata projectMetadata) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .filter(e -> e.getValue().isNewEnough(clusterState.getMetadata().settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); From 5f651a3c53715843a12493d5d536c59bab21ef64 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 25 Jul 2025 12:40:35 -0400 Subject: [PATCH 05/18] share yaml test code WIP --- modules/ingest-geoip/build.gradle | 9 +++ .../qa/multi-project/build.gradle | 6 +- ...eoIpClientMultiProjectYamlTestSuiteIT.java | 59 ++++++++++--------- .../IngestGeoIpClientYamlTestSuiteIT.java | 6 +- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index bdda4872fb4dc..0e1145a51d602 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -85,7 +85,16 @@ tasks.named("dependencyLicenses").configure { ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt' } +configurations { + yamlRestImplementation { + attributes { + attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JVM_CLASS_DIRECTORY) + } + } +} + artifacts { + yamlRestImplementation(new File(projectDir, "src/yamlRestTest/java")) restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index 25b5ba72f4dd4..a97a15fe4f24d 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -9,14 +9,18 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.yaml-rest-compat-test' +//apply plugin: 'elasticsearch.yaml-rest-compat-test' dependencies { + yamlRestTestImplementation project(path: ':modules:ingest-geoip', configuration: 'yamlRestImplementation') javaRestTestImplementation project(':modules:ingest-geoip') javaRestTestImplementation project(':test:external-modules:test-multi-project') javaRestTestImplementation project(':test:fixtures:geoip-fixture') +// testImplementation project(':modules:ingest-geoip') yamlRestTestImplementation project(':modules:ingest-geoip') + +// yamlRestTestImplementation(testArtifact(project(":modules:ingest-geoip"))) yamlRestTestImplementation project(':test:external-modules:test-multi-project') yamlRestTestImplementation project(':test:fixtures:geoip-fixture') yamlRestTestImplementation project(':x-pack:qa:multi-project:yaml-test-framework') diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java index 703a0e6968976..b03474b796866 100644 --- a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline; public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { @@ -79,7 +80,7 @@ public static Iterable parameters() throws Exception { @Before public void waitForDatabases() throws Exception { - putGeoipPipeline(); + putGeoIpPipeline(); assertBusy(() -> { Request request = new Request("GET", "/_ingest/geoip/stats"); Map response = entityAsMap(client().performRequest(request)); @@ -104,33 +105,33 @@ public void waitForDatabases() throws Exception { * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. * @throws IOException */ - private void putGeoipPipeline() throws IOException { - final BytesReference bytes; - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.startObject(); - { - builder.startArray("processors"); - { - builder.startObject(); - { - builder.startObject("geoip"); - { - builder.field("field", "ip"); - builder.field("target_field", "ip-city"); - builder.field("database_file", "GeoLite2-City.mmdb"); - } - builder.endObject(); - } - builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - bytes = BytesReference.bytes(builder); - } - Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); - putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); - client().performRequest(putPipelineRequest); - } +// private void putGeoipPipeline() throws IOException { +// final BytesReference bytes; +// try (XContentBuilder builder = JsonXContent.contentBuilder()) { +// builder.startObject(); +// { +// builder.startArray("processors"); +// { +// builder.startObject(); +// { +// builder.startObject("geoip"); +// { +// builder.field("field", "ip"); +// builder.field("target_field", "ip-city"); +// builder.field("database_file", "GeoLite2-City.mmdb"); +// } +// builder.endObject(); +// } +// builder.endObject(); +// } +// builder.endArray(); +// } +// builder.endObject(); +// bytes = BytesReference.bytes(builder); +// } +// Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); +// putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); +// client().performRequest(putPipelineRequest); +// } } diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index f7943342b5352..47c9cdfb1c303 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -78,10 +78,10 @@ public void waitForDatabases() throws Exception { assertBusy(() -> { Request request = new Request("GET", "/_ingest/geoip/stats"); Map response = entityAsMap(client().performRequest(request)); - + // assert databases are downloaded Map downloadStats = (Map) response.get("stats"); assertThat(downloadStats.get("databases_count"), equalTo(4)); - + // assert databases are loaded to node Map nodes = (Map) response.get("nodes"); assertThat(nodes.size(), equalTo(1)); Map node = (Map) nodes.values().iterator().next(); @@ -99,7 +99,7 @@ public void waitForDatabases() throws Exception { * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. * @throws IOException */ - private void putGeoipPipeline() throws IOException { + public static void putGeoipPipeline() throws IOException { final BytesReference bytes; try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.startObject(); From d4a08b3e9dc3c312b72da405eb76ee65f4e6e34f Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 25 Jul 2025 13:29:07 -0400 Subject: [PATCH 06/18] use projectState.cluster() --- .../org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index ac0f4152b662c..038438becce21 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -297,7 +297,7 @@ void checkDatabases(ClusterState state) { void checkDatabases(ProjectState projectState) { ProjectId projectId = projectState.projectId(); ProjectMetadata projectMetadata = projectState.metadata(); - ClusterState clusterState = clusterService.state(); + ClusterState clusterState = projectState.cluster(); PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks == null) { logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); From 92935c881c9360ed71112d939db742506229f502 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Fri, 25 Jul 2025 13:29:31 -0400 Subject: [PATCH 07/18] Revert "share yaml test code WIP" This reverts commit 5f651a3c53715843a12493d5d536c59bab21ef64. --- modules/ingest-geoip/build.gradle | 9 --- .../qa/multi-project/build.gradle | 6 +- ...eoIpClientMultiProjectYamlTestSuiteIT.java | 59 +++++++++---------- .../IngestGeoIpClientYamlTestSuiteIT.java | 6 +- 4 files changed, 33 insertions(+), 47 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 0e1145a51d602..bdda4872fb4dc 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -85,16 +85,7 @@ tasks.named("dependencyLicenses").configure { ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt' } -configurations { - yamlRestImplementation { - attributes { - attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JVM_CLASS_DIRECTORY) - } - } -} - artifacts { - yamlRestImplementation(new File(projectDir, "src/yamlRestTest/java")) restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index a97a15fe4f24d..25b5ba72f4dd4 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -9,18 +9,14 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-yaml-rest-test' -//apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' dependencies { - yamlRestTestImplementation project(path: ':modules:ingest-geoip', configuration: 'yamlRestImplementation') javaRestTestImplementation project(':modules:ingest-geoip') javaRestTestImplementation project(':test:external-modules:test-multi-project') javaRestTestImplementation project(':test:fixtures:geoip-fixture') -// testImplementation project(':modules:ingest-geoip') yamlRestTestImplementation project(':modules:ingest-geoip') - -// yamlRestTestImplementation(testArtifact(project(":modules:ingest-geoip"))) yamlRestTestImplementation project(':test:external-modules:test-multi-project') yamlRestTestImplementation project(':test:fixtures:geoip-fixture') yamlRestTestImplementation project(':x-pack:qa:multi-project:yaml-test-framework') diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java index b03474b796866..703a0e6968976 100644 --- a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -38,7 +38,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline; public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { @@ -80,7 +79,7 @@ public static Iterable parameters() throws Exception { @Before public void waitForDatabases() throws Exception { - putGeoIpPipeline(); + putGeoipPipeline(); assertBusy(() -> { Request request = new Request("GET", "/_ingest/geoip/stats"); Map response = entityAsMap(client().performRequest(request)); @@ -105,33 +104,33 @@ public void waitForDatabases() throws Exception { * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. * @throws IOException */ -// private void putGeoipPipeline() throws IOException { -// final BytesReference bytes; -// try (XContentBuilder builder = JsonXContent.contentBuilder()) { -// builder.startObject(); -// { -// builder.startArray("processors"); -// { -// builder.startObject(); -// { -// builder.startObject("geoip"); -// { -// builder.field("field", "ip"); -// builder.field("target_field", "ip-city"); -// builder.field("database_file", "GeoLite2-City.mmdb"); -// } -// builder.endObject(); -// } -// builder.endObject(); -// } -// builder.endArray(); -// } -// builder.endObject(); -// bytes = BytesReference.bytes(builder); -// } -// Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); -// putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); -// client().performRequest(putPipelineRequest); -// } + private void putGeoipPipeline() throws IOException { + final BytesReference bytes; + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + { + builder.startArray("processors"); + { + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", "ip"); + builder.field("target_field", "ip-city"); + builder.field("database_file", "GeoLite2-City.mmdb"); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + bytes = BytesReference.bytes(builder); + } + Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); + putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); + client().performRequest(putPipelineRequest); + } } diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 47c9cdfb1c303..f7943342b5352 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -78,10 +78,10 @@ public void waitForDatabases() throws Exception { assertBusy(() -> { Request request = new Request("GET", "/_ingest/geoip/stats"); Map response = entityAsMap(client().performRequest(request)); - // assert databases are downloaded + Map downloadStats = (Map) response.get("stats"); assertThat(downloadStats.get("databases_count"), equalTo(4)); - // assert databases are loaded to node + Map nodes = (Map) response.get("nodes"); assertThat(nodes.size(), equalTo(1)); Map node = (Map) nodes.values().iterator().next(); @@ -99,7 +99,7 @@ public void waitForDatabases() throws Exception { * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. * @throws IOException */ - public static void putGeoipPipeline() throws IOException { + private void putGeoipPipeline() throws IOException { final BytesReference bytes; try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.startObject(); From ee11ebaedef32aad2c973cdc744287cdc304fa46 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Mon, 28 Jul 2025 12:30:39 -0400 Subject: [PATCH 08/18] share test code --- modules/ingest-geoip/build.gradle | 1 + .../qa/multi-project/build.gradle | 1 + ...eoIpClientMultiProjectYamlTestSuiteIT.java | 71 ++----------------- .../IngestGeoIpClientYamlTestSuiteIT.java | 46 ++++++------ 4 files changed, 31 insertions(+), 88 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index bdda4872fb4dc..34efcb290885b 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -12,6 +12,7 @@ import org.elasticsearch.gradle.OS apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-test-artifact' esplugin { description = 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database' diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index 25b5ba72f4dd4..fffac6c556e81 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -16,6 +16,7 @@ dependencies { javaRestTestImplementation project(':test:external-modules:test-multi-project') javaRestTestImplementation project(':test:fixtures:geoip-fixture') + yamlRestTestImplementation(testArtifact(project(":modules:ingest-geoip"), "yamlRestTest")) // includes yaml test code from ingest-geoip yamlRestTestImplementation project(':modules:ingest-geoip') yamlRestTestImplementation project(':test:external-modules:test-multi-project') yamlRestTestImplementation project(':test:fixtures:geoip-fixture') diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java index 703a0e6968976..fa6c5ef8fb108 100644 --- a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -14,30 +14,18 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.elasticsearch.client.Request; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Booleans; import org.elasticsearch.multiproject.test.MultipleProjectsClientYamlSuiteTestCase; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; +import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded; +import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline; public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { @@ -79,58 +67,7 @@ public static Iterable parameters() throws Exception { @Before public void waitForDatabases() throws Exception { - putGeoipPipeline(); - assertBusy(() -> { - Request request = new Request("GET", "/_ingest/geoip/stats"); - Map response = entityAsMap(client().performRequest(request)); - // assert databases are downloaded - Map downloadStats = (Map) response.get("stats"); - assertThat(downloadStats.get("databases_count"), equalTo(4)); - // assert databases are loaded to node - Map nodes = (Map) response.get("nodes"); - assertThat(nodes.size(), equalTo(1)); - Map node = (Map) nodes.values().iterator().next(); - List databases = ((List) node.get("databases")); - assertThat(databases, notNullValue()); - List databaseNames = databases.stream().map(o -> (String) ((Map) o).get("name")).toList(); - assertThat( - databaseNames, - containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb") - ); - }, 10, TimeUnit.SECONDS); - } - - /** - * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. - * @throws IOException - */ - private void putGeoipPipeline() throws IOException { - final BytesReference bytes; - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.startObject(); - { - builder.startArray("processors"); - { - builder.startObject(); - { - builder.startObject("geoip"); - { - builder.field("field", "ip"); - builder.field("target_field", "ip-city"); - builder.field("database_file", "GeoLite2-City.mmdb"); - } - builder.endObject(); - } - builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - bytes = BytesReference.bytes(builder); - } - Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); - putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); - client().performRequest(putPipelineRequest); + putGeoipPipeline("pipeline-with-geoip"); + assertDatabasesLoaded(); } - } diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index f7943342b5352..48718ce02f3d5 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -74,32 +74,15 @@ public static Iterable parameters() throws Exception { @Before public void waitForDatabases() throws Exception { - putGeoipPipeline(); - assertBusy(() -> { - Request request = new Request("GET", "/_ingest/geoip/stats"); - Map response = entityAsMap(client().performRequest(request)); - - Map downloadStats = (Map) response.get("stats"); - assertThat(downloadStats.get("databases_count"), equalTo(4)); - - Map nodes = (Map) response.get("nodes"); - assertThat(nodes.size(), equalTo(1)); - Map node = (Map) nodes.values().iterator().next(); - List databases = ((List) node.get("databases")); - assertThat(databases, notNullValue()); - List databaseNames = databases.stream().map(o -> (String) ((Map) o).get("name")).toList(); - assertThat( - databaseNames, - containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb") - ); - }); + putGeoipPipeline("pipeline-with-geoip"); + assertDatabasesLoaded(); } /** * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases. * @throws IOException */ - private void putGeoipPipeline() throws IOException { + static void putGeoipPipeline(String pipelineName) throws Exception { final BytesReference bytes; try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.startObject(); @@ -123,9 +106,30 @@ private void putGeoipPipeline() throws IOException { builder.endObject(); bytes = BytesReference.bytes(builder); } - Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip"); + Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineName); putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON)); client().performRequest(putPipelineRequest); } + static void assertDatabasesLoaded() throws Exception { + // assert that the databases are downloaded and loaded + assertBusy(() -> { + Request request = new Request("GET", "/_ingest/geoip/stats"); + Map response = entityAsMap(client().performRequest(request)); + + Map downloadStats = (Map) response.get("stats"); + assertThat(downloadStats.get("databases_count"), equalTo(4)); + + Map nodes = (Map) response.get("nodes"); + assertThat(nodes.size(), equalTo(1)); + Map node = (Map) nodes.values().iterator().next(); + List databases = ((List) node.get("databases")); + assertThat(databases, notNullValue()); + List databaseNames = databases.stream().map(o -> (String) ((Map) o).get("name")).toList(); + assertThat( + databaseNames, + containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb") + ); + }); + } } From 0fdf1b01ab60addfdf8a9fce08f2b69ab276bd15 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Mon, 28 Jul 2025 14:29:41 -0400 Subject: [PATCH 09/18] remove unncessary plugin --- modules/ingest-geoip/qa/multi-project/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-geoip/qa/multi-project/build.gradle b/modules/ingest-geoip/qa/multi-project/build.gradle index fffac6c556e81..6d8597e0c0a9a 100644 --- a/modules/ingest-geoip/qa/multi-project/build.gradle +++ b/modules/ingest-geoip/qa/multi-project/build.gradle @@ -9,7 +9,6 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.yaml-rest-compat-test' dependencies { javaRestTestImplementation project(':modules:ingest-geoip') From d2c338ce6aa1880b50dbaa477bdb87757076604e Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Mon, 28 Jul 2025 15:55:05 -0400 Subject: [PATCH 10/18] remove clusterState --- .../elasticsearch/ingest/geoip/DatabaseNodeService.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 038438becce21..7575d86e6a509 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -297,7 +297,6 @@ void checkDatabases(ClusterState state) { void checkDatabases(ProjectState projectState) { ProjectId projectId = projectState.projectId(); ProjectMetadata projectMetadata = projectState.metadata(); - ClusterState clusterState = projectState.cluster(); PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks == null) { logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); @@ -311,7 +310,7 @@ void checkDatabases(ProjectState projectState) { } else { // regardless of whether DATABASES_INDEX is an alias, resolve it to a concrete index Index databasesIndex = databasesAbstraction.getWriteIndex(); - IndexRoutingTable databasesIndexRT = clusterState.routingTable(projectId).index(databasesIndex); + IndexRoutingTable databasesIndexRT = projectState.routingTable().index(databasesIndex); if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { logger.trace( "Not checking databases because geoip databases index does not have all active primary shards for" + " project [{}]", @@ -335,7 +334,7 @@ void checkDatabases(ProjectState projectState) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(clusterState.getMetadata().settings())) + .filter(e -> e.getValue().isNewEnough(projectState.settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); @@ -343,7 +342,7 @@ void checkDatabases(ProjectState projectState) { // process the geoip task state for the enterprise geoip downloader { - EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(clusterState); + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(projectState.cluster()); if (taskState == null) { // Note: an empty state will purge stale entries in databases map taskState = EnterpriseGeoIpTaskState.EMPTY; @@ -352,7 +351,7 @@ void checkDatabases(ProjectState projectState) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(clusterState.getMetadata().settings())) + .filter(e -> e.getValue().isNewEnough(projectState.settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); From c611574fb8be3c87da923808368e075be98f2bc4 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Mon, 28 Jul 2025 17:25:07 -0400 Subject: [PATCH 11/18] fix --- .../org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 7575d86e6a509..946f7e95a5960 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -297,6 +297,7 @@ void checkDatabases(ClusterState state) { void checkDatabases(ProjectState projectState) { ProjectId projectId = projectState.projectId(); ProjectMetadata projectMetadata = projectState.metadata(); + ClusterState clusterState = projectState.cluster(); PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks == null) { logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); @@ -334,7 +335,7 @@ void checkDatabases(ProjectState projectState) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(projectState.settings())) + .filter(e -> e.getValue().isNewEnough(projectState.cluster().metadata().settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); @@ -351,7 +352,7 @@ void checkDatabases(ProjectState projectState) { taskState.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isNewEnough(projectState.settings())) + .filter(e -> e.getValue().isNewEnough(projectState.cluster().metadata().settings())) .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList() ); From 31fca859924f4dd9b1a6ad55f7bc32a27b18f9b5 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Tue, 29 Jul 2025 14:04:20 -0400 Subject: [PATCH 12/18] Add FixForMultiProject and other fix --- .../ingest/geoip/DatabaseNodeService.java | 12 +++++++----- .../ingest/geoip/EnterpriseGeoIpTaskState.java | 9 +++------ .../elasticsearch/ingest/geoip/GeoIpTaskState.java | 2 ++ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 946f7e95a5960..6b489e14b2cdf 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -29,7 +29,9 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.NotMultiProjectCapable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; @@ -289,15 +291,12 @@ void checkDatabases(ClusterState state) { } // Optimization: only load the .geoip_databases for projects that are allocated to this node - for (ProjectId projectId : state.metadata().projects().keySet()) { - checkDatabases(state.projectState(projectId)); - } + state.forEachProject(this::checkDatabases); } void checkDatabases(ProjectState projectState) { ProjectId projectId = projectState.projectId(); ProjectMetadata projectMetadata = projectState.metadata(); - ClusterState clusterState = projectState.cluster(); PersistentTasksCustomMetadata persistentTasks = projectMetadata.custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks == null) { logger.trace("Not checking databases for project [{}] because persistent tasks are null", projectId); @@ -343,7 +342,10 @@ void checkDatabases(ProjectState projectState) { // process the geoip task state for the enterprise geoip downloader { - EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(projectState.cluster()); + @NotMultiProjectCapable(description = "Enterprise GeoIP not supported in serverless") + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState( + projectState.cluster().metadata().getProject(ProjectId.DEFAULT) + ); if (taskState == null) { // Note: an empty state will purge stale entries in databases map taskState = EnterpriseGeoIpTaskState.EMPTY; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java index 39b037cc88af5..360149fc56f29 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java @@ -11,12 +11,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.VersionedNamedWriteable; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.EnterpriseGeoIpTask; @@ -148,10 +146,9 @@ public void writeTo(StreamOutput out) throws IOException { * @return the geoip downloader's task state or null if there is not a state to read */ @Nullable - @FixForMultiProject(description = "Replace ProjectId.DEFAULT") - static EnterpriseGeoIpTaskState getEnterpriseGeoIpTaskState(ClusterState state) { + static EnterpriseGeoIpTaskState getEnterpriseGeoIpTaskState(ProjectMetadata projectMetadata) { PersistentTasksCustomMetadata.PersistentTask task = getTaskWithId( - state.metadata().getProject(ProjectId.DEFAULT), + projectMetadata, EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER ); return (task == null) ? null : (EnterpriseGeoIpTaskState) task.getState(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index 4014291cfaf6b..f8544b1fcb90f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -214,6 +215,7 @@ public boolean isCloseToExpiration() { private static final TimeValue THIRTY_DAYS = TimeValue.timeValueDays(30); private static final long THIRTY_DAYS_MILLIS = THIRTY_DAYS.millis(); + @FixForMultiProject(description = "Replace caller from cluster settings to project settings") public boolean isNewEnough(Settings settings) { // micro optimization: this looks a little silly, but the expected case is that database_validity is only used in tests. // we run this code on every document, though, so the argument checking and other bits that getAsTime does is enough From 4953636e8895c067b33b09a90e1ec66e660433c1 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 29 Jul 2025 18:16:08 +0000 Subject: [PATCH 13/18] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 6b489e14b2cdf..87fafcc1fba06 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; -import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.NotMultiProjectCapable; import org.elasticsearch.core.Nullable; From 892d97d2db7e78de4a3b4d70078d812409332790 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Tue, 29 Jul 2025 14:28:13 -0400 Subject: [PATCH 14/18] fix doc --- .../elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java index 360149fc56f29..0deb8a8cc673e 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java @@ -142,7 +142,7 @@ public void writeTo(StreamOutput out) throws IOException { * Retrieves the geoip downloader's task state from the cluster state. This may return null in some circumstances, * for example if the geoip downloader task hasn't been created yet (which it wouldn't be if it's disabled). * - * @param state the cluster state to read the task state from + * @param projectMetadata the project metatdata to read the task state from * @return the geoip downloader's task state or null if there is not a state to read */ @Nullable From 24cd03ce5992af04c56f40263487d155d3115989 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Thu, 31 Jul 2025 12:48:56 -0400 Subject: [PATCH 15/18] Remove NotMultiProjectCapable --- .../org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 87fafcc1fba06..20d1721ca3a5c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -341,10 +341,7 @@ void checkDatabases(ProjectState projectState) { // process the geoip task state for the enterprise geoip downloader { - @NotMultiProjectCapable(description = "Enterprise GeoIP not supported in serverless") - EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState( - projectState.cluster().metadata().getProject(ProjectId.DEFAULT) - ); + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(projectState.metadata()); if (taskState == null) { // Note: an empty state will purge stale entries in databases map taskState = EnterpriseGeoIpTaskState.EMPTY; From 1287fdc1375f09bf5f4e6c822ac1af6571d48a6f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 31 Jul 2025 17:01:46 +0000 Subject: [PATCH 16/18] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 20d1721ca3a5c..e3ad2b49dcb58 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -30,7 +30,6 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.NotMultiProjectCapable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; From 743602ea0950f58b01e44a81f873c363c05bf3ed Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Thu, 31 Jul 2025 13:03:27 -0400 Subject: [PATCH 17/18] Add reminder to remove test --- .../geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java index fa6c5ef8fb108..201de45dc4efd 100644 --- a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -15,6 +15,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.multiproject.test.MultipleProjectsClientYamlSuiteTestCase; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -27,6 +28,7 @@ import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded; import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline; +@FixForMultiProject(description = "Potentially remove this test after https://elasticco.atlassian.net/browse/ES-12094 is implemented") public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false; From f02df24275ee6d5ad96fbd4151209c60007c4f9d Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Thu, 31 Jul 2025 16:30:23 -0400 Subject: [PATCH 18/18] comment --- .../org/elasticsearch/ingest/geoip/DatabaseNodeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index e3ad2b49dcb58..b3038d2fd59fa 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -311,7 +311,7 @@ void checkDatabases(ProjectState projectState) { IndexRoutingTable databasesIndexRT = projectState.routingTable().index(databasesIndex); if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { logger.trace( - "Not checking databases because geoip databases index does not have all active primary shards for" + " project [{}]", + "Not checking databases because geoip databases index does not have all active primary shards for project [{}]", projectId ); return;