diff --git a/build.gradle b/build.gradle index 615714b26..06d0cf77e 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,7 @@ import java.util.concurrent.Callable import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.ExtensionsProperties import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask buildscript { @@ -34,6 +35,8 @@ buildscript { js_resource_folder = "src/test/resources/job-scheduler" common_utils_version = System.getProperty("common_utils.version", opensearch_build) job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build) + job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + plugin_no_snapshot + '.zip' bwcVersionShort = "2.6.0" bwcVersion = bwcVersionShort + ".0" bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' + @@ -92,8 +95,8 @@ repositories { } apply plugin: 'java' -apply plugin: 'application' apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' apply plugin: 'base' apply plugin: 'jacoco' @@ -101,8 +104,6 @@ apply plugin: 'eclipse' //apply plugin: 'opensearch.pluginzip' apply plugin: 'maven-publish' -mainClassName = 'org.opensearch.ad.AnomalyDetectorExtension' - ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier") @@ -118,6 +119,13 @@ allprojects { } } +opensearchplugin { + name 'opensearch-anomaly-detection' + description 'OpenSearch anomaly detector plugin' + classname 'org.opensearch.ad.AnomalyDetectorExtension' + extendedPlugins = ['lang-painless', 'opensearch-job-scheduler'] +} + ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') @@ -139,10 +147,35 @@ configurations.all { force "net.bytebuddy:byte-buddy-agent:1.9.15" force "com.google.code.gson:gson:2.8.9" force "junit:junit:4.13.2" + + force "jakarta.json:jakarta.json-api:2.1.1" + force "com.google.guava:guava:31.1-jre" + force "org.apache.logging.log4j:log4j-api:2.20.0" + force "org.apache.logging.log4j:log4j-core:2.20.0" + force "org.apache.logging.log4j:log4j-jul:2.20.0" + force "com.fasterxml.jackson:jackson-bom:2.15.1" + force "com.fasterxml.jackson.core:jackson-databind:2.15.0" + force "com.fasterxml.jackson.core:jackson-core:2.15.0" + force "com.fasterxml.jackson.core:jackson-annotations:2.15.1" + force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2" + force "com.fasterxml.jackson.datatype:jackson-datatype-guava:2.15.2" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.15.2" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.2" + force "io.netty:netty-buffer:4.1.92.Final" + force "io.netty:netty-codec:4.1.92.Final" + force "io.netty:netty-codec-http:4.1.92.Final" + force "io.netty:netty-codec-http2:4.1.92.Final" + force "io.netty:netty-common:4.1.92.Final" + force "io.netty:netty-handler:4.1.92.Final" + force "io.netty:netty-transport-native-unix-common:4.1.92.Final" + force "io.netty:netty-resolver:4.1.92.Final" + force "io.netty:netty-transport:4.1.92.Final" } } configurations { + zipArchive testImplementation { exclude group: 'org.hamcrest', module: 'hamcrest-core' } @@ -170,7 +203,6 @@ publishing { } } } - // Commented this code until https://github.com/opensearch-project/opensearch-sdk-java/issues/144 /*tasks.named('forbiddenApisMain').configure { // Only enable limited check because AD code has too many violations. @@ -185,12 +217,18 @@ publishing { // Allow test cases to be named Tests without having to be inherited from LuceneTestCase. // see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java -//testingConventions.enabled = false - -//licenseHeaders.enabled = true -//dependencyLicenses.enabled = false -//thirdPartyAudit.enabled = false -//loggerUsageCheck.enabled = false +testingConventions.enabled = false + +licenseHeaders.enabled = false +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +validatePluginZipPom.enabled = false +validateNebulaPom.enabled = false +forbiddenApisMain.enabled = false +forbiddenApisTest.enabled = false +generatePomFileForPluginZipPublication.enabled = false +publishPluginZipPublicationToMavenLocal.enabled = false // See package README.md for details on using these tasks. def _numNodes = findProperty('numNodes') as Integer ?: 1 @@ -213,6 +251,13 @@ test { systemProperty 'tests.security.manager', 'false' } +task anomalyDetection(type: JavaExec) { + group = 'Execution' + description = 'Run AnomalyDetection Extension.' + mainClass = 'org.opensearch.ad.AnomalyDetectorExtension' + classpath = sourceSets.main.runtimeClasspath +} + task integTest(type: RestIntegTestTask) { description = "Run tests against a cluster" testClassesDirs = sourceSets.test.output.classesDirs @@ -222,6 +267,11 @@ task integTest(type: RestIntegTestTask) { // Commented this code until https://github.com/opensearch-project/opensearch-sdk-java/issues/144 /* tasks.named("check").configure { dependsOn(integTest) } +*/ + +task startTestExtension(type: Exec) { + commandLine 'bash', '-c', "./gradlew anomalyDetection &" +} integTest { retry { @@ -231,7 +281,7 @@ integTest { maxFailures = 10 } } -// dependsOn "bundlePlugin" + dependsOn "startTestExtension" systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath @@ -286,9 +336,17 @@ integTest { } } } -*/ -/*testClusters.integTest { +def integADExtensionYml = new org.yaml.snakeyaml.Yaml().load(new File("src/test/resources/org/opensearch/ad/integ/anomaly-detection-extension.yml").newInputStream()) + +task closeTestExtension (type: Exec) { + commandLine 'bash', '-c', "kill \$(lsof -i:${integADExtensionYml.port})" +} + +tasks.named("integTest").configure { finalizedBy(closeTestExtension) } + +testClusters.integTest { + extension(new ExtensionsProperties("${integADExtensionYml.name}", "${integADExtensionYml.uniqueId}", "${integADExtensionYml.hostAddress}", "${integADExtensionYml.port}", "${integADExtensionYml.version}", "${integADExtensionYml.opensearchVersion}", "${integADExtensionYml.minimumCompatibleVersion}")) testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes @@ -302,53 +360,19 @@ integTest { debugPort += 1 } } - plugin(project.tasks.bundlePlugin.archiveFile)*/ - - -// Commented this code until we have support of Job Scheduler for extensibility -// plugin(provider(new Callable(){ -// @Override -// RegularFile call() throws Exception { -// return new RegularFile() { -// @Override -// File getAsFile() { -// if (new File("$project.rootDir/$js_resource_folder").exists()) { -// project.delete(files("$project.rootDir/$js_resource_folder")) -// } -// project.mkdir js_resource_folder -// ant.get(src: job_scheduler_build_download, -// dest: js_resource_folder, -// httpusecaches: false) -// return fileTree(js_resource_folder).getSingleFile() -// } -// } -// } -// })) -// -// // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before -// // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug. -// // -// // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opensearch-job-scheduler], dependency of [opendistro-anomaly-detection] -// // at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452) -// // -// // One explanation is that ES build script sort plugins according to the natural ordering of their names. -// // opendistro-anomaly-detection comes before opensearch-job-scheduler. -// // -// // The following is a comparison of different plugin installation order: -// // Before 7.7: -// // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opensearch-job-scheduler.zip -// // -// // After 7.7: -// // ./bin/elasticsearch-plugin install --batch file:opensearch-job-scheduler.zip file:opendistro-anomaly-detection.zip -// // -// // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed. -/* nodes.each { node -> - def plugins = node.plugins - def firstPlugin = plugins.get(0) - plugins.remove(0) - plugins.add(firstPlugin) - } -}*/ + + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.getSingleFile() + } + } + } + })) +} task integTestRemote(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs @@ -792,6 +816,7 @@ check.dependsOn jacocoTestCoverageVerification jacocoTestCoverageVerification.dependsOn jacocoTestReport dependencies { + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" compileOnly "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${opensearch_version}" implementation "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" @@ -799,11 +824,11 @@ dependencies { // implementation "org.opensearch:common-utils:${common_utils_version}" implementation "org.opensearch:opensearch-job-scheduler:${job_scheduler_version}" implementation "org.opensearch.sdk:opensearch-sdk-java:2.0.0-SNAPSHOT" - implementation "com.google.inject:guice:5.1.0" + implementation "com.google.inject:guice:7.0.0" implementation "org.opensearch.client:opensearch-java:${opensearch_version}" implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" - implementation group: 'com.google.guava', name: 'guava', version:'31.0.1-jre' + implementation group: 'com.google.guava', name: 'guava', version:'31.1-jre' implementation group: 'com.google.guava', name: 'failureaccess', version:'1.0.1' implementation group: 'org.javassist', name: 'javassist', version:'3.28.0-GA' implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' diff --git a/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java b/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java index 881edd9ec..fe2582a61 100644 --- a/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java +++ b/src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java @@ -190,7 +190,6 @@ public void runJob(ScheduledJobParameter scheduledJobParameter, JobExecutionCont * Get anomaly result, index result or handle exception if failed. * * @param jobParameter scheduled job parameter - * @param lockService lock service * @param lock lock to run job * @param detectionStartTime detection start time * @param executionStartTime detection end time diff --git a/src/test/java/org/opensearch/ad/ADExtensionIntegTestCase.java b/src/test/java/org/opensearch/ad/ADExtensionIntegTestCase.java new file mode 100644 index 000000000..217dea986 --- /dev/null +++ b/src/test/java/org/opensearch/ad/ADExtensionIntegTestCase.java @@ -0,0 +1,173 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.ad; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.net.ssl.SSLEngine; + +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.junit.AfterClass; +import org.junit.Before; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.sdk.ExtensionSettings; +import org.opensearch.sdk.ExtensionsRunner; +import org.opensearch.sdk.SDKClient; +import org.opensearch.sdk.SDKClient.SDKClusterAdminClient; +import org.opensearch.sdk.SDKClient.SDKRestClient; +import org.opensearch.sdk.SDKClusterService; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public abstract class ADExtensionIntegTestCase extends OpenSearchRestTestCase { + + private static final String EXTENSION_SETTINGS_PATH = "/ad-extension.yml"; + + private static ExtensionSettings extensionSettings; + private static String openSearchIntegTestClusterHost; + private static int openSearchIntegTestClusterPort; + private static SDKClient sdkClient = null; + private static SDKRestClient sdkRestClient = null; + + // TODO : Remove ExtensionsRunner + private static ExtensionsRunner extensionsRunner = null; + // TODO : Create new SDKClusterService + private static SDKClusterService sdkClusterService = null; + + @Override + @Before + public void setUp() throws Exception { + + super.setUp(); + + // Retrieve AD Extension Settings + extensionSettings = ExtensionSettings.readSettingsFromYaml(EXTENSION_SETTINGS_PATH); + + // Determine opensearch integration test cluster host and port + String cluster = getTestRestCluster(); + String[] stringUrls = cluster.split(","); + List hosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); + } + String host = stringUrl.substring(0, portSeparator); + int port = Integer.valueOf(stringUrl.substring(portSeparator + 1)); + hosts.add(buildHttpHost(host, port)); + } + + openSearchIntegTestClusterHost = hosts.get(1).getHostName(); + openSearchIntegTestClusterPort = hosts.get(1).getPort(); + } + + @AfterClass + public static void closeSdkRestClient() throws IOException { + try { + sdkRestClient.close(); + } finally { + sdkRestClient = null; + } + } + + private static SDKClient sdkClient() { + + if (sdkClient == null) { + // Set OpenSearch Address/Port to test cluster host/port + ExtensionSettings integTestClusterExtensionSettings = new ExtensionSettings( + extensionSettings.getExtensionName(), + extensionSettings.getHostAddress(), + extensionSettings.getHostPort(), + openSearchIntegTestClusterHost, + String.valueOf(openSearchIntegTestClusterPort) + ); + sdkClient = new SDKClient(integTestClusterExtensionSettings); + } + return sdkClient; + } + + public static SDKRestClient sdkRestClient() { + if (sdkRestClient == null) { + sdkRestClient = new SDKRestClient( + sdkClient(), + new RestHighLevelClient(builder(openSearchIntegTestClusterHost, openSearchIntegTestClusterPort)) + ); + } + return sdkRestClient; + } + + public static SDKRestClient sdkAdminClient() { + return sdkRestClient().admin(); + } + + public static SDKClusterAdminClient sdkClusterAdmin() { + return sdkRestClient().cluster(); + } + + public static SDKClusterService sdkClusterService() { + if (sdkClusterService == null) { + sdkClusterService = extensionsRunner.getSdkClusterService(); + } + return sdkClusterService; + } + + /** + * Create and configure a RestClientBuilder + * + * @param hostAddress The address the client should connect to + * @param port The port the client should connect to + * @return An instance of the builder + */ + private static RestClientBuilder builder(String hostAddress, int port) { + RestClientBuilder builder = RestClient.builder(new HttpHost(hostAddress, port)); + builder.setStrictDeprecationMode(true); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + try { + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder + .create() + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // disable the certificate since our cluster currently just uses the default security + // configuration + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + // See please https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder + .create() + .setTlsStrategy(tlsStrategy) + .build(); + return httpClientBuilder.setConnectionManager(connectionManager); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return builder; + } + +} diff --git a/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java b/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java index c255ada01..2d53e9a5e 100644 --- a/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java +++ b/src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java @@ -11,6 +11,8 @@ package org.opensearch.ad; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.opensearch.common.xcontent.json.JsonXContent.jsonXContent; import java.io.IOException; @@ -23,14 +25,16 @@ import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.ad.model.ADTask; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.AnomalyDetectorExecutionInput; +import org.opensearch.ad.model.AnomalyDetectorJob; import org.opensearch.ad.model.DetectionDateRange; import org.opensearch.ad.util.RestHandlerUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -43,6 +47,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestStatus; +import org.opensearch.sdk.SDKClient.SDKRestClient; import org.opensearch.test.rest.OpenSearchRestTestCase; import com.google.common.collect.ImmutableList; @@ -51,6 +56,8 @@ public abstract class AnomalyDetectorRestTestCase extends ODFERestTestCase { + private static final Logger logger = LogManager.getLogger(AnomalyDetectorRestTestCase.class); + public static final int MAX_RETRY_TIMES = 10; @Override @@ -63,24 +70,28 @@ protected Settings restClientSettings() { return super.restClientSettings(); } - protected AnomalyDetector createRandomAnomalyDetector(Boolean refresh, Boolean withMetadata, String indexName, RestClient client) + protected AnomalyDetector createRandomAnomalyDetector(Boolean refresh, Boolean withMetadata, String indexName, SDKRestClient client) throws IOException { return createRandomAnomalyDetector(refresh, withMetadata, client, true, indexName); } - protected AnomalyDetector createRandomAnomalyDetector(Boolean refresh, Boolean withMetadata, RestClient client) throws IOException { + protected AnomalyDetector createRandomAnomalyDetector(Boolean refresh, Boolean withMetadata, SDKRestClient client) throws IOException { return createRandomAnomalyDetector(refresh, withMetadata, client, true); } - protected AnomalyDetector createRandomAnomalyDetector(Boolean refresh, Boolean withMetadata, RestClient client, boolean featureEnabled) - throws IOException { + protected AnomalyDetector createRandomAnomalyDetector( + Boolean refresh, + Boolean withMetadata, + SDKRestClient client, + boolean featureEnabled + ) throws IOException { return createRandomAnomalyDetector(refresh, withMetadata, client, featureEnabled, null); } protected AnomalyDetector createRandomAnomalyDetector( Boolean refresh, Boolean withMetadata, - RestClient client, + SDKRestClient client, boolean featureEnabled, String indexName ) throws IOException { @@ -124,7 +135,7 @@ protected AnomalyDetector createRandomAnomalyDetector( return getAnomalyDetector(createdDetector.getDetectorId(), new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), client); } - protected AnomalyDetector createAnomalyDetector(AnomalyDetector detector, Boolean refresh, RestClient client) throws IOException { + protected AnomalyDetector createAnomalyDetector(AnomalyDetector detector, Boolean refresh, SDKRestClient client) throws IOException { Response response = TestHelpers .makeRequest(client, "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), TestHelpers.toHttpEntity(detector), null); assertEquals("Create anomaly detector failed", RestStatus.CREATED, TestHelpers.restStatus(response)); @@ -161,7 +172,7 @@ protected AnomalyDetector createAnomalyDetector(AnomalyDetector detector, Boolea return detectorInIndex; } - protected Response startAnomalyDetector(String detectorId, DetectionDateRange dateRange, RestClient client) throws IOException { + protected Response startAnomalyDetector(String detectorId, DetectionDateRange dateRange, SDKRestClient client) throws IOException { return TestHelpers .makeRequest( client, @@ -173,7 +184,7 @@ protected Response startAnomalyDetector(String detectorId, DetectionDateRange da ); } - protected Response stopAnomalyDetector(String detectorId, RestClient client, boolean realtime) throws IOException { + protected Response stopAnomalyDetector(String detectorId, SDKRestClient client, boolean realtime) throws IOException { String jobType = realtime ? "" : "?historical"; return TestHelpers .makeRequest( @@ -186,11 +197,11 @@ protected Response stopAnomalyDetector(String detectorId, RestClient client, boo ); } - protected Response deleteAnomalyDetector(String detectorId, RestClient client) throws IOException { + protected Response deleteAnomalyDetector(String detectorId, SDKRestClient client) throws IOException { return TestHelpers.makeRequest(client, "DELETE", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detectorId, ImmutableMap.of(), "", null); } - protected Response previewAnomalyDetector(String detectorId, RestClient client, AnomalyDetectorExecutionInput input) + protected Response previewAnomalyDetector(String detectorId, SDKRestClient client, AnomalyDetectorExecutionInput input) throws IOException { return TestHelpers .makeRequest( @@ -203,11 +214,11 @@ protected Response previewAnomalyDetector(String detectorId, RestClient client, ); } - public AnomalyDetector getAnomalyDetector(String detectorId, RestClient client) throws IOException { + public AnomalyDetector getAnomalyDetector(String detectorId, SDKRestClient client) throws IOException { return (AnomalyDetector) getAnomalyDetector(detectorId, false, client)[0]; } - public Response updateAnomalyDetector(String detectorId, AnomalyDetector newDetector, RestClient client) throws IOException { + public Response updateAnomalyDetector(String detectorId, AnomalyDetector newDetector, SDKRestClient client) throws IOException { BasicHeader header = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); return TestHelpers .makeRequest( @@ -220,11 +231,11 @@ public Response updateAnomalyDetector(String detectorId, AnomalyDetector newDete ); } - public AnomalyDetector getAnomalyDetector(String detectorId, BasicHeader header, RestClient client) throws IOException { + public AnomalyDetector getAnomalyDetector(String detectorId, BasicHeader header, SDKRestClient client) throws IOException { return (AnomalyDetector) getAnomalyDetector(detectorId, header, false, false, client)[0]; } - public ToXContentObject[] getAnomalyDetector(String detectorId, boolean returnJob, RestClient client) throws IOException { + public ToXContentObject[] getAnomalyDetector(String detectorId, boolean returnJob, SDKRestClient client) throws IOException { BasicHeader header = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); return getAnomalyDetector(detectorId, header, returnJob, false, client); } @@ -234,7 +245,7 @@ public ToXContentObject[] getAnomalyDetector( BasicHeader header, boolean returnJob, boolean returnTask, - RestClient client + SDKRestClient client ) throws IOException { Response response = TestHelpers .makeRequest( @@ -253,8 +264,7 @@ public ToXContentObject[] getAnomalyDetector( String id = null; Long version = null; AnomalyDetector detector = null; - // @anomaly-detection.create-detector Commented this code until we have support of Job Scheduler for extensibility - // AnomalyDetectorJob detectorJob = null; + AnomalyDetectorJob detectorJob = null; ADTask realtimeAdTask = null; ADTask historicalAdTask = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -270,10 +280,9 @@ public ToXContentObject[] getAnomalyDetector( case "anomaly_detector": detector = AnomalyDetector.parse(parser); break; - // @anomaly-detection.create-detector Commented this code until we have support of Job Scheduler for extensibility - /*case "anomaly_detector_job": + case "anomaly_detector_job": detectorJob = AnomalyDetectorJob.parse(parser); - break;*/ + break; case "realtime_detection_task": if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { realtimeAdTask = ADTask.parse(parser); @@ -310,8 +319,7 @@ public ToXContentObject[] getAnomalyDetector( detector.getUser(), detector.getResultIndex() ), - // @anomaly-detection.create-detector Commented this code until we have support of Job Scheduler for extensibility - // detectorJob, + detectorJob, historicalAdTask, realtimeAdTask }; } @@ -330,12 +338,12 @@ public void updateClusterSettings(String settingKey, Object value) throws Except .endObject(); Request request = new Request("PUT", "_cluster/settings"); request.setJsonEntity(Strings.toString(builder)); - Response response = client().performRequest(request); + Response response = sdkRestClient().performRequest(request); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); Thread.sleep(2000); // sleep some time to resolve flaky test } - public Response getDetectorProfile(String detectorId, boolean all, String customizedProfile, RestClient client) throws IOException { + public Response getDetectorProfile(String detectorId, boolean all, String customizedProfile, SDKRestClient client) throws IOException { return TestHelpers .makeRequest( client, @@ -348,17 +356,17 @@ public Response getDetectorProfile(String detectorId, boolean all, String custom } public Response getDetectorProfile(String detectorId) throws IOException { - return getDetectorProfile(detectorId, false, "", client()); + return getDetectorProfile(detectorId, false, "", sdkRestClient()); } public Response getDetectorProfile(String detectorId, boolean all) throws IOException { - return getDetectorProfile(detectorId, all, "", client()); + return getDetectorProfile(detectorId, all, "", sdkRestClient()); } public Response getSearchDetectorCount() throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "GET", TestHelpers.AD_BASE_DETECTORS_URI + "/" + RestHandlerUtils.COUNT, null, @@ -370,7 +378,7 @@ public Response getSearchDetectorCount() throws IOException { public Response getSearchDetectorMatch(String name) throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "GET", TestHelpers.AD_BASE_DETECTORS_URI + "/" + RestHandlerUtils.MATCH, ImmutableMap.of("name", name), @@ -379,7 +387,7 @@ public Response getSearchDetectorMatch(String name) throws IOException { ); } - public Response searchTopAnomalyResults(String detectorId, boolean historical, String bodyAsJsonString, RestClient client) + public Response searchTopAnomalyResults(String detectorId, boolean historical, String bodyAsJsonString, SDKRestClient client) throws IOException { return TestHelpers .makeRequest( @@ -405,7 +413,7 @@ public Response createUser(String name, String password, ArrayList backe } return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/_opendistro/_security/api/internalusers/" + name, null, @@ -432,7 +440,7 @@ public Response createRoleMapping(String role, ArrayList users) throws I } return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/_opendistro/_security/api/rolesmapping/" + role, null, @@ -447,7 +455,7 @@ public Response createRoleMapping(String role, ArrayList users) throws I public Response createIndexRole(String role, String index) throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/_opendistro/_security/api/roles/" + role, null, @@ -482,7 +490,7 @@ public Response createIndexRole(String role, String index) throws IOException { public Response createSearchRole(String role, String index) throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/_opendistro/_security/api/roles/" + role, null, @@ -516,7 +524,7 @@ public Response createSearchRole(String role, String index) throws IOException { public Response deleteUser(String user) throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", "/_opendistro/_security/api/internalusers/" + user, null, @@ -528,7 +536,7 @@ public Response deleteUser(String user) throws IOException { public Response deleteRoleMapping(String user) throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", "/_opendistro/_security/api/rolesmapping/" + user, null, @@ -540,7 +548,7 @@ public Response deleteRoleMapping(String user) throws IOException { public Response enableFilterBy() throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "_cluster/settings", null, @@ -559,7 +567,7 @@ public Response enableFilterBy() throws IOException { public Response disableFilterBy() throws IOException { return TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "_cluster/settings", null, @@ -598,7 +606,7 @@ protected AnomalyDetector cloneDetector(AnomalyDetector anomalyDetector, String return detector; } - protected Response validateAnomalyDetector(AnomalyDetector detector, RestClient client) throws IOException { + protected Response validateAnomalyDetector(AnomalyDetector detector, SDKRestClient client) throws IOException { return TestHelpers .makeRequest( client, diff --git a/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java b/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java index cc6c8f7b3..697fdf403 100644 --- a/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java +++ b/src/test/java/org/opensearch/ad/HistoricalAnalysisRestTestCase.java @@ -11,6 +11,9 @@ package org.opensearch.ad; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; import java.io.IOException; @@ -26,6 +29,8 @@ import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Before; import org.opensearch.ad.mock.model.MockSimpleLog; import org.opensearch.ad.model.ADTaskProfile; @@ -33,10 +38,10 @@ import org.opensearch.ad.model.DetectionDateRange; import org.opensearch.ad.model.Feature; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestStatus; +import org.opensearch.sdk.SDKClient.SDKRestClient; import org.opensearch.search.aggregations.AggregationBuilder; import com.google.common.collect.ImmutableList; @@ -44,6 +49,8 @@ public abstract class HistoricalAnalysisRestTestCase extends AnomalyDetectorRestTestCase { + private static final Logger logger = LogManager.getLogger(HistoricalAnalysisRestTestCase.class); + public static final int MAX_RETRY_TIMES = 200; protected String historicalAnalysisTestIndex = "test_historical_analysis_data"; protected int detectionIntervalInMinutes = 1; @@ -58,7 +65,7 @@ public void setUp() throws Exception { ingestTestDataForHistoricalAnalysis(historicalAnalysisTestIndex, detectionIntervalInMinutes); } - public ToXContentObject[] getHistoricalAnomalyDetector(String detectorId, boolean returnTask, RestClient client) throws IOException { + public ToXContentObject[] getHistoricalAnomalyDetector(String detectorId, boolean returnTask, SDKRestClient client) throws IOException { BasicHeader header = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); return getAnomalyDetector(detectorId, header, false, returnTask, client); } @@ -66,7 +73,7 @@ public ToXContentObject[] getHistoricalAnomalyDetector(String detectorId, boolea public ADTaskProfile getADTaskProfile(String detectorId) throws IOException, ParseException { Response profileResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "GET", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detectorId + "/_profile/ad_task", ImmutableMap.of(), @@ -79,7 +86,7 @@ public ADTaskProfile getADTaskProfile(String detectorId) throws IOException, Par public Response searchTaskResult(String resultIndex, String taskId) throws IOException { Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "GET", TestHelpers.AD_BASE_RESULT_URI + "/_search/" + resultIndex, ImmutableMap.of(), @@ -104,7 +111,7 @@ public Response ingestSimpleMockLog( ParseException { TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", indexName, null, @@ -112,7 +119,7 @@ public Response ingestSimpleMockLog( ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana")) ); - Response statsResponse = TestHelpers.makeRequest(client(), "GET", indexName, ImmutableMap.of(), "", null); + Response statsResponse = TestHelpers.makeRequest(sdkRestClient(), "GET", indexName, ImmutableMap.of(), "", null); assertEquals(RestStatus.OK, TestHelpers.restStatus(statsResponse)); String result = EntityUtils.toString(statsResponse.getEntity()); assertTrue(result.contains(indexName)); @@ -142,7 +149,7 @@ public Response ingestSimpleMockLog( } Response bulkResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", "_bulk?refresh=true", null, @@ -213,14 +220,14 @@ protected AnomalyDetector createAnomalyDetector(int categoryFieldSize, String re categoryField, resultIndex ); - return createAnomalyDetector(detector, true, client()); + return createAnomalyDetector(detector, true, sdkRestClient()); } protected String startHistoricalAnalysis(String detectorId) throws IOException { Instant endTime = Instant.now().truncatedTo(ChronoUnit.SECONDS); Instant startTime = endTime.minus(10, ChronoUnit.DAYS).truncatedTo(ChronoUnit.SECONDS); DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); - Response startDetectorResponse = startAnomalyDetector(detectorId, dateRange, client()); + Response startDetectorResponse = startAnomalyDetector(detectorId, dateRange, sdkRestClient()); Map startDetectorResponseMap = responseAsMap(startDetectorResponse); String taskId = (String) startDetectorResponseMap.get("_id"); assertNotNull(taskId); diff --git a/src/test/java/org/opensearch/ad/ODFERestTestCase.java b/src/test/java/org/opensearch/ad/ODFERestTestCase.java index d7fec8ba4..c0b86a593 100644 --- a/src/test/java/org/opensearch/ad/ODFERestTestCase.java +++ b/src/test/java/org/opensearch/ad/ODFERestTestCase.java @@ -55,12 +55,11 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestStatus; -import org.opensearch.test.rest.OpenSearchRestTestCase; /** * ODFE integration test base class to support both security disabled and enabled ODFE cluster. */ -public abstract class ODFERestTestCase extends OpenSearchRestTestCase { +public abstract class ODFERestTestCase extends ADExtensionIntegTestCase { protected boolean isHttps() { boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); @@ -99,14 +98,14 @@ protected Settings restAdminSettings() { // (e.g., deleting system indices) protected static void deleteIndexWithAdminClient(String name) throws IOException { Request request = new Request("DELETE", "/" + name); - adminClient().performRequest(request); + sdkAdminClient().performRequest(request); } // Utility fn for checking if an index exists. Should only be used when not allowed in a regular context // (e.g., checking existence of system indices) protected static boolean indexExistsWithAdminClient(String indexName) throws IOException { Request request = new Request("HEAD", "/" + indexName); - Response response = adminClient().performRequest(request); + Response response = sdkAdminClient().performRequest(request); return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } @@ -141,7 +140,7 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE @SuppressWarnings("unchecked") @After protected void wipeAllODFEIndices() throws IOException { - Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + Response response = sdkAdminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); try ( XContentParser parser = xContentType @@ -163,7 +162,7 @@ protected void wipeAllODFEIndices() throws IOException { for (Map index : parserList) { String indexName = (String) index.get("index"); if (indexName != null && !".opendistro_security".equals(indexName)) { - adminClient().performRequest(new Request("DELETE", "/" + indexName)); + sdkAdminClient().performRequest(new Request("DELETE", "/" + indexName)); } } } diff --git a/src/test/java/org/opensearch/ad/TestHelpers.java b/src/test/java/org/opensearch/ad/TestHelpers.java index 518e704aa..e8922c6f6 100644 --- a/src/test/java/org/opensearch/ad/TestHelpers.java +++ b/src/test/java/org/opensearch/ad/TestHelpers.java @@ -99,7 +99,6 @@ import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.client.WarningsHandler; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -135,6 +134,7 @@ import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.schedule.Schedule; import org.opensearch.rest.RestStatus; +import org.opensearch.sdk.SDKClient.SDKRestClient; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchModule; @@ -155,11 +155,12 @@ public class TestHelpers { - public static final String LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI = "/_opendistro/_anomaly_detection/detectors"; - public static final String AD_BASE_DETECTORS_URI = "/_plugins/_anomaly_detection/detectors"; + public static final String LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI = + "/_extensions/_ad-extension/_opendistro/_anomaly_detection/detectors"; + public static final String AD_BASE_DETECTORS_URI = "/_extensions/_ad-extension/detectors"; public static final String AD_BASE_RESULT_URI = AD_BASE_DETECTORS_URI + "/results"; public static final String AD_BASE_PREVIEW_URI = AD_BASE_DETECTORS_URI + "/%s/_preview"; - public static final String AD_BASE_STATS_URI = "/_plugins/_anomaly_detection/stats"; + public static final String AD_BASE_STATS_URI = "/_extensions/_ad-extension/stats"; public static ImmutableSet HISTORICAL_ANALYSIS_RUNNING_STATS = ImmutableSet .of(ADTaskState.CREATED.name(), ADTaskState.INIT.name(), ADTaskState.RUNNING.name()); // Task may fail if memory circuit breaker triggered. @@ -171,7 +172,7 @@ public class TestHelpers { public static final Random random = new Random(42); public static Response makeRequest( - RestClient client, + SDKRestClient client, String method, String endpoint, Map params, @@ -183,7 +184,7 @@ public static Response makeRequest( } public static Response makeRequest( - RestClient client, + SDKRestClient client, String method, String endpoint, Map params, @@ -194,7 +195,7 @@ public static Response makeRequest( } public static Response makeRequest( - RestClient client, + SDKRestClient client, String method, String endpoint, Map params, @@ -1103,7 +1104,7 @@ public static CreateIndexResponse createIndex(AdminClient adminClient, String in return adminClient.indices().create(request).actionGet(5_000); } - public static void createIndex(RestClient client, String indexName, HttpEntity data) throws IOException { + public static void createIndex(SDKRestClient client, String indexName, HttpEntity data) throws IOException { TestHelpers .makeRequest( client, @@ -1115,7 +1116,7 @@ public static void createIndex(RestClient client, String indexName, HttpEntity d ); } - public static void createIndexWithTimeField(RestClient client, String indexName, String timeField) throws IOException { + public static void createIndexWithTimeField(SDKRestClient client, String indexName, String timeField) throws IOException { StringBuilder indexMappings = new StringBuilder(); indexMappings.append("{\"properties\":{"); indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}"); @@ -1124,7 +1125,7 @@ public static void createIndexWithTimeField(RestClient client, String indexName, createIndexMapping(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity(indexMappings.toString())); } - public static void createEmptyIndexWithTimeField(RestClient client, String indexName, String timeField) throws IOException { + public static void createEmptyIndexWithTimeField(SDKRestClient client, String indexName, String timeField) throws IOException { StringBuilder indexMappings = new StringBuilder(); indexMappings.append("{\"properties\":{"); indexMappings.append("\"" + timeField + "\":{\"type\":\"date\"}"); @@ -1133,7 +1134,7 @@ public static void createEmptyIndexWithTimeField(RestClient client, String index createIndexMapping(client, indexName.toLowerCase(Locale.ROOT), TestHelpers.toHttpEntity(indexMappings.toString())); } - public static void createIndexWithHCADFields(RestClient client, String indexName, Map categoryFieldsAndTypes) + public static void createIndexWithHCADFields(SDKRestClient client, String indexName, Map categoryFieldsAndTypes) throws IOException { StringBuilder indexMappings = new StringBuilder(); indexMappings.append("{\"properties\":{"); @@ -1146,20 +1147,20 @@ public static void createIndexWithHCADFields(RestClient client, String indexName createIndexMapping(client, indexName, TestHelpers.toHttpEntity(indexMappings.toString())); } - public static void createEmptyAnomalyResultIndex(RestClient client) throws IOException { + public static void createEmptyAnomalyResultIndex(SDKRestClient client) throws IOException { createEmptyIndex(client, CommonName.ANOMALY_RESULT_INDEX_ALIAS); createIndexMapping(client, CommonName.ANOMALY_RESULT_INDEX_ALIAS, toHttpEntity(AnomalyDetectionIndices.getAnomalyResultMappings())); } - public static void createEmptyIndex(RestClient client, String indexName) throws IOException { + public static void createEmptyIndex(SDKRestClient client, String indexName) throws IOException { TestHelpers.makeRequest(client, "PUT", "/" + indexName, ImmutableMap.of(), "", null); } - public static void createIndexMapping(RestClient client, String indexName, HttpEntity mappings) throws IOException { + public static void createIndexMapping(SDKRestClient client, String indexName, HttpEntity mappings) throws IOException { TestHelpers.makeRequest(client, "POST", "/" + indexName + "/_mapping", ImmutableMap.of(), mappings, null); } - public static void ingestDataToIndex(RestClient client, String indexName, HttpEntity data) throws IOException { + public static void ingestDataToIndex(SDKRestClient client, String indexName, HttpEntity data) throws IOException { TestHelpers .makeRequest( client, diff --git a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java index d4551f62f..3bf8aa1b1 100644 --- a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java +++ b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java @@ -5,28 +5,9 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ - +/* @anomaly.detection extension, commented until we have BWC test support for extensions package org.opensearch.ad.bwc; -import static org.opensearch.ad.rest.ADRestTestUtils.countADResultOfDetector; -import static org.opensearch.ad.rest.ADRestTestUtils.deleteDetector; -import static org.opensearch.ad.rest.ADRestTestUtils.searchLatestAdTaskOfDetector; -import static org.opensearch.ad.rest.ADRestTestUtils.startHistoricalAnalysis; -import static org.opensearch.ad.rest.ADRestTestUtils.waitUntilTaskDone; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.junit.Before; -import org.opensearch.ad.TestHelpers; -import org.opensearch.ad.model.ADTask; -import org.opensearch.ad.util.ExceptionUtil; -import org.opensearch.ad.util.RestHandlerUtils; -import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; -import org.opensearch.test.rest.OpenSearchRestTestCase; public class ADBackwardsCompatibilityIT extends OpenSearchRestTestCase { @@ -452,3 +433,4 @@ private void verifyAnomalyDetectorCount(String uri, long expectedCount) throws E } } +*/ diff --git a/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java b/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java index cad3b7a78..48868447d 100644 --- a/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java +++ b/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java @@ -49,13 +49,13 @@ import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.client.WarningsHandler; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sdk.SDKClient.SDKRestClient; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -86,7 +86,7 @@ private void verifyAnomaly( double minRecall, double maxError ) throws Exception { - RestClient client = client(); + SDKRestClient client = sdkRestClient(); String dataFileName = String.format(Locale.ROOT, "data/%s.data", datasetName); String labelFileName = String.format(Locale.ROOT, "data/%s.label", datasetName); @@ -144,7 +144,7 @@ private double[] getTestResults( int trainTestSplit, int intervalMinutes, List> anomalies, - RestClient client + SDKRestClient client ) throws Exception { double positives = 0; @@ -181,7 +181,7 @@ private double[] getTestResults( * @param trainTestSplit Training data size * @param shingleSize Shingle size * @param intervalMinutes Detector Interval - * @param client OpenSearch Client + * @param client SDKRestClient Client * @throws Exception when failing to query/indexing from/to OpenSearch */ private void simulateSingleStreamStartDetector( @@ -190,7 +190,7 @@ private void simulateSingleStreamStartDetector( int trainTestSplit, int shingleSize, int intervalMinutes, - RestClient client + SDKRestClient client ) throws Exception { Instant trainTime = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(data.get(trainTestSplit - 1).get("timestamp").getAsString())); @@ -228,7 +228,7 @@ private void simulateSingleStreamStartDetector( * @param trainTestSplit Training data size * @param shingleSize Shingle size * @param intervalMinutes Detector Interval - * @param client OpenSearch Client + * @param client SDKRest Client * @throws Exception when failing to query/indexing from/to OpenSearch */ private void simulateHCADStartDetector( @@ -237,7 +237,7 @@ private void simulateHCADStartDetector( int trainTestSplit, int shingleSize, int intervalMinutes, - RestClient client + SDKRestClient client ) throws Exception { Instant trainTime = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(data.get(trainTestSplit - 1).get("timestamp").getAsString())); @@ -277,8 +277,13 @@ private void simulateHCADStartDetector( } while (duration <= 60_000); } - private String createDetector(String datasetName, int intervalMinutes, RestClient client, String categoryField, long windowDelayInMins) - throws Exception { + private String createDetector( + String datasetName, + int intervalMinutes, + SDKRestClient client, + String categoryField, + long windowDelayInMins + ) throws Exception { Request request = new Request("POST", "/_plugins/_anomaly_detection/detectors/"); String requestBody = null; if (Strings.isEmpty(categoryField)) { @@ -336,8 +341,13 @@ private List> getAnomalyWindows(String labalFileName) th return anomalies; } - private void bulkIndexTrainData(String datasetName, List data, int trainTestSplit, RestClient client, String categoryField) - throws Exception { + private void bulkIndexTrainData( + String datasetName, + List data, + int trainTestSplit, + SDKRestClient client, + String categoryField + ) throws Exception { Request request = new Request("PUT", datasetName); String requestBody = null; if (Strings.isEmpty(categoryField)) { @@ -377,7 +387,7 @@ private void bulkIndexTrainData(String datasetName, List data, int t waitAllSyncheticDataIngested(trainTestSplit, datasetName, client); } - private void bulkIndexTestData(List data, String datasetName, int trainTestSplit, RestClient client) throws Exception { + private void bulkIndexTestData(List data, String datasetName, int trainTestSplit, SDKRestClient client) throws Exception { StringBuilder bulkRequestBuilder = new StringBuilder(); for (int i = trainTestSplit; i < data.size(); i++) { bulkRequestBuilder.append("{ \"index\" : { \"_index\" : \"" + datasetName + "\", \"_id\" : \"" + i + "\" } }\n"); @@ -396,7 +406,7 @@ private void bulkIndexTestData(List data, String datasetName, int tr waitAllSyncheticDataIngested(data.size(), datasetName, client); } - private void waitAllSyncheticDataIngested(int expectedSize, String datasetName, RestClient client) throws Exception { + private void waitAllSyncheticDataIngested(int expectedSize, String datasetName, SDKRestClient client) throws Exception { int maxWaitCycles = 3; do { Request request = new Request("POST", String.format(Locale.ROOT, "/%s/_search", datasetName)); @@ -453,7 +463,7 @@ private List getData(String datasetFileName) throws Exception { return list; } - private Map getDetectionResult(String detectorId, Instant begin, Instant end, RestClient client) { + private Map getDetectionResult(String detectorId, Instant begin, Instant end, SDKRestClient client) { try { Request request = new Request( "POST", @@ -493,11 +503,11 @@ private void disableResourceNotFoundFaultTolerence() throws IOException { request.setJsonEntity(BytesReference.bytes(settingCommand).utf8ToString()); - adminClient().performRequest(request); + sdkAdminClient().performRequest(request); } public void testValidationIntervalRecommendation() throws Exception { - RestClient client = client(); + SDKRestClient client = sdkRestClient(); long recDetectorIntervalMillis = 180000; long recDetectorIntervalMinutes = recDetectorIntervalMillis / 60000; List data = createData(2000, recDetectorIntervalMillis); @@ -516,7 +526,7 @@ public void testValidationIntervalRecommendation() throws Exception { ); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/model", ImmutableMap.of(), @@ -534,7 +544,7 @@ public void testValidationIntervalRecommendation() throws Exception { } public void testValidationWindowDelayRecommendation() throws Exception { - RestClient client = client(); + SDKRestClient client = sdkRestClient(); long recDetectorIntervalMillis = 180000; // this would be equivalent to the window delay in this data test long recDetectorIntervalMinutes = recDetectorIntervalMillis / 60000; @@ -554,7 +564,7 @@ public void testValidationWindowDelayRecommendation() throws Exception { ); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/model", ImmutableMap.of(), @@ -588,7 +598,7 @@ private List createData(int numOfDataPoints, long detectorIntervalMS return list; } - private void indexTrainData(String datasetName, List data, int trainTestSplit, RestClient client) throws Exception { + private void indexTrainData(String datasetName, List data, int trainTestSplit, SDKRestClient client) throws Exception { Request request = new Request("PUT", datasetName); String requestBody = "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\": \"date\"}," + " \"Feature1\": { \"type\": \"long\" }, \"Feature2\": { \"type\": \"long\" } } } }"; @@ -623,7 +633,7 @@ public void testRestartHCADDetector() throws Exception { } private void verifyRestart(String datasetName, int intervalMinutes, int shingleSize) throws Exception { - RestClient client = client(); + SDKRestClient client = sdkRestClient(); String dataFileName = String.format(Locale.ROOT, "data/%s.data", datasetName); @@ -688,7 +698,7 @@ private void verifyRestart(String datasetName, int intervalMinutes, int shingleS assertEquals("init progress is " + initProgress, "100%", initProgress); } - private void stopDetector(String detectorId, RestClient client) throws Exception { + private void stopDetector(String detectorId, SDKRestClient client) throws Exception { Request request = new Request("POST", String.format(Locale.ROOT, "/_plugins/_anomaly_detection/detectors/%s/_stop", detectorId)); Map response = entityAsMap(client.performRequest(request)); @@ -696,7 +706,7 @@ private void stopDetector(String detectorId, RestClient client) throws Exception assertEquals(detectorId, responseDetectorId); } - private void startDetector(String detectorId, RestClient client) throws Exception { + private void startDetector(String detectorId, SDKRestClient client) throws Exception { Request request = new Request("POST", String.format(Locale.ROOT, "/_plugins/_anomaly_detection/detectors/%s/_start", detectorId)); Map response = entityAsMap(client.performRequest(request)); @@ -704,7 +714,7 @@ private void startDetector(String detectorId, RestClient client) throws Exceptio assertEquals(detectorId, responseDetectorId); } - private String profileDetectorInitProgress(String detectorId, RestClient client) throws Exception { + private String profileDetectorInitProgress(String detectorId, SDKRestClient client) throws Exception { Request request = new Request( "GET", String.format(Locale.ROOT, "/_plugins/_anomaly_detection/detectors/%s/_profile/init_progress", detectorId) diff --git a/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java b/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java index 39f6c2e99..456f82135 100644 --- a/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java +++ b/src/test/java/org/opensearch/ad/rest/ADRestTestUtils.java @@ -40,8 +40,8 @@ import org.opensearch.ad.model.DetectionDateRange; import org.opensearch.ad.model.IntervalTimeConfiguration; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.sdk.SDKClient.SDKRestClient; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -57,7 +57,7 @@ public enum DetectorType { } public static Response ingestSimpleMockLog( - RestClient client, + SDKRestClient client, String indexName, int startDays, int totalDocsPerCategory, @@ -115,7 +115,7 @@ public static Response ingestSimpleMockLog( } public static Response ingestTestDataForHistoricalAnalysis( - RestClient client, + SDKRestClient client, String indexName, int detectionIntervalInMinutes, boolean createIndex, @@ -133,7 +133,7 @@ public static Response ingestTestDataForHistoricalAnalysis( } @SuppressWarnings("unchecked") - public static int getDocCountOfIndex(RestClient client, String indexName) throws IOException { + public static int getDocCountOfIndex(SDKRestClient client, String indexName) throws IOException { Response searchResponse = TestHelpers .makeRequest( client, @@ -150,7 +150,7 @@ public static int getDocCountOfIndex(RestClient client, String indexName) throws } public static Response createAnomalyDetector( - RestClient client, + SDKRestClient client, String indexName, String timeField, int detectionIntervalInMinutes, @@ -175,7 +175,7 @@ public static Response createAnomalyDetector( } public static Response createAnomalyDetector( - RestClient client, + SDKRestClient client, String indexName, String timeField, int detectionIntervalInMinutes, @@ -224,7 +224,7 @@ public static Response createAnomalyDetector( } @SuppressWarnings("unchecked") - public static List searchLatestAdTaskOfDetector(RestClient client, String detectorId, String taskType) throws IOException { + public static List searchLatestAdTaskOfDetector(SDKRestClient client, String detectorId, String taskType) throws IOException { List adTasks = new ArrayList<>(); Response searchAdTaskResponse = TestHelpers .makeRequest( @@ -276,7 +276,7 @@ public static List searchLatestAdTaskOfDetector(RestClient client, Strin } @SuppressWarnings("unchecked") - public static int countADResultOfDetector(RestClient client, String detectorId, String taskId) throws IOException { + public static int countADResultOfDetector(SDKRestClient client, String detectorId, String taskId) throws IOException { String taskFilter = "TASK_FILTER"; String query = "{\"query\":{\"bool\":{\"filter\":[{\"term\":{\"detector_id\":\"" + detectorId @@ -304,7 +304,7 @@ public static int countADResultOfDetector(RestClient client, String detectorId, } @SuppressWarnings("unchecked") - public static int countDetectors(RestClient client, String detectorType) throws IOException { + public static int countDetectors(SDKRestClient client, String detectorType) throws IOException { String detectorTypeFilter = "DETECTOR_TYPE_FILTER"; String query = "{\"query\":{\"bool\":{\"filter\":[{\"exists\":{\"field\":\"name\"}}" + detectorTypeFilter @@ -413,7 +413,7 @@ private static ADTask parseAdTask(Map taskMap) { * @throws IOException exception may throw in entityAsMap */ @SuppressWarnings("unchecked") - public static String startAnomalyDetectorDirectly(RestClient client, String detectorId) throws IOException { + public static String startAnomalyDetectorDirectly(SDKRestClient client, String detectorId) throws IOException { Response response = TestHelpers .makeRequest( client, @@ -441,7 +441,7 @@ public static String startAnomalyDetectorDirectly(RestClient client, String dete * @throws IOException exception may throw in toHttpEntity and entityAsMap */ @SuppressWarnings("unchecked") - public static String startHistoricalAnalysis(RestClient client, String detectorId) throws IOException { + public static String startHistoricalAnalysis(SDKRestClient client, String detectorId) throws IOException { Instant now = Instant.now(); DetectionDateRange dateRange = new DetectionDateRange(now.minus(30, ChronoUnit.DAYS), now); Response response = TestHelpers @@ -460,11 +460,11 @@ public static String startHistoricalAnalysis(RestClient client, String detectorI return taskId; } - public static ADTaskProfile waitUntilTaskDone(RestClient client, String detectorId) throws InterruptedException { + public static ADTaskProfile waitUntilTaskDone(SDKRestClient client, String detectorId) throws InterruptedException { return waitUntilTaskReachState(client, detectorId, TestHelpers.HISTORICAL_ANALYSIS_DONE_STATS); } - public static ADTaskProfile waitUntilTaskReachState(RestClient client, String detectorId, Set targetStates) + public static ADTaskProfile waitUntilTaskReachState(SDKRestClient client, String detectorId, Set targetStates) throws InterruptedException { int i = 0; int retryTimes = 200; @@ -483,7 +483,7 @@ public static ADTaskProfile waitUntilTaskReachState(RestClient client, String de return adTaskProfile; } - public static ADTaskProfile getADTaskProfile(RestClient client, String detectorId) throws IOException, ParseException { + public static ADTaskProfile getADTaskProfile(SDKRestClient client, String detectorId) throws IOException, ParseException { Response profileResponse = TestHelpers .makeRequest( client, @@ -512,15 +512,15 @@ public static ADTaskProfile parseADTaskProfile(Response profileResponse) throws return adTaskProfile; } - public static Response stopRealtimeJob(RestClient client, String detectorId) throws IOException { + public static Response stopRealtimeJob(SDKRestClient client, String detectorId) throws IOException { return stopDetector(client, detectorId, false); } - public static Response stopHistoricalAnalysis(RestClient client, String detectorId) throws IOException { + public static Response stopHistoricalAnalysis(SDKRestClient client, String detectorId) throws IOException { return stopDetector(client, detectorId, true); } - public static Response stopDetector(RestClient client, String detectorId, boolean historicalAnalysis) throws IOException { + public static Response stopDetector(SDKRestClient client, String detectorId, boolean historicalAnalysis) throws IOException { String param = historicalAnalysis ? "?historical" : ""; Response response = TestHelpers .makeRequest( @@ -534,7 +534,7 @@ public static Response stopDetector(RestClient client, String detectorId, boolea return response; } - public static Response deleteDetector(RestClient client, String detectorId) throws IOException { + public static Response deleteDetector(SDKRestClient client, String detectorId) throws IOException { Response response = TestHelpers .makeRequest( client, diff --git a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java index 04dbd94e2..262a50449 100644 --- a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -12,6 +12,10 @@ package org.opensearch.ad.rest; import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG; import static org.opensearch.ad.rest.handler.AbstractAnomalyDetectorActionHandler.DUPLICATE_DETECTOR_MSG; import static org.opensearch.ad.rest.handler.AbstractAnomalyDetectorActionHandler.NO_DOCS_IN_USER_INDEX_MSG; @@ -30,7 +34,6 @@ import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.StringEntity; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.AnomalyDetectorRestTestCase; import org.opensearch.ad.TestHelpers; @@ -43,7 +46,6 @@ import org.opensearch.ad.model.Feature; import org.opensearch.ad.rest.handler.AbstractAnomalyDetectorActionHandler; import org.opensearch.ad.settings.AnomalyDetectorSettings; -import org.opensearch.ad.settings.EnabledSetting; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.UUIDs; @@ -68,7 +70,7 @@ public void testCreateAnomalyDetectorWithNotExistingIndices() throws Exception { "index_not_found_exception", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), @@ -82,7 +84,7 @@ public void testCreateAnomalyDetectorWithEmptyIndices() throws Exception { AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TestHelpers.randomUiMetadata(), null); TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/" + detector.getIndices().get(0), ImmutableMap.of(), @@ -99,7 +101,7 @@ public void testCreateAnomalyDetectorWithEmptyIndices() throws Exception { "Can't create anomaly detector as no document is found in the indices", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), @@ -114,9 +116,9 @@ private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName) throw } private AnomalyDetector createIndexAndGetAnomalyDetector(String indexName, List features) throws IOException { - TestHelpers.createIndexWithTimeField(client(), indexName, TIME_FIELD); + TestHelpers.createIndexWithTimeField(sdkRestClient(), indexName, TIME_FIELD); String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}"; - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TIME_FIELD, indexName, features); return detector; } @@ -149,7 +151,7 @@ public void testCreateAnomalyDetectorWithDuplicateName() throws Exception { "Cannot create anomaly detector with name", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), @@ -161,25 +163,15 @@ public void testCreateAnomalyDetectorWithDuplicateName() throws Exception { public void testCreateAnomalyDetector() throws Exception { AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "POST", - TestHelpers.AD_BASE_DETECTORS_URI, - ImmutableMap.of(), - TestHelpers.toHttpEntity(detector), - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); Response response = TestHelpers - .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), TestHelpers.toHttpEntity(detector), null); + .makeRequest( + sdkRestClient(), + "POST", + TestHelpers.AD_BASE_DETECTORS_URI, + ImmutableMap.of(), + TestHelpers.toHttpEntity(detector), + null + ); assertEquals("Create anomaly detector failed", RestStatus.CREATED, TestHelpers.restStatus(response)); Map responseMap = entityAsMap(response); String id = (String) responseMap.get("_id"); @@ -188,11 +180,17 @@ public void testCreateAnomalyDetector() throws Exception { assertTrue("incorrect version", version > 0); } - // @anomaly-detection.create-detector Commented this code until we have support of Update Detector for extensibility - /*public void testUpdateAnomalyDetectorCategoryField() throws Exception { + public void testUpdateAnomalyDetectorCategoryField() throws Exception { AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); Response response = TestHelpers - .makeRequest(client(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), TestHelpers.toHttpEntity(detector), null); + .makeRequest( + sdkRestClient(), + "POST", + TestHelpers.AD_BASE_DETECTORS_URI, + ImmutableMap.of(), + TestHelpers.toHttpEntity(detector), + null + ); assertEquals("Create anomaly detector failed", RestStatus.CREATED, TestHelpers.restStatus(response)); Map responseMap = entityAsMap(response); String id = (String) responseMap.get("_id"); @@ -219,7 +217,7 @@ public void testCreateAnomalyDetector() throws Exception { ResponseException.class, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", TestHelpers.AD_BASE_DETECTORS_URI + "/" + id + "?refresh=true", ImmutableMap.of(), @@ -228,32 +226,22 @@ public void testCreateAnomalyDetector() throws Exception { ) ); assertThat(ex.getMessage(), containsString(CommonErrorMessages.CAN_NOT_CHANGE_CATEGORY_FIELD)); - }*/ + } - @Ignore public void testGetAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows(ResponseException.class, () -> getAnomalyDetector(detector.getDetectorId(), client())); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - - AnomalyDetector createdDetector = getAnomalyDetector(detector.getDetectorId(), client()); + AnomalyDetector createdDetector = getAnomalyDetector(detector.getDetectorId(), sdkRestClient()); assertEquals("Incorrect Location header", detector, createdDetector); } - @Ignore public void testGetNotExistingAnomalyDetector() throws Exception { - createRandomAnomalyDetector(true, true, client()); - TestHelpers.assertFailWith(ResponseException.class, null, () -> getAnomalyDetector(randomAlphaOfLength(5), client())); + createRandomAnomalyDetector(true, true, sdkRestClient()); + TestHelpers.assertFailWith(ResponseException.class, null, () -> getAnomalyDetector(randomAlphaOfLength(5), sdkRestClient())); } - @Ignore public void testUpdateAnomalyDetector() throws Exception { - AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, client()); + AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, sdkRestClient()); String newDescription = randomAlphaOfLength(5); AnomalyDetector newDetector = new AnomalyDetector( detector.getDetectorId(), @@ -275,27 +263,9 @@ public void testUpdateAnomalyDetector() throws Exception { null ); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "PUT", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "?refresh=true", - ImmutableMap.of(), - TestHelpers.toHttpEntity(newDetector), - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response updateResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "?refresh=true", ImmutableMap.of(), @@ -308,7 +278,7 @@ public void testUpdateAnomalyDetector() throws Exception { assertEquals("Updated anomaly detector id doesn't match", detector.getDetectorId(), responseBody.get("_id")); assertEquals("Version not incremented", (detector.getVersion().intValue() + 1), (int) responseBody.get("_version")); - AnomalyDetector updatedDetector = getAnomalyDetector(detector.getDetectorId(), client()); + AnomalyDetector updatedDetector = getAnomalyDetector(detector.getDetectorId(), sdkRestClient()); assertNotEquals("Anomaly detector last update time not changed", updatedDetector.getLastUpdateTime(), detector.getLastUpdateTime()); assertEquals("Anomaly detector description not updated", newDescription, updatedDetector.getDescription()); } @@ -342,7 +312,7 @@ public void testUpdateAnomalyDetectorNameToExisting() throws Exception { "Cannot create anomaly detector with name", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI, ImmutableMap.of(), @@ -352,9 +322,8 @@ public void testUpdateAnomalyDetectorNameToExisting() throws Exception { ); } - @Ignore public void testUpdateAnomalyDetectorNameToNew() throws Exception { - AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, client()); + AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, sdkRestClient()); AnomalyDetector detectorWithNewName = new AnomalyDetector( detector.getDetectorId(), detector.getVersion(), @@ -377,7 +346,7 @@ public void testUpdateAnomalyDetectorNameToNew() throws Exception { TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "?refresh=true", ImmutableMap.of(), @@ -385,7 +354,7 @@ public void testUpdateAnomalyDetectorNameToNew() throws Exception { null ); - AnomalyDetector resultDetector = getAnomalyDetector(detectorWithNewName.getDetectorId(), client()); + AnomalyDetector resultDetector = getAnomalyDetector(detectorWithNewName.getDetectorId(), sdkRestClient()); assertEquals("Detector name updating failed", detectorWithNewName.getName(), resultDetector.getName()); assertEquals("Updated anomaly detector id doesn't match", detectorWithNewName.getDetectorId(), resultDetector.getDetectorId()); assertNotEquals( @@ -395,9 +364,8 @@ public void testUpdateAnomalyDetectorNameToNew() throws Exception { ); } - @Ignore public void testUpdateAnomalyDetectorWithNotExistingIndex() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); String newDescription = randomAlphaOfLength(5); @@ -429,7 +397,7 @@ public void testUpdateAnomalyDetectorWithNotExistingIndex() throws Exception { null, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), ImmutableMap.of(), @@ -439,32 +407,13 @@ public void testUpdateAnomalyDetectorWithNotExistingIndex() throws Exception { ); } - @Ignore public void testSearchAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); SearchSourceBuilder search = (new SearchSourceBuilder()).query(QueryBuilders.termQuery("_id", detector.getDetectorId())); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "GET", - TestHelpers.AD_BASE_DETECTORS_URI + "/_search", - ImmutableMap.of(), - new StringEntity(search.toString(), ContentType.APPLICATION_JSON), - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response searchResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "GET", TestHelpers.AD_BASE_DETECTORS_URI + "/_search", ImmutableMap.of(), @@ -474,26 +423,15 @@ public void testSearchAnomalyDetector() throws Exception { assertEquals("Search anomaly detector failed", RestStatus.OK, TestHelpers.restStatus(searchResponse)); } - @Ignore public void testStatsAnomalyDetector() throws Exception { - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers.makeRequest(client(), "GET", AnomalyDetectorPlugin.LEGACY_AD_BASE + "/stats", ImmutableMap.of(), "", null) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response statsResponse = TestHelpers - .makeRequest(client(), "GET", AnomalyDetectorPlugin.LEGACY_AD_BASE + "/stats", ImmutableMap.of(), "", null); + .makeRequest(sdkRestClient(), "GET", AnomalyDetectorPlugin.LEGACY_AD_BASE + "/stats", ImmutableMap.of(), "", null); assertEquals("Get stats failed", RestStatus.OK, TestHelpers.restStatus(statsResponse)); } - @Ignore public void testPreviewAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( detector.getDetectorId(), Instant.now().minusSeconds(60 * 10), @@ -501,27 +439,9 @@ public void testPreviewAnomalyDetector() throws Exception { null ); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "POST", - String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), - ImmutableMap.of(), - TestHelpers.toHttpEntity(input), - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), ImmutableMap.of(), @@ -531,9 +451,8 @@ public void testPreviewAnomalyDetector() throws Exception { assertEquals("Execute anomaly detector failed", RestStatus.OK, TestHelpers.restStatus(response)); } - @Ignore public void testPreviewAnomalyDetectorWhichNotExist() throws Exception { - createRandomAnomalyDetector(true, false, client()); + createRandomAnomalyDetector(true, false, sdkRestClient()); AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( randomAlphaOfLength(5), Instant.now().minusSeconds(60 * 10), @@ -545,7 +464,7 @@ public void testPreviewAnomalyDetectorWhichNotExist() throws Exception { ResponseException.class, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), ImmutableMap.of(), @@ -555,7 +474,6 @@ public void testPreviewAnomalyDetectorWhichNotExist() throws Exception { ); } - @Ignore public void testExecuteAnomalyDetectorWithNullDetectorId() throws Exception { AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( null, @@ -568,7 +486,7 @@ public void testExecuteAnomalyDetectorWithNullDetectorId() throws Exception { ResponseException.class, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), ImmutableMap.of(), @@ -578,9 +496,8 @@ public void testExecuteAnomalyDetectorWithNullDetectorId() throws Exception { ); } - @Ignore public void testPreviewAnomalyDetectorWithDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( detector.getDetectorId(), Instant.now().minusSeconds(60 * 10), @@ -589,7 +506,7 @@ public void testPreviewAnomalyDetectorWithDetector() throws Exception { ); Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), ImmutableMap.of(), @@ -600,9 +517,8 @@ public void testPreviewAnomalyDetectorWithDetector() throws Exception { assertEquals("Execute anomaly detector failed", RestStatus.OK, TestHelpers.restStatus(response)); } - @Ignore public void testPreviewAnomalyDetectorWithDetectorAndNoFeatures() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); AnomalyDetectorExecutionInput input = new AnomalyDetectorExecutionInput( detector.getDetectorId(), Instant.now().minusSeconds(60 * 10), @@ -615,7 +531,7 @@ public void testPreviewAnomalyDetectorWithDetectorAndNoFeatures() throws Excepti "Can't preview detector without feature", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", String.format(Locale.ROOT, TestHelpers.AD_BASE_PREVIEW_URI, input.getDetectorId()), ImmutableMap.of(), @@ -625,12 +541,11 @@ public void testPreviewAnomalyDetectorWithDetectorAndNoFeatures() throws Excepti ); } - @Ignore public void testSearchAnomalyResult() throws Exception { AnomalyResult anomalyResult = TestHelpers.randomAnomalyDetectResult(); Response response = TestHelpers .makeRequest( - adminClient(), + sdkAdminClient(), "POST", "/.opendistro-anomaly-results/_doc/" + UUIDs.base64UUID(), ImmutableMap.of(), @@ -643,27 +558,9 @@ public void testSearchAnomalyResult() throws Exception { SearchSourceBuilder search = (new SearchSourceBuilder()) .query(QueryBuilders.termQuery("detector_id", anomalyResult.getDetectorId())); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "POST", - TestHelpers.AD_BASE_RESULT_URI + "/_search", - ImmutableMap.of(), - new StringEntity(search.toString(), ContentType.APPLICATION_JSON), - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response searchResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_RESULT_URI + "/_search", ImmutableMap.of(), @@ -675,7 +572,7 @@ public void testSearchAnomalyResult() throws Exception { SearchSourceBuilder searchAll = SearchSourceBuilder.fromXContent(TestHelpers.parser("{\"query\":{\"match_all\":{}}}")); Response searchAllResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_RESULT_URI + "/_search", ImmutableMap.of(), @@ -685,30 +582,12 @@ public void testSearchAnomalyResult() throws Exception { assertEquals("Search anomaly result failed", RestStatus.OK, TestHelpers.restStatus(searchAllResponse)); } - @Ignore public void testDeleteAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "DELETE", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), - ImmutableMap.of(), - "", - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), ImmutableMap.of(), @@ -718,14 +597,13 @@ public void testDeleteAnomalyDetector() throws Exception { assertEquals("Delete anomaly detector failed", RestStatus.OK, TestHelpers.restStatus(response)); } - @Ignore public void testDeleteAnomalyDetectorWhichNotExist() throws Exception { TestHelpers .assertFailWith( ResponseException.class, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", TestHelpers.AD_BASE_DETECTORS_URI + "/" + randomAlphaOfLength(5), ImmutableMap.of(), @@ -735,12 +613,11 @@ public void testDeleteAnomalyDetectorWhichNotExist() throws Exception { ); } - @Ignore public void testDeleteAnomalyDetectorWithNoAdJob() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), ImmutableMap.of(), @@ -750,12 +627,11 @@ public void testDeleteAnomalyDetectorWithNoAdJob() throws Exception { assertEquals("Delete anomaly detector failed", RestStatus.OK, TestHelpers.restStatus(response)); } - @Ignore public void testDeleteAnomalyDetectorWithRunningAdJob() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -771,7 +647,7 @@ public void testDeleteAnomalyDetectorWithRunningAdJob() throws Exception { "Detector job is running", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), ImmutableMap.of(), @@ -781,12 +657,11 @@ public void testDeleteAnomalyDetectorWithRunningAdJob() throws Exception { ); } - @Ignore public void testUpdateAnomalyDetectorWithRunningAdJob() throws Exception { - AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, client()); + AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, sdkRestClient()); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -824,7 +699,7 @@ public void testUpdateAnomalyDetectorWithRunningAdJob() throws Exception { "Detector job is running", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId(), ImmutableMap.of(), @@ -836,10 +711,10 @@ public void testUpdateAnomalyDetectorWithRunningAdJob() throws Exception { // // @anomaly-detection.create-detector Commented this code until we have support of Get Detector for extensibility // public void testGetDetectorWithAdJob() throws Exception { - // AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + // AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); // Response startAdJobResponse = TestHelpers // .makeRequest( - // client(), + // sdkRestClient(), // "POST", // TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", // ImmutableMap.of(), @@ -849,40 +724,22 @@ public void testUpdateAnomalyDetectorWithRunningAdJob() throws Exception { // // assertEquals("Fail to start AD job", RestStatus.OK, TestHelpers.restStatus(startAdJobResponse)); // - // ToXContentObject[] results = getAnomalyDetector(detector.getDetectorId(), true, client()); + // ToXContentObject[] results = getAnomalyDetector(detector.getDetectorId(), true, sdkRestClient()); // assertEquals("Incorrect Location header", detector, results[0]); // assertEquals("Incorrect detector job name", detector.getDetectorId(), ((AnomalyDetectorJob) results[1]).getName()); // assertTrue(((AnomalyDetectorJob) results[1]).isEnabled()); // - // results = getAnomalyDetector(detector.getDetectorId(), false, client()); + // results = getAnomalyDetector(detector.getDetectorId(), false, sdkRestClient()); // assertEquals("Incorrect Location header", detector, results[0]); // assertEquals("Should not return detector job", null, results[1]); // } - @Ignore public void testStartAdJobWithExistingDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "POST", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", - ImmutableMap.of(), - "", - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -894,7 +751,7 @@ public void testStartAdJobWithExistingDetector() throws Exception { startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -905,7 +762,6 @@ public void testStartAdJobWithExistingDetector() throws Exception { assertEquals("Fail to start AD job", RestStatus.OK, TestHelpers.restStatus(startAdJobResponse)); } - @Ignore public void testStartAdJobWithNonexistingDetectorIndex() throws Exception { TestHelpers .assertFailWith( @@ -913,7 +769,7 @@ public void testStartAdJobWithNonexistingDetectorIndex() throws Exception { "no such index [.opendistro-anomaly-detectors]", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + randomAlphaOfLength(10) + "/_start", ImmutableMap.of(), @@ -923,16 +779,15 @@ public void testStartAdJobWithNonexistingDetectorIndex() throws Exception { ); } - @Ignore public void testStartAdJobWithNonexistingDetector() throws Exception { - createRandomAnomalyDetector(true, false, client()); + createRandomAnomalyDetector(true, false, sdkRestClient()); TestHelpers .assertFailWith( ResponseException.class, FAIL_TO_FIND_DETECTOR_MSG, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + randomAlphaOfLength(10) + "/_start", ImmutableMap.of(), @@ -942,13 +797,11 @@ public void testStartAdJobWithNonexistingDetector() throws Exception { ); } - @Ignore public void testStopAdJob() throws Exception { - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -957,27 +810,9 @@ public void testStopAdJob() throws Exception { ); assertEquals("Fail to start AD job", RestStatus.OK, TestHelpers.restStatus(startAdJobResponse)); - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows( - ResponseException.class, - () -> TestHelpers - .makeRequest( - client(), - "POST", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", - ImmutableMap.of(), - "", - null - ) - ); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); - Response stopAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", ImmutableMap.of(), @@ -988,7 +823,7 @@ public void testStopAdJob() throws Exception { stopAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", ImmutableMap.of(), @@ -998,16 +833,15 @@ public void testStopAdJob() throws Exception { assertEquals("Fail to stop AD job", RestStatus.OK, TestHelpers.restStatus(stopAdJobResponse)); } - @Ignore public void testStopNonExistingAdJobIndex() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); TestHelpers .assertFailWith( ResponseException.class, "no such index [.opendistro-anomaly-detector-jobs]", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", ImmutableMap.of(), @@ -1017,12 +851,11 @@ public void testStopNonExistingAdJobIndex() throws Exception { ); } - @Ignore public void testStopNonExistingAdJob() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -1037,7 +870,7 @@ public void testStopNonExistingAdJob() throws Exception { FAIL_TO_FIND_DETECTOR_MSG, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + randomAlphaOfLength(10) + "/_stop", ImmutableMap.of(), @@ -1047,12 +880,11 @@ public void testStopNonExistingAdJob() throws Exception { ); } - @Ignore public void testStartDisabledAdjob() throws IOException { - AnomalyDetector detector = createRandomAnomalyDetector(true, false, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, false, sdkRestClient()); Response startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -1063,7 +895,7 @@ public void testStartDisabledAdjob() throws IOException { Response stopAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", ImmutableMap.of(), @@ -1074,7 +906,7 @@ public void testStartDisabledAdjob() throws IOException { startAdJobResponse = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -1085,19 +917,18 @@ public void testStartDisabledAdjob() throws IOException { assertEquals("Fail to start AD job", RestStatus.OK, TestHelpers.restStatus(startAdJobResponse)); } - @Ignore public void testStartAdjobWithNullFeatures() throws Exception { AnomalyDetector detectorWithoutFeature = TestHelpers.randomAnomalyDetector(null, null, Instant.now()); String indexName = detectorWithoutFeature.getIndices().get(0); - TestHelpers.createIndex(client(), indexName, TestHelpers.toHttpEntity("{\"name\": \"test\"}")); - AnomalyDetector detector = createAnomalyDetector(detectorWithoutFeature, true, client()); + TestHelpers.createIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity("{\"name\": \"test\"}")); + AnomalyDetector detector = createAnomalyDetector(detectorWithoutFeature, true, sdkRestClient()); TestHelpers .assertFailWith( ResponseException.class, "Can't start detector job as no features configured", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -1107,19 +938,18 @@ public void testStartAdjobWithNullFeatures() throws Exception { ); } - @Ignore public void testStartAdjobWithEmptyFeatures() throws Exception { AnomalyDetector detectorWithoutFeature = TestHelpers.randomAnomalyDetector(ImmutableList.of(), null, Instant.now()); String indexName = detectorWithoutFeature.getIndices().get(0); - TestHelpers.createIndex(client(), indexName, TestHelpers.toHttpEntity("{\"name\": \"test\"}")); - AnomalyDetector detector = createAnomalyDetector(detectorWithoutFeature, true, client()); + TestHelpers.createIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity("{\"name\": \"test\"}")); + AnomalyDetector detector = createAnomalyDetector(detectorWithoutFeature, true, sdkRestClient()); TestHelpers .assertFailWith( ResponseException.class, "Can't start detector job as no features configured", () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_start", ImmutableMap.of(), @@ -1129,36 +959,25 @@ public void testStartAdjobWithEmptyFeatures() throws Exception { ); } - @Ignore public void testDefaultProfileAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, false); - - Exception ex = expectThrows(ResponseException.class, () -> getDetectorProfile(detector.getDetectorId())); - assertThat(ex.getMessage(), containsString(CommonErrorMessages.DISABLED_ERR_MSG)); - - updateClusterSettings(EnabledSetting.AD_PLUGIN_ENABLED, true); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Response profileResponse = getDetectorProfile(detector.getDetectorId()); assertEquals("Incorrect profile status", RestStatus.OK, TestHelpers.restStatus(profileResponse)); } - @Ignore public void testAllProfileAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Response profileResponse = getDetectorProfile(detector.getDetectorId(), true); assertEquals("Incorrect profile status", RestStatus.OK, TestHelpers.restStatus(profileResponse)); } - @Ignore public void testCustomizedProfileAnomalyDetector() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); - Response profileResponse = getDetectorProfile(detector.getDetectorId(), true, "/models/", client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); + Response profileResponse = getDetectorProfile(detector.getDetectorId(), true, "/models/", sdkRestClient()); assertEquals("Incorrect profile status", RestStatus.OK, TestHelpers.restStatus(profileResponse)); } - @Ignore public void testSearchAnomalyDetectorCountNoIndex() throws Exception { Response countResponse = getSearchDetectorCount(); Map responseMap = entityAsMap(countResponse); @@ -1166,16 +985,14 @@ public void testSearchAnomalyDetectorCountNoIndex() throws Exception { assertEquals((long) count, 0); } - @Ignore public void testSearchAnomalyDetectorCount() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Response countResponse = getSearchDetectorCount(); Map responseMap = entityAsMap(countResponse); Integer count = (Integer) responseMap.get("count"); assertEquals((long) count, 1); } - @Ignore public void testSearchAnomalyDetectorMatchNoIndex() throws Exception { Response matchResponse = getSearchDetectorMatch("name"); Map responseMap = entityAsMap(matchResponse); @@ -1183,61 +1000,60 @@ public void testSearchAnomalyDetectorMatchNoIndex() throws Exception { assertEquals(nameExists, false); } - @Ignore public void testSearchAnomalyDetectorNoMatch() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Response matchResponse = getSearchDetectorMatch(detector.getName()); Map responseMap = entityAsMap(matchResponse); boolean nameExists = (boolean) responseMap.get("match"); assertEquals(nameExists, true); } - @Ignore public void testSearchAnomalyDetectorMatch() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Response matchResponse = getSearchDetectorMatch(detector.getName() + "newDetector"); Map responseMap = entityAsMap(matchResponse); boolean nameExists = (boolean) responseMap.get("match"); assertEquals(nameExists, false); } - @Ignore public void testRunDetectorWithNoEnabledFeature() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client(), false); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient(), false); Assert.assertNotNull(detector.getDetectorId()); Instant now = Instant.now(); ResponseException e = expectThrows( ResponseException.class, - () -> startAnomalyDetector(detector.getDetectorId(), new DetectionDateRange(now.minus(10, ChronoUnit.DAYS), now), client()) + () -> startAnomalyDetector( + detector.getDetectorId(), + new DetectionDateRange(now.minus(10, ChronoUnit.DAYS), now), + sdkRestClient() + ) ); assertTrue(e.getMessage().contains("Can't start detector job as no enabled features configured")); } - @Ignore public void testDeleteAnomalyDetectorWhileRunning() throws Exception { - AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); + AnomalyDetector detector = createRandomAnomalyDetector(true, true, sdkRestClient()); Assert.assertNotNull(detector.getDetectorId()); Instant now = Instant.now(); Response response = startAnomalyDetector( detector.getDetectorId(), new DetectionDateRange(now.minus(10, ChronoUnit.DAYS), now), - client() + sdkRestClient() ); Assert.assertEquals(response.getStatusLine().toString(), "HTTP/1.1 200 OK"); // Deleting detector should fail while its running - Exception exception = expectThrows(IOException.class, () -> { deleteAnomalyDetector(detector.getDetectorId(), client()); }); + Exception exception = expectThrows(IOException.class, () -> { deleteAnomalyDetector(detector.getDetectorId(), sdkRestClient()); }); Assert.assertTrue(exception.getMessage().contains("Detector is running")); } - @Ignore public void testBackwardCompatibilityWithOpenDistro() throws IOException { // Create a detector AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); // Verify the detector is created using legacy _opendistro API Response response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI, ImmutableMap.of(), @@ -1252,13 +1068,13 @@ public void testBackwardCompatibilityWithOpenDistro() throws IOException { assertTrue("incorrect version", version > 0); // Get the detector using new _plugins API - AnomalyDetector createdDetector = getAnomalyDetector(id, client()); + AnomalyDetector createdDetector = getAnomalyDetector(id, sdkRestClient()); assertEquals("Get anomaly detector failed", createdDetector.getDetectorId(), id); // Delete the detector using legacy _opendistro API response = TestHelpers .makeRequest( - client(), + sdkRestClient(), "DELETE", TestHelpers.LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI + "/" + createdDetector.getDetectorId(), ImmutableMap.of(), @@ -1269,12 +1085,11 @@ public void testBackwardCompatibilityWithOpenDistro() throws IOException { } - @Ignore public void testValidateAnomalyDetectorWithDuplicateName() throws Exception { - AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, client()); + AnomalyDetector detector = createAnomalyDetector(createIndexAndGetAnomalyDetector(INDEX_NAME), true, sdkRestClient()); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate", ImmutableMap.of(), @@ -1306,12 +1121,12 @@ public void testValidateAnomalyDetectorWithDuplicateName() throws Exception { assertEquals("duplicate error message", errorMsg, messageMap.get("name").get("message")); } - @Ignore public void testValidateAnomalyDetectorWithNoTimeField() throws Exception { - TestHelpers.createIndex(client(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); + TestHelpers + .createIndex(sdkRestClient(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate", ImmutableMap.of(), @@ -1333,12 +1148,12 @@ public void testValidateAnomalyDetectorWithNoTimeField() throws Exception { assertEquals("time field missing", CommonErrorMessages.NULL_TIME_FIELD, messageMap.get("time_field").get("message")); } - @Ignore public void testValidateAnomalyDetectorWithIncorrectShingleSize() throws Exception { - TestHelpers.createIndex(client(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); + TestHelpers + .createIndex(sdkRestClient(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate", ImmutableMap.of(), @@ -1370,12 +1185,11 @@ public void testValidateAnomalyDetectorWithIncorrectShingleSize() throws Excepti assertEquals("shingle size error message", errorMessage, messageMap.get("shingle_size").get("message")); } - @Ignore public void testValidateAnomalyDetectorWithNoIssue() throws Exception { AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/detector", ImmutableMap.of(), @@ -1386,7 +1200,6 @@ public void testValidateAnomalyDetectorWithNoIssue() throws Exception { assertEquals("no issue, empty response body", new HashMap(), responseMap); } - @Ignore public void testValidateAnomalyDetectorOnWrongValidationType() throws Exception { AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME); TestHelpers @@ -1395,7 +1208,7 @@ public void testValidateAnomalyDetectorOnWrongValidationType() throws Exception CommonErrorMessages.NOT_EXISTENT_VALIDATION_TYPE, () -> TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/models", ImmutableMap.of(), @@ -1405,12 +1218,11 @@ public void testValidateAnomalyDetectorOnWrongValidationType() throws Exception ); } - @Ignore public void testValidateAnomalyDetectorWithEmptyIndices() throws Exception { AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TIME_FIELD, INDEX_NAME); TestHelpers .makeRequest( - client(), + sdkRestClient(), "PUT", "/" + detector.getIndices().get(0), ImmutableMap.of(), @@ -1425,7 +1237,7 @@ public void testValidateAnomalyDetectorWithEmptyIndices() throws Exception { ); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate", ImmutableMap.of(), @@ -1441,12 +1253,12 @@ public void testValidateAnomalyDetectorWithEmptyIndices() throws Exception { assertEquals("duplicate error message", errorMessage, messageMap.get("indices").get("message")); } - @Ignore public void testValidateAnomalyDetectorWithInvalidName() throws Exception { - TestHelpers.createIndex(client(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); + TestHelpers + .createIndex(sdkRestClient(), "test-index", TestHelpers.toHttpEntity("{\"timestamp\": " + Instant.now().toEpochMilli() + "}")); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/detector", ImmutableMap.of(), @@ -1467,13 +1279,12 @@ public void testValidateAnomalyDetectorWithInvalidName() throws Exception { assertEquals("invalid detector Name", CommonErrorMessages.INVALID_DETECTOR_NAME, messageMap.get("name").get("message")); } - @Ignore public void testValidateAnomalyDetectorWithFeatureQueryReturningNoData() throws Exception { Feature emptyFeature = TestHelpers.randomFeature("f-empty", "cpu", "avg", true); AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME, ImmutableList.of(emptyFeature)); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/detector", ImmutableMap.of(), @@ -1491,13 +1302,12 @@ public void testValidateAnomalyDetectorWithFeatureQueryReturningNoData() throws ); } - @Ignore public void testValidateAnomalyDetectorWithFeatureQueryRuntimeException() throws Exception { Feature nonNumericFeature = TestHelpers.randomFeature("non-numeric-feature", "_index", "avg", true); AnomalyDetector detector = createIndexAndGetAnomalyDetector(INDEX_NAME, ImmutableList.of(nonNumericFeature)); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/detector", ImmutableMap.of(), @@ -1515,7 +1325,6 @@ public void testValidateAnomalyDetectorWithFeatureQueryRuntimeException() throws ); } - @Ignore public void testValidateAnomalyDetectorWithWrongCategoryField() throws Exception { AnomalyDetector detector = TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1524,10 +1333,10 @@ public void testValidateAnomalyDetectorWithWrongCategoryField() throws Exception ImmutableList.of("index-test"), Arrays.asList("host.keyword") ); - TestHelpers.createIndexWithTimeField(client(), "index-test", TIME_FIELD); + TestHelpers.createIndexWithTimeField(sdkRestClient(), "index-test", TIME_FIELD); Response resp = TestHelpers .makeRequest( - client(), + sdkRestClient(), "POST", TestHelpers.AD_BASE_DETECTORS_URI + "/_validate/detector", ImmutableMap.of(), @@ -1546,7 +1355,6 @@ public void testValidateAnomalyDetectorWithWrongCategoryField() throws Exception } - @Ignore public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); Map categoryFieldsAndTypes = new HashMap() { @@ -1556,8 +1364,8 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { } }; String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}"; - TestHelpers.createIndexWithHCADFields(client(), indexName, categoryFieldsAndTypes); - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.createIndexWithHCADFields(sdkRestClient(), indexName, categoryFieldsAndTypes); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = createAnomalyDetector( TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1567,30 +1375,35 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { categoryFieldsAndTypes.keySet().stream().collect(Collectors.toList()) ), true, - client() + sdkRestClient() ); // Missing start time Exception missingStartTimeException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), false, "{\"end_time_ms\":2}", client()); + searchTopAnomalyResults(detector.getDetectorId(), false, "{\"end_time_ms\":2}", sdkRestClient()); }); assertTrue(missingStartTimeException.getMessage().contains("Must set both start time and end time with epoch of milliseconds")); // Missing end time Exception missingEndTimeException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":1}", client()); + searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":1}", sdkRestClient()); }); assertTrue(missingEndTimeException.getMessage().contains("Must set both start time and end time with epoch of milliseconds")); // Start time > end time Exception invalidTimeException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":2, \"end_time_ms\":1}", client()); + searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":2, \"end_time_ms\":1}", sdkRestClient()); }); assertTrue(invalidTimeException.getMessage().contains("Start time should be before end time")); // Invalid detector ID Exception invalidDetectorIdException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId() + "-invalid", false, "{\"start_time_ms\":1, \"end_time_ms\":2}", client()); + searchTopAnomalyResults( + detector.getDetectorId() + "-invalid", + false, + "{\"start_time_ms\":1, \"end_time_ms\":2}", + sdkRestClient() + ); }); assertTrue(invalidDetectorIdException.getMessage().contains("Can't find detector with id")); @@ -1600,20 +1413,30 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { detector.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2, \"order\":\"invalid-order\"}", - client() + sdkRestClient() ); }); assertTrue(invalidOrderException.getMessage().contains("Ordering by invalid-order is not a valid option")); // Negative size field Exception negativeSizeException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2, \"size\":-1}", client()); + searchTopAnomalyResults( + detector.getDetectorId(), + false, + "{\"start_time_ms\":1, \"end_time_ms\":2, \"size\":-1}", + sdkRestClient() + ); }); assertTrue(negativeSizeException.getMessage().contains("Size must be a positive integer")); // Zero size field Exception zeroSizeException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2, \"size\":0}", client()); + searchTopAnomalyResults( + detector.getDetectorId(), + false, + "{\"start_time_ms\":1, \"end_time_ms\":2, \"size\":0}", + sdkRestClient() + ); }); assertTrue(zeroSizeException.getMessage().contains("Size must be a positive integer")); @@ -1623,14 +1446,14 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { detector.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2, \"size\":9999999}", - client() + sdkRestClient() ); }); assertTrue(tooLargeSizeException.getMessage().contains("Size cannot exceed")); // No existing task ID for detector Exception noTaskIdException = expectThrows(IOException.class, () -> { - searchTopAnomalyResults(detector.getDetectorId(), true, "{\"start_time_ms\":1, \"end_time_ms\":2}", client()); + searchTopAnomalyResults(detector.getDetectorId(), true, "{\"start_time_ms\":1, \"end_time_ms\":2}", sdkRestClient()); }); assertTrue(noTaskIdException.getMessage().contains("No historical tasks found for detector ID " + detector.getDetectorId())); @@ -1640,7 +1463,7 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { detector.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2, \"category_field\":[\"invalid-field\"]}", - client() + sdkRestClient() ); }); assertTrue( @@ -1659,14 +1482,14 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { ImmutableList.of() ), true, - client() + sdkRestClient() ); Exception noCategoryFieldsException = expectThrows(IOException.class, () -> { searchTopAnomalyResults( detectorWithNoCategoryFields.getDetectorId(), false, "{\"start_time_ms\":1, \"end_time_ms\":2}", - client() + sdkRestClient() ); }); assertTrue( @@ -1676,7 +1499,6 @@ public void testSearchTopAnomalyResultsWithInvalidInputs() throws IOException { ); } - @Ignore public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOException { String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); Map categoryFieldsAndTypes = new HashMap() { @@ -1686,8 +1508,8 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep } }; String testIndexData = "{\"keyword-field\": \"test-value\"}"; - TestHelpers.createIndexWithHCADFields(client(), indexName, categoryFieldsAndTypes); - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.createIndexWithHCADFields(sdkRestClient(), indexName, categoryFieldsAndTypes); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = createAnomalyDetector( TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1697,7 +1519,7 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep categoryFieldsAndTypes.keySet().stream().collect(Collectors.toList()) ), true, - client() + sdkRestClient() ); // Delete any existing result index @@ -1708,7 +1530,7 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep detector.getDetectorId(), false, "{\"size\":3,\"category_field\":[\"keyword-field\"]," + "\"start_time_ms\":0, \"end_time_ms\":1}", - client() + sdkRestClient() ); Map responseMap = entityAsMap(response); @SuppressWarnings("unchecked") @@ -1716,7 +1538,6 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep assertEquals(0, buckets.size()); } - @Ignore public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); Map categoryFieldsAndTypes = new HashMap() { @@ -1726,8 +1547,8 @@ public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { } }; String testIndexData = "{\"keyword-field\": \"test-value\"}"; - TestHelpers.createIndexWithHCADFields(client(), indexName, categoryFieldsAndTypes); - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.createIndexWithHCADFields(sdkRestClient(), indexName, categoryFieldsAndTypes); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = createAnomalyDetector( TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1737,19 +1558,19 @@ public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { categoryFieldsAndTypes.keySet().stream().collect(Collectors.toList()) ), true, - client() + sdkRestClient() ); // Clear any existing result index, create an empty one if (indexExistsWithAdminClient(CommonName.ANOMALY_RESULT_INDEX_ALIAS)) { deleteIndexWithAdminClient(CommonName.ANOMALY_RESULT_INDEX_ALIAS); } - TestHelpers.createEmptyAnomalyResultIndex(adminClient()); + TestHelpers.createEmptyAnomalyResultIndex(sdkAdminClient()); Response response = searchTopAnomalyResults( detector.getDetectorId(), false, "{\"size\":3,\"category_field\":[\"keyword-field\"]," + "\"start_time_ms\":0, \"end_time_ms\":1}", - client() + sdkRestClient() ); Map responseMap = entityAsMap(response); @SuppressWarnings("unchecked") @@ -1757,7 +1578,6 @@ public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { assertEquals(0, buckets.size()); } - @Ignore public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOException { String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); Map categoryFieldsAndTypes = new HashMap() { @@ -1767,8 +1587,8 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti } }; String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}"; - TestHelpers.createIndexWithHCADFields(client(), indexName, categoryFieldsAndTypes); - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.createIndexWithHCADFields(sdkRestClient(), indexName, categoryFieldsAndTypes); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = createAnomalyDetector( TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1778,12 +1598,12 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti categoryFieldsAndTypes.keySet().stream().collect(Collectors.toList()) ), true, - client() + sdkRestClient() ); // Ingest some sample results if (!indexExistsWithAdminClient(CommonName.ANOMALY_RESULT_INDEX_ALIAS)) { - TestHelpers.createEmptyAnomalyResultIndex(adminClient()); + TestHelpers.createEmptyAnomalyResultIndex(sdkAdminClient()); } Map entityAttrs1 = new HashMap() { { @@ -1810,16 +1630,16 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti AnomalyResult anomalyResult3 = TestHelpers .randomHCADAnomalyDetectResult(detector.getDetectorId(), null, entityAttrs3, 0.5, 0.2, null, 5L, 5L); - TestHelpers.ingestDataToIndex(adminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult1)); - TestHelpers.ingestDataToIndex(adminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult2)); - TestHelpers.ingestDataToIndex(adminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult3)); + TestHelpers.ingestDataToIndex(sdkAdminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult1)); + TestHelpers.ingestDataToIndex(sdkAdminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult2)); + TestHelpers.ingestDataToIndex(sdkAdminClient(), CommonName.ANOMALY_RESULT_INDEX_ALIAS, TestHelpers.toHttpEntity(anomalyResult3)); // Sorting by severity Response severityResponse = searchTopAnomalyResults( detector.getDetectorId(), false, "{\"category_field\":[\"keyword-field\"]," + "\"start_time_ms\":0, \"end_time_ms\":10, \"order\":\"severity\"}", - client() + sdkRestClient() ); Map severityResponseMap = entityAsMap(severityResponse); @SuppressWarnings("unchecked") @@ -1838,7 +1658,7 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti detector.getDetectorId(), false, "{\"category_field\":[\"keyword-field\"]," + "\"start_time_ms\":0, \"end_time_ms\":10, \"order\":\"occurrence\"}", - client() + sdkRestClient() ); Map occurrenceResponseMap = entityAsMap(occurrenceResponse); @SuppressWarnings("unchecked") @@ -1857,7 +1677,7 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti detector.getDetectorId(), false, "{\"category_field\":[\"keyword-field\", \"ip-field\"]," + "\"start_time_ms\":0, \"end_time_ms\":10, \"order\":\"severity\"}", - client() + sdkRestClient() ); Map allFieldsResponseMap = entityAsMap(allFieldsResponse); @SuppressWarnings("unchecked") @@ -1874,7 +1694,6 @@ public void testSearchTopAnomalyResultsOnPopulatedResultIndex() throws IOExcepti assertEquals("5.6.7.8", allFieldsBucketKey2.get("ip-field")); } - @Ignore public void testSearchTopAnomalyResultsWithCustomResultIndex() throws IOException { String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); String customResultIndexName = CommonName.CUSTOM_RESULT_INDEX_PREFIX + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); @@ -1885,8 +1704,8 @@ public void testSearchTopAnomalyResultsWithCustomResultIndex() throws IOExceptio } }; String testIndexData = "{\"keyword-field\": \"field-1\", \"ip-field\": \"1.2.3.4\", \"timestamp\": 1}"; - TestHelpers.createIndexWithHCADFields(client(), indexName, categoryFieldsAndTypes); - TestHelpers.ingestDataToIndex(client(), indexName, TestHelpers.toHttpEntity(testIndexData)); + TestHelpers.createIndexWithHCADFields(sdkRestClient(), indexName, categoryFieldsAndTypes); + TestHelpers.ingestDataToIndex(sdkRestClient(), indexName, TestHelpers.toHttpEntity(testIndexData)); AnomalyDetector detector = createAnomalyDetector( TestHelpers .randomAnomalyDetectorUsingCategoryFields( @@ -1897,7 +1716,7 @@ public void testSearchTopAnomalyResultsWithCustomResultIndex() throws IOExceptio customResultIndexName ), true, - client() + sdkRestClient() ); Map entityAttrs = new HashMap() { @@ -1908,9 +1727,14 @@ public void testSearchTopAnomalyResultsWithCustomResultIndex() throws IOExceptio }; AnomalyResult anomalyResult = TestHelpers .randomHCADAnomalyDetectResult(detector.getDetectorId(), null, entityAttrs, 0.5, 0.8, null, 5L, 5L); - TestHelpers.ingestDataToIndex(client(), customResultIndexName, TestHelpers.toHttpEntity(anomalyResult)); + TestHelpers.ingestDataToIndex(sdkRestClient(), customResultIndexName, TestHelpers.toHttpEntity(anomalyResult)); - Response response = searchTopAnomalyResults(detector.getDetectorId(), false, "{\"start_time_ms\":0, \"end_time_ms\":10}", client()); + Response response = searchTopAnomalyResults( + detector.getDetectorId(), + false, + "{\"start_time_ms\":0, \"end_time_ms\":10}", + sdkRestClient() + ); Map responseMap = entityAsMap(response); @SuppressWarnings("unchecked") List> buckets = (ArrayList>) XContentMapValues.extractValue("buckets", responseMap); diff --git a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java index 5c491febd..64e7bfdb2 100644 --- a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java +++ b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java @@ -25,28 +25,28 @@ import org.opensearch.ad.model.AnomalyDetectorExecutionInput; import org.opensearch.ad.model.DetectionDateRange; import org.opensearch.client.Response; -import org.opensearch.client.RestClient; import org.opensearch.rest.RestStatus; +import org.opensearch.sdk.SDKClient.SDKRestClient; import com.google.common.collect.ImmutableList; public class SecureADRestIT extends AnomalyDetectorRestTestCase { String aliceUser = "alice"; - RestClient aliceClient; + SDKRestClient aliceClient; String bobUser = "bob"; - RestClient bobClient; + SDKRestClient bobClient; String catUser = "cat"; - RestClient catClient; + SDKRestClient catClient; String dogUser = "dog"; - RestClient dogClient; + SDKRestClient dogClient; String elkUser = "elk"; - RestClient elkClient; + SDKRestClient elkClient; String fishUser = "fish"; - RestClient fishClient; + SDKRestClient fishClient; String goatUser = "goat"; - RestClient goatClient; + SDKRestClient goatClient; String lionUser = "lion"; - RestClient lionClient; + SDKRestClient lionClient; private String indexAllAccessRole = "index_all_access"; private String indexSearchAccessRole = "index_all_search"; @@ -209,7 +209,7 @@ public void testUpdateApiFilterByEnabled() throws IOException { // User Fish has AD full access, and has "odfe" backend role which is one of Alice's backend role, so // Fish should be able to update detectors created by Alice. But the detector's backend role should // not be replaced as Fish's backend roles. - TestHelpers.createIndexWithTimeField(client(), newDetector.getIndices().get(0), newDetector.getTimeField()); + TestHelpers.createIndexWithTimeField(sdkRestClient(), newDetector.getIndices().get(0), newDetector.getTimeField()); Response response = updateAnomalyDetector(aliceDetector.getDetectorId(), newDetector, fishClient); Assert.assertEquals(response.getStatusLine().getStatusCode(), 200); AnomalyDetector anomalyDetector = getAnomalyDetector(aliceDetector.getDetectorId(), aliceClient); @@ -300,7 +300,7 @@ public void testCreateAnomalyDetectorWithCustomResultIndex() throws IOException // User cat has permission to create index resultIndex = CommonName.CUSTOM_RESULT_INDEX_PREFIX + "test2"; - TestHelpers.createIndexWithTimeField(client(), anomalyDetector.getIndices().get(0), anomalyDetector.getTimeField()); + TestHelpers.createIndexWithTimeField(sdkRestClient(), anomalyDetector.getIndices().get(0), anomalyDetector.getTimeField()); AnomalyDetector detectorOfCat = createAnomalyDetector(cloneDetector(anomalyDetector, resultIndex), true, catClient); assertEquals(resultIndex, detectorOfCat.getResultIndex()); } diff --git a/src/test/resources/org/opensearch/ad/integ/anomaly-detection-extension.yml b/src/test/resources/org/opensearch/ad/integ/anomaly-detection-extension.yml new file mode 100644 index 000000000..79f4a4b44 --- /dev/null +++ b/src/test/resources/org/opensearch/ad/integ/anomaly-detection-extension.yml @@ -0,0 +1,7 @@ +name: ad-extension +uniqueId: ad-extension +hostAddress: '127.0.0.1' +port: '4532' +version: '1.0' +opensearchVersion: '3.0.0' +minimumCompatibleVersion: '3.0.0' \ No newline at end of file