From b4f88e8e538feb79cfa692d790fef431ec56d4dc Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Sun, 14 Sep 2025 12:55:25 +0200 Subject: [PATCH 1/5] ES|QL: refactor query generator and move it to testFixtures --- .../rest/generative/GenerativeRestTest.java | 56 +++++++------- .../esql/qa/rest/generative/README.asciidoc | 15 +--- .../xpack/esql/generator/Column.java | 12 +++ .../esql/generator}/EsqlQueryGenerator.java | 77 +++++++++---------- .../xpack/esql/generator/LookupIdx.java | 12 +++ .../xpack/esql/generator/LookupIdxColumn.java | 10 +++ .../xpack/esql/generator/QueryExecuted.java | 12 +++ .../xpack/esql/generator/QueryExecutor.java | 12 +++ .../xpack/esql/generator/README.asciidoc | 24 ++++++ .../generator}/command/CommandGenerator.java | 46 ++++++----- .../command/pipe/ChangePointGenerator.java | 25 +++--- .../command/pipe/DissectGenerator.java | 17 ++-- .../command/pipe/DropGenerator.java | 19 +++-- .../command/pipe/EnrichGenerator.java | 17 ++-- .../command/pipe/EvalGenerator.java | 19 +++-- .../command/pipe/ForkGenerator.java | 40 +++++----- .../command/pipe/GrokGenerator.java | 17 ++-- .../command/pipe/KeepGenerator.java | 17 ++-- .../command/pipe/LimitGenerator.java | 16 ++-- .../command/pipe/LookupJoinGenerator.java | 29 +++---- .../command/pipe/MvExpandGenerator.java | 17 ++-- .../command/pipe/RenameGenerator.java | 24 +++--- .../command/pipe/SortGenerator.java | 17 ++-- .../command/pipe/StatsGenerator.java | 19 +++-- .../command/pipe/WhereGenerator.java | 19 +++-- .../command/source/FromGenerator.java | 20 ++--- 26 files changed, 361 insertions(+), 247 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/EsqlQueryGenerator.java (80%) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdx.java create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdxColumn.java create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecuted.java create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecutor.java create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/CommandGenerator.java (81%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/ChangePointGenerator.java (66%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/DissectGenerator.java (83%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/DropGenerator.java (85%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/EnrichGenerator.java (78%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/EvalGenerator.java (85%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/ForkGenerator.java (81%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/GrokGenerator.java (83%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/KeepGenerator.java (84%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/LimitGenerator.java (77%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/LookupJoinGenerator.java (80%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/MvExpandGenerator.java (73%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/RenameGenerator.java (82%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/SortGenerator.java (78%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/StatsGenerator.java (81%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/pipe/WhereGenerator.java (79%) rename x-pack/plugin/esql/qa/{server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative => testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator}/command/source/FromGenerator.java (68%) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 46bcfb72fb07a..759952e5de556 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -9,13 +9,18 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.core.Nullable; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.esql.AssertWarnings; import org.elasticsearch.xpack.esql.CsvTestsDataLoader; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.LookupIdx; +import org.elasticsearch.xpack.esql.generator.LookupIdxColumn; +import org.elasticsearch.xpack.esql.generator.QueryExecuted; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; @@ -32,11 +37,11 @@ import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_POLICIES; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.availableDatasetsForEs; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs; -import static org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator.COLUMN_NAME; -import static org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator.COLUMN_ORIGINAL_TYPES; -import static org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator.COLUMN_TYPE; +import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_NAME; +import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_ORIGINAL_TYPES; +import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_TYPE; -public abstract class GenerativeRestTest extends ESRestTestCase { +public abstract class GenerativeRestTest extends ESRestTestCase implements QueryExecutor { @Rule(order = Integer.MIN_VALUE) public ProfileLogger profileLogger = new ProfileLogger(); @@ -104,9 +109,9 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription previousCommands.add(current); final String command = current.commandString(); - final EsqlQueryGenerator.QueryExecuted result = previousResult == null - ? execute(command, 0, profileLogger) - : execute(previousResult.query() + command, previousResult.depth(), profileLogger); + final QueryExecuted result = previousResult == null + ? execute(command, 0) + : execute(previousResult.query() + command, previousResult.depth()); previousResult = result; final boolean hasException = result.exception() != null; @@ -133,16 +138,16 @@ public boolean continueExecuting() { } @Override - public List currentSchema() { + public List currentSchema() { return currentSchema; } boolean continueExecuting; - List currentSchema; + List currentSchema; final List previousCommands = new ArrayList<>(); - EsqlQueryGenerator.QueryExecuted previousResult; + QueryExecuted previousResult; }; - EsqlQueryGenerator.generatePipeline(MAX_DEPTH, EsqlQueryGenerator.sourceCommand(), mappingInfo, exec); + EsqlQueryGenerator.generatePipeline(MAX_DEPTH, EsqlQueryGenerator.sourceCommand(), mappingInfo, exec, this); } } @@ -150,8 +155,8 @@ private static CommandGenerator.ValidationResult checkResults( List previousCommands, CommandGenerator commandGenerator, CommandGenerator.CommandDescription commandDescription, - EsqlQueryGenerator.QueryExecuted previousResult, - EsqlQueryGenerator.QueryExecuted result + QueryExecuted previousResult, + QueryExecuted result ) { CommandGenerator.ValidationResult outputValidation = commandGenerator.validateOutput( previousCommands, @@ -172,7 +177,7 @@ private static CommandGenerator.ValidationResult checkResults( return outputValidation; } - private void checkException(EsqlQueryGenerator.QueryExecuted query) { + private void checkException(QueryExecuted query) { for (Pattern allowedError : ALLOWED_ERROR_PATTERNS) { if (isAllowedError(query.exception().getMessage(), allowedError)) { return; @@ -192,8 +197,9 @@ private static boolean isAllowedError(String errorMessage, Pattern allowedPatter return allowedPattern.matcher(errorWithoutLineBreaks).matches(); } + @Override @SuppressWarnings("unchecked") - public static EsqlQueryGenerator.QueryExecuted execute(String command, int depth, @Nullable ProfileLogger profileLogger) { + public QueryExecuted execute(String command, int depth) { try { Map json = RestEsqlTestCase.runEsql( new RestEsqlTestCase.RequestObjectBuilder().query(command).build(), @@ -201,26 +207,26 @@ public static EsqlQueryGenerator.QueryExecuted execute(String command, int depth profileLogger, RestEsqlTestCase.Mode.SYNC ); - List outputSchema = outputSchema(json); + List outputSchema = outputSchema(json); List> values = (List>) json.get("values"); - return new EsqlQueryGenerator.QueryExecuted(command, depth, outputSchema, values, null); + return new QueryExecuted(command, depth, outputSchema, values, null); } catch (Exception e) { - return new EsqlQueryGenerator.QueryExecuted(command, depth, null, null, e); + return new QueryExecuted(command, depth, null, null, e); } catch (AssertionError ae) { // this is for ensureNoWarnings() - return new EsqlQueryGenerator.QueryExecuted(command, depth, null, null, new RuntimeException(ae.getMessage())); + return new QueryExecuted(command, depth, null, null, new RuntimeException(ae.getMessage())); } } @SuppressWarnings("unchecked") - private static List outputSchema(Map a) { + private static List outputSchema(Map a) { List> cols = (List>) a.get("columns"); if (cols == null) { return null; } return cols.stream() - .map(x -> new EsqlQueryGenerator.Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))) + .map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))) .collect(Collectors.toList()); } @@ -240,10 +246,6 @@ private List availableIndices() throws IOException { .toList(); } - public record LookupIdxColumn(String name, String type) {} - - public record LookupIdx(String idxName, List keys) {} - private List lookupIndices() { List result = new ArrayList<>(); // we don't have key info from the dataset loader, let's hardcode it for now diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/README.asciidoc b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/README.asciidoc index 0c01e7c524ce3..d286cfa3687ad 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/README.asciidoc +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/README.asciidoc @@ -26,18 +26,11 @@ The result check happens at two levels: == Implementing your own command generator -If you implement a new command, and you want it to be tested by the generative tests, you can add a command generator here. +If you create a new command, we strongly recommend you to implement a command generator for it, so that it is tested +by the generative tests. -All you have to do is: +To do so, see the documentation in the +`x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc` file. -* add a class in `org.elasticsearch.xpack.esql.qa.rest.generative.command.source` (if it's a source command) or in `org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe` (if it's a pipe command) -* Implement `CommandGenerator` interface (see its javadoc, it should be explicative. Or just have a look at one of the existing commands, eg. `SortGenerator`) -** Implement `CommandGenerator.generate()` method, that will return the command. -*** Have a look at `EsqlQueryGenerator`, it contains many utility methods that will help you generate random expressions. -** Implement `CommandGenerator.validateOutput()` to validate the output of the query. -* Add your class to `EsqlQueryGenerator.SOURCE_COMMANDS` (if it's a source command) or `EsqlQueryGenerator.PIPE_COMMANDS` (if it's a pipe command). -* Run `GenerativeIT` at least a couple of times: these tests can be pretty noisy. -* If you get unexpected errors (real bugs in ES|QL), please open an issue and add the error to `GenerativeRestTest.ALLOWED_ERRORS`. Run tests again until everything works fine. -IMPORTANT: be careful when validating the output (Eg. the row count), as ES|QL can be quite non-deterministic when there are no SORTs diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java new file mode 100644 index 0000000000000..43f7776aadb87 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator; + +import java.util.List; + +public record Column(String name, String type, List originalTypes) {} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java similarity index 80% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java index b6cf5ff3a8d15..b868c232b65dc 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java @@ -5,26 +5,26 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative; +package org.elasticsearch.xpack.esql.generator; import org.elasticsearch.xpack.esql.CsvTestsDataLoader; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.ChangePointGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.DissectGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.DropGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.EnrichGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.EvalGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.ForkGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.GrokGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.KeepGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.LimitGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.LookupJoinGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.MvExpandGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.RenameGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.SortGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.StatsGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.WhereGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.source.FromGenerator; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.ChangePointGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.DissectGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.DropGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EnrichGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.ForkGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.GrokGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.KeepGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.LimitGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.LookupJoinGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.MvExpandGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.RenameGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.SortGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.StatsGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.WhereGenerator; +import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; import java.util.List; import java.util.Set; @@ -42,10 +42,6 @@ public class EsqlQueryGenerator { public static final String COLUMN_TYPE = "type"; public static final String COLUMN_ORIGINAL_TYPES = "original_types"; - public record Column(String name, String type, List originalTypes) {} - - public record QueryExecuted(String query, int depth, List outputSchema, List> result, Exception exception) {} - /** * These are commands that are at the beginning of the query, eg. FROM */ @@ -87,7 +83,7 @@ public interface Executor { boolean continueExecuting(); - List currentSchema(); + List currentSchema(); } @@ -95,9 +91,10 @@ public static void generatePipeline( final int depth, CommandGenerator commandGenerator, final CommandGenerator.QuerySchema schema, - Executor executor + Executor executor, + QueryExecutor queryExecutor ) { - CommandGenerator.CommandDescription desc = commandGenerator.generate(List.of(), List.of(), schema); + CommandGenerator.CommandDescription desc = commandGenerator.generate(List.of(), List.of(), schema, queryExecutor); executor.run(commandGenerator, desc); if (executor.continueExecuting() == false) { return; @@ -108,7 +105,7 @@ public static void generatePipeline( break; } commandGenerator = EsqlQueryGenerator.randomPipeCommandGenerator(); - desc = commandGenerator.generate(executor.previousCommands(), executor.currentSchema(), schema); + desc = commandGenerator.generate(executor.previousCommands(), executor.currentSchema(), schema, queryExecutor); if (desc == CommandGenerator.EMPTY_DESCRIPTION) { continue; } @@ -188,12 +185,12 @@ public static String randomGroupableName(List previousOutput) { } public static boolean groupable(Column col) { - return col.type.equals("keyword") - || col.type.equals("text") - || col.type.equals("long") - || col.type.equals("integer") - || col.type.equals("ip") - || col.type.equals("version"); + return col.type().equals("keyword") + || col.type().equals("text") + || col.type().equals("long") + || col.type().equals("integer") + || col.type().equals("ip") + || col.type().equals("version"); } /** @@ -209,12 +206,12 @@ public static String randomSortableName(List previousOutput) { } public static boolean sortable(Column col) { - return col.type.equals("keyword") - || col.type.equals("text") - || col.type.equals("long") - || col.type.equals("integer") - || col.type.equals("ip") - || col.type.equals("version"); + return col.type().equals("keyword") + || col.type().equals("text") + || col.type().equals("long") + || col.type().equals("integer") + || col.type().equals("ip") + || col.type().equals("version"); } public static String agg(List previousOutput) { @@ -308,7 +305,7 @@ public static String constantExpression() { /** * returns a random identifier or one of the existing names */ - public static String randomAttributeOrIdentifier(List previousOutput) { + public static String randomAttributeOrIdentifier(List previousOutput) { String name; if (randomBoolean()) { name = EsqlQueryGenerator.randomIdentifier(); @@ -335,7 +332,7 @@ public static boolean fieldCanBeUsed(Column field) { || field.name().equals("") // no dense vectors for now, they are not supported in most commands || field.type().contains("vector") - || field.originalTypes.stream().anyMatch(x -> x.contains("vector"))) == false; + || field.originalTypes().stream().anyMatch(x -> x.contains("vector"))) == false; } public static String unquote(String colName) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdx.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdx.java new file mode 100644 index 0000000000000..d03b086b97d6c --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdx.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator; + +import java.util.List; + +public record LookupIdx(String idxName, List keys) {} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdxColumn.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdxColumn.java new file mode 100644 index 0000000000000..8683d68682717 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/LookupIdxColumn.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator; + +public record LookupIdxColumn(String name, String type) {} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecuted.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecuted.java new file mode 100644 index 0000000000000..d9a37949e5eba --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecuted.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator; + +import java.util.List; + +public record QueryExecuted(String query, int depth, List outputSchema, List> result, Exception exception) {} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecutor.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecutor.java new file mode 100644 index 0000000000000..a6684aeb68ce3 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/QueryExecutor.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator; + +public interface QueryExecutor { + QueryExecuted execute(String command, int depth); +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc new file mode 100644 index 0000000000000..d9613f01e0b6b --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc @@ -0,0 +1,24 @@ += ES|QL Query Genereator + +These classes generate random ES|QL queries, that can then be used to test the ES|QL engine (eg. see `GenerativeIT`). +The intention is not to test the single commands, but rather to test how ES|QL query engine +(parser, optimizers, query layout, compute) manages very complex queries. + +== Implementing your own command generator + +If you implement a new command, and you want it to be tested by the generative tests, you can add a command generator here. + +All you have to do is: + +* add a class in `org.elasticsearch.xpack.esql.generator.command.source` (if it's a source command) + or in `org.elasticsearch.xpack.esql.generator.command.pipe` (if it's a pipe command) +* Implement `CommandGenerator` interface (see its javadoc, it should be explicative. Or just have a look at one of the existing commands, eg. `SortGenerator`) +** Implement `CommandGenerator.generate()` method, that will return the command. +*** Have a look at `EsqlQueryGenerator`, it contains many utility methods that will help you generate random expressions. +** Implement `CommandGenerator.validateOutput()` to validate the output of the query. +* Add your class to `EsqlQueryGenerator.SOURCE_COMMANDS` (if it's a source command) or `EsqlQueryGenerator.PIPE_COMMANDS` (if it's a pipe command). +* Run `GenerativeIT` at least a couple of times: these tests can be pretty noisy. +* If you get unexpected errors (real bugs in ES|QL), please open an issue and add the error to `GenerativeRestTest.ALLOWED_ERRORS`. Run tests again until everything works fine. + + +IMPORTANT: be careful when validating the output (Eg. the row count), as ES|QL can be quite non-deterministic when there are no SORTs diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/CommandGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java similarity index 81% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/CommandGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java index 7d22ea4c66a94..e18748be154f8 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/CommandGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java @@ -5,11 +5,13 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command; +package org.elasticsearch.xpack.esql.generator.command; import org.elasticsearch.xpack.esql.CsvTestsDataLoader; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeRestTest; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.LookupIdx; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; import java.util.List; import java.util.Map; @@ -30,11 +32,7 @@ public interface CommandGenerator { */ record CommandDescription(String commandName, CommandGenerator generator, String commandString, Map context) {} - record QuerySchema( - List baseIndices, - List lookupIndices, - List enrichPolicies - ) {} + record QuerySchema(List baseIndices, List lookupIndices, List enrichPolicies) {} record ValidationResult(boolean success, String errorMessage) {} @@ -42,8 +40,9 @@ record ValidationResult(boolean success, String errorMessage) {} @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { return EMPTY_DESCRIPTION; } @@ -52,9 +51,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription command, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return VALIDATION_OK; @@ -70,13 +69,15 @@ public ValidationResult validateOutput( * @param previousCommands the list of the previous commands in the query * @param previousOutput the output returned by the query so far. * @param schema The columns returned by the query so far. It contains name and type information for each column. + * @param executor * @return All the details about the generated command. See {@link CommandDescription}. * If something goes wrong and for some reason you can't generate a command, you should return {@link CommandGenerator#EMPTY_DESCRIPTION} */ CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ); /** @@ -87,7 +88,7 @@ CommandDescription generate( * @param command The description of the command you just generated. * It also contains the context information you stored during command generation. * @param previousColumns The output schema of the original query (without last generated command). - * It contains name and type information for each column, see {@link EsqlQueryGenerator.Column} + * It contains name and type information for each column, see {@link Column} * @param previousOutput The output of the original query (without last generated command), as a list (rows) of lists (columns) of values * @param columns The output schema of the full query (WITH last generated command). * @param output The output of the full query (WITH last generated command), as a list (rows) of lists (columns) of values @@ -98,9 +99,9 @@ CommandDescription generate( ValidationResult validateOutput( List previousCommands, CommandDescription command, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ); @@ -118,7 +119,7 @@ static ValidationResult expectSameRowCount( return VALIDATION_OK; } - static ValidationResult expectSameColumns(List previousColumns, List columns) { + static ValidationResult expectSameColumns(List previousColumns, List columns) { if (previousColumns.stream().anyMatch(x -> x.name().contains(""))) { return VALIDATION_OK; // known bug @@ -128,8 +129,8 @@ static ValidationResult expectSameColumns(List previo return new ValidationResult(false, "Expecting [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - List prevColNames = previousColumns.stream().map(EsqlQueryGenerator.Column::name).toList(); - List newColNames = columns.stream().map(EsqlQueryGenerator.Column::name).toList(); + List prevColNames = previousColumns.stream().map(Column::name).toList(); + List newColNames = columns.stream().map(Column::name).toList(); if (prevColNames.equals(newColNames) == false) { return new ValidationResult( false, @@ -143,10 +144,7 @@ static ValidationResult expectSameColumns(List previo /** * The command doesn't have to produce LESS columns than the previous query */ - static ValidationResult expectAtLeastSameNumberOfColumns( - List previousColumns, - List columns - ) { + static ValidationResult expectAtLeastSameNumberOfColumns(List previousColumns, List columns) { if (previousColumns.stream().anyMatch(x -> x.name().contains(""))) { return VALIDATION_OK; // known bug } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ChangePointGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java similarity index 66% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ChangePointGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java index a0d71c47d413b..2b04cc9aced80 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ChangePointGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java @@ -5,16 +5,16 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; -import static org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator.randomAttributeOrIdentifier; - public class ChangePointGenerator implements CommandGenerator { public static final String CHANGE_POINT = "change_point"; public static final CommandGenerator INSTANCE = new ChangePointGenerator(); @@ -22,18 +22,19 @@ public class ChangePointGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String timestampField = EsqlQueryGenerator.randomDateField(previousOutput); String numericField = EsqlQueryGenerator.randomNumericField(previousOutput); if (timestampField == null || numericField == null) { return EMPTY_DESCRIPTION; } - String alias1 = randomAttributeOrIdentifier(previousOutput); - String alias2 = randomAttributeOrIdentifier(previousOutput); + String alias1 = EsqlQueryGenerator.randomAttributeOrIdentifier(previousOutput); + String alias2 = EsqlQueryGenerator.randomAttributeOrIdentifier(previousOutput); while (alias1.equals(alias2)) { - alias2 = randomAttributeOrIdentifier(previousOutput); + alias2 = EsqlQueryGenerator.randomAttributeOrIdentifier(previousOutput); } String cmd = " | CHANGE_POINT " + numericField + " ON " + timestampField + " AS " + alias1 + ", " + alias2; @@ -45,9 +46,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription command, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return CommandGenerator.expectAtLeastSameNumberOfColumns(previousColumns, columns); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DissectGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java similarity index 83% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DissectGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java index 83f8ae983dcde..6e0f95c7e7846 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DissectGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -24,8 +26,9 @@ public class DissectGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String field = EsqlQueryGenerator.randomStringField(previousOutput); if (field == null) { @@ -60,9 +63,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DropGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java similarity index 85% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DropGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java index 8bf2597f808c0..97b754692f392 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/DropGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.HashSet; import java.util.List; @@ -29,8 +31,9 @@ public class DropGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { if (previousOutput.size() < 2) { return CommandGenerator.EMPTY_DESCRIPTION; // don't drop all of them, just do nothing @@ -67,16 +70,16 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; } Set droppedColumns = (Set) commandDescription.context().get(DROPPED_COLUMNS); - List resultColNames = columns.stream().map(EsqlQueryGenerator.Column::name).toList(); + List resultColNames = columns.stream().map(Column::name).toList(); // expected column names are unquoted already for (String droppedColumn : droppedColumns) { if (resultColNames.contains(droppedColumn)) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EnrichGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java similarity index 78% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EnrichGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java index aac8f16e13285..be3d547cf535c 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EnrichGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -23,8 +25,9 @@ public class EnrichGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String field = EsqlQueryGenerator.randomKeywordField(previousOutput); if (field == null || schema.enrichPolicies().isEmpty()) { @@ -43,9 +46,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EvalGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java similarity index 85% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EvalGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java index b49e313fa4e3f..8c2841dda3f04 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/EvalGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.ArrayList; import java.util.List; @@ -26,8 +28,9 @@ public class EvalGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { StringBuilder cmd = new StringBuilder(" | eval "); int nFields = randomIntBetween(1, 10); @@ -63,13 +66,13 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { List expectedColumns = (List) commandDescription.context().get(NEW_COLUMNS); - List resultColNames = columns.stream().map(EsqlQueryGenerator.Column::name).toList(); + List resultColNames = columns.stream().map(Column::name).toList(); List lastColumns = resultColNames.subList(resultColNames.size() - expectedColumns.size(), resultColNames.size()); lastColumns = lastColumns.stream().map(EvalGenerator::unquote).toList(); // expected column names are unquoted already diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ForkGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java similarity index 81% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ForkGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java index 74c4122156bc6..3dfca414a997a 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ForkGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java @@ -5,11 +5,13 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeRestTest; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecuted; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.ArrayList; import java.util.List; @@ -26,8 +28,9 @@ public class ForkGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { // FORK can only be allowed once - so we skip adding another FORK if we already have one // otherwise, most generated queries would only result in a validation error @@ -60,9 +63,9 @@ public void run(CommandGenerator generator, CommandDescription current) { // schema, we append the command. Enforcing the same schema is stricter than the Fork needs (it only needs types to be // the same on columns which are present), but given we currently generate independent sub-pipelines, this way we can // generate more valid Fork queries. - final EsqlQueryGenerator.QueryExecuted result = previousResult == null - ? GenerativeRestTest.execute(command, 0, null) - : GenerativeRestTest.execute(previousResult.query() + command, previousResult.depth(), null); + final QueryExecuted result = previousResult == null + ? executor.execute(command, 0) + : executor.execute(previousResult.query() + command, previousResult.depth()); previousResult = result; continueExecuting = result.exception() == null && result.outputSchema().equals(previousOutput); @@ -82,21 +85,22 @@ public boolean continueExecuting() { } @Override - public List currentSchema() { + public List currentSchema() { return previousOutput; } final List previousCommands = new ArrayList<>(); boolean continueExecuting; - EsqlQueryGenerator.QueryExecuted previousResult; + QueryExecuted previousResult; }; var gen = new CommandGenerator() { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { return new CommandDescription(FORK, this, completeCommand.toString(), Map.of()); } @@ -105,16 +109,16 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription command, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return VALIDATION_OK; } }; - EsqlQueryGenerator.generatePipeline(3, gen, schema, exec); + EsqlQueryGenerator.generatePipeline(3, gen, schema, exec, executor); if (exec.previousCommands().size() > 1) { String previousCmd = exec.previousCommands() .stream() @@ -136,9 +140,9 @@ public ValidationResult validateOutput( public ValidationResult validateOutput( List previousCommands, CommandDescription command, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/GrokGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java similarity index 83% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/GrokGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java index 60322eb12c351..162ccbfc21e15 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/GrokGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -24,8 +26,9 @@ public class GrokGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String field = EsqlQueryGenerator.randomStringField(previousOutput); if (field == null) { @@ -59,9 +62,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/KeepGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java similarity index 84% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/KeepGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java index f3f522576124e..e4cceabb21c9f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/KeepGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.HashSet; import java.util.List; @@ -28,8 +30,9 @@ public class KeepGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { int n = randomIntBetween(1, previousOutput.size()); Set proj = new HashSet<>(); @@ -63,9 +66,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LimitGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java similarity index 77% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LimitGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java index fe94c8feda14a..bd2f49e4b079b 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LimitGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java @@ -5,10 +5,11 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -23,8 +24,9 @@ public class LimitGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { int limit = randomIntBetween(0, 15000); String cmd = " | limit " + limit; @@ -36,9 +38,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { int limit = (int) commandDescription.context().get(LIMIT); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LookupJoinGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java similarity index 80% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LookupJoinGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java index b1348491e73c3..011db8e2f0e9f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/LookupJoinGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java @@ -5,11 +5,14 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.GenerativeRestTest; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.LookupIdx; +import org.elasticsearch.xpack.esql.generator.LookupIdxColumn; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.ArrayList; import java.util.HashSet; @@ -19,7 +22,6 @@ import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomInt; -import static org.elasticsearch.test.ESTestCase.randomSubsetOf; public class LookupJoinGenerator implements CommandGenerator { @@ -29,17 +31,18 @@ public class LookupJoinGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { - GenerativeRestTest.LookupIdx lookupIdx = randomFrom(schema.lookupIndices()); + LookupIdx lookupIdx = randomFrom(schema.lookupIndices()); String lookupIdxName = lookupIdx.idxName(); int joinColumnsCount = randomInt(lookupIdx.keys().size() - 1) + 1; // at least one column must be used for the join - List joinColumns = randomSubsetOf(joinColumnsCount, lookupIdx.keys()); + List joinColumns = ESTestCase.randomSubsetOf(joinColumnsCount, lookupIdx.keys()); List keyNames = new ArrayList<>(); List joinOn = new ArrayList<>(); Set usedColumns = new HashSet<>(); - for (GenerativeRestTest.LookupIdxColumn joinColumn : joinColumns) { + for (LookupIdxColumn joinColumn : joinColumns) { String idxKey = joinColumn.name(); String keyType = joinColumn.type(); @@ -47,7 +50,7 @@ public CommandDescription generate( if (candidateKeys.isEmpty()) { continue; // no candidate keys of the right type, skip this column } - EsqlQueryGenerator.Column key = randomFrom(candidateKeys); + Column key = randomFrom(candidateKeys); if (usedColumns.contains(key.name()) || usedColumns.contains(idxKey)) { continue; // already used this column from the lookup index, or will discard the main index column by RENAME'ing below, skip } else { @@ -82,9 +85,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/MvExpandGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java similarity index 73% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/MvExpandGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java index 317a2e459094e..7be8d0bdff62a 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/MvExpandGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -22,8 +24,9 @@ public class MvExpandGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String toExpand = EsqlQueryGenerator.randomName(previousOutput); if (toExpand == null) { @@ -37,9 +40,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/RenameGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java similarity index 82% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/RenameGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java index 80b36c06e524e..132c0dc9840a7 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/RenameGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.ArrayList; import java.util.HashMap; @@ -29,21 +31,19 @@ public class RenameGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { int n = randomIntBetween(1, Math.min(3, previousOutput.size())); List proj = new ArrayList<>(); Map nameToType = new HashMap<>(); - for (EsqlQueryGenerator.Column column : previousOutput) { + for (Column column : previousOutput) { nameToType.put(column.name(), column.type()); } List names = new ArrayList<>( - previousOutput.stream() - .filter(EsqlQueryGenerator::fieldCanBeUsed) - .map(EsqlQueryGenerator.Column::name) - .collect(Collectors.toList()) + previousOutput.stream().filter(EsqlQueryGenerator::fieldCanBeUsed).map(Column::name).collect(Collectors.toList()) ); if (names.isEmpty()) { return EMPTY_DESCRIPTION; @@ -86,9 +86,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { if (commandDescription == EMPTY_DESCRIPTION) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/SortGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java similarity index 78% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/SortGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java index f7849d1c202f1..4d61c00d1391c 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/SortGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.HashSet; import java.util.List; @@ -27,8 +29,9 @@ public class SortGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { int n = randomIntBetween(1, previousOutput.size()); Set proj = new HashSet<>(); @@ -50,9 +53,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return CommandGenerator.expectSameColumns(previousColumns, columns); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/StatsGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java similarity index 81% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/StatsGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java index b0ce7f43af997..30464c28a8464 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/StatsGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -25,10 +27,11 @@ public class StatsGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { - List nonNull = previousOutput.stream() + List nonNull = previousOutput.stream() .filter(EsqlQueryGenerator::fieldCanBeUsed) .filter(x -> x.type().equals("null") == false) .collect(Collectors.toList()); @@ -69,9 +72,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { // TODO validate columns diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/WhereGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java similarity index 79% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/WhereGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java index 28d9f563896d9..00a7f6cfe76d2 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/WhereGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java @@ -5,10 +5,12 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe; +package org.elasticsearch.xpack.esql.generator.command.pipe; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; @@ -21,7 +23,7 @@ public class WhereGenerator implements CommandGenerator { public static final String WHERE = "where"; public static final CommandGenerator INSTANCE = new WhereGenerator(); - public static String randomExpression(final int nConditions, List previousOutput) { + public static String randomExpression(final int nConditions, List previousOutput) { // TODO more complex conditions var result = new StringBuilder(); @@ -46,8 +48,9 @@ public static String randomExpression(final int nConditions, List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { String expression = randomExpression(randomIntBetween(1, 5), previousOutput); if (expression == null) { @@ -60,9 +63,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return CommandGenerator.expectSameColumns(previousColumns, columns); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/source/FromGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java similarity index 68% rename from x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/source/FromGenerator.java rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java index 05cd307a50755..13bfe92692f1b 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/source/FromGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java @@ -5,16 +5,17 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.qa.rest.generative.command.source; +package org.elasticsearch.xpack.esql.generator.command.source; -import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator; -import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; import java.util.List; import java.util.Map; import static org.elasticsearch.test.ESTestCase.randomIntBetween; -import static org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator.indexPattern; public class FromGenerator implements CommandGenerator { @@ -23,14 +24,15 @@ public class FromGenerator implements CommandGenerator { @Override public CommandDescription generate( List previousCommands, - List previousOutput, - QuerySchema schema + List previousOutput, + QuerySchema schema, + QueryExecutor executor ) { StringBuilder result = new StringBuilder("from "); int items = randomIntBetween(1, 3); List availableIndices = schema.baseIndices(); for (int i = 0; i < items; i++) { - String pattern = indexPattern(availableIndices.get(randomIntBetween(0, availableIndices.size() - 1))); + String pattern = EsqlQueryGenerator.indexPattern(availableIndices.get(randomIntBetween(0, availableIndices.size() - 1))); if (i > 0) { result.append(","); } @@ -44,9 +46,9 @@ public CommandDescription generate( public ValidationResult validateOutput( List previousCommands, CommandDescription commandDescription, - List previousColumns, + List previousColumns, List> previousOutput, - List columns, + List columns, List> output ) { return VALIDATION_OK; From 115d4fdc1519ac174d51ba969b95e9f20a70dfb5 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Sun, 14 Sep 2025 13:15:46 +0200 Subject: [PATCH 2/5] ES|QL: add GenerativeCsvTests --- .../rest/generative/GenerativeRestTest.java | 12 +- .../esql/generator/EsqlQueryGenerator.java | 33 +- .../xpack/esql/generator/README.asciidoc | 4 + .../generator/command/CommandGenerator.java | 20 +- .../command/pipe/ChangePointGenerator.java | 3 +- .../command/pipe/DissectGenerator.java | 5 +- .../generator/command/pipe/DropGenerator.java | 6 +- .../command/pipe/EnrichGenerator.java | 5 +- .../generator/command/pipe/EvalGenerator.java | 5 +- .../generator/command/pipe/ForkGenerator.java | 10 +- .../generator/command/pipe/GrokGenerator.java | 5 +- .../generator/command/pipe/KeepGenerator.java | 5 +- .../command/pipe/LimitGenerator.java | 3 +- .../command/pipe/LookupJoinGenerator.java | 3 +- .../command/pipe/MvExpandGenerator.java | 3 +- .../command/pipe/RenameGenerator.java | 5 +- .../generator/command/pipe/SortGenerator.java | 3 +- .../command/pipe/StatsGenerator.java | 3 +- .../command/pipe/WhereGenerator.java | 3 +- .../command/source/FromGenerator.java | 3 +- .../command/source/SimpleFromGenerator.java | 49 ++ .../xpack/esql/GenerativeCsvTests.java | 782 ++++++++++++++++++ 22 files changed, 929 insertions(+), 41 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/SimpleFromGenerator.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 759952e5de556..47bad6ac94123 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -147,7 +147,14 @@ public List currentSchema() { final List previousCommands = new ArrayList<>(); QueryExecuted previousResult; }; - EsqlQueryGenerator.generatePipeline(MAX_DEPTH, EsqlQueryGenerator.sourceCommand(), mappingInfo, exec, this); + EsqlQueryGenerator.generatePipeline( + MAX_DEPTH, + EsqlQueryGenerator.sourceCommand(), + EsqlQueryGenerator.PIPE_COMMANDS, + mappingInfo, + exec, + this + ); } } @@ -164,7 +171,8 @@ private static CommandGenerator.ValidationResult checkResults( previousResult == null ? null : previousResult.outputSchema(), previousResult == null ? null : previousResult.result(), result.outputSchema(), - result.result() + result.result(), + false ); if (outputValidation.success() == false) { for (Pattern allowedError : ALLOWED_ERROR_PATTERNS) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java index b868c232b65dc..72b171ac822b3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.generator.command.pipe.StatsGenerator; import org.elasticsearch.xpack.esql.generator.command.pipe.WhereGenerator; import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; +import org.elasticsearch.xpack.esql.generator.command.source.SimpleFromGenerator; import java.util.List; import java.util.Set; @@ -45,12 +46,14 @@ public class EsqlQueryGenerator { /** * These are commands that are at the beginning of the query, eg. FROM */ - static List SOURCE_COMMANDS = List.of(FromGenerator.INSTANCE); + public static List SOURCE_COMMANDS = List.of(FromGenerator.INSTANCE); + + public static List SIMPLIFIED_SOURCE_COMMANDS = List.of(SimpleFromGenerator.INSTANCE); /** * These are downstream commands, ie. that cannot appear as the first command in a query */ - static List PIPE_COMMANDS = List.of( + public static List PIPE_COMMANDS = List.of( ChangePointGenerator.INSTANCE, DissectGenerator.INSTANCE, DropGenerator.INSTANCE, @@ -68,12 +71,31 @@ public class EsqlQueryGenerator { WhereGenerator.INSTANCE ); + /** + * Same as PIPE_COMMANDS but without the more complex commands (Fork, Enrich, Join). + * This is needed in CSV tests, that don't support the full ES capabilities + */ + public static List SIMPLIFIED_PIPE_COMMANDS = List.of( + ChangePointGenerator.INSTANCE, + DissectGenerator.INSTANCE, + DropGenerator.INSTANCE, + EvalGenerator.INSTANCE, + GrokGenerator.INSTANCE, + KeepGenerator.INSTANCE, + LimitGenerator.INSTANCE, + MvExpandGenerator.INSTANCE, + RenameGenerator.INSTANCE, + SortGenerator.INSTANCE, + StatsGenerator.INSTANCE, + WhereGenerator.INSTANCE + ); + public static CommandGenerator sourceCommand() { return randomFrom(SOURCE_COMMANDS); } - public static CommandGenerator randomPipeCommandGenerator() { - return randomFrom(PIPE_COMMANDS); + public static CommandGenerator simplifiedSourceCommand() { + return randomFrom(SIMPLIFIED_SOURCE_COMMANDS); } public interface Executor { @@ -90,6 +112,7 @@ public interface Executor { public static void generatePipeline( final int depth, CommandGenerator commandGenerator, + List pipelineGenerators, final CommandGenerator.QuerySchema schema, Executor executor, QueryExecutor queryExecutor @@ -104,7 +127,7 @@ public static void generatePipeline( if (executor.currentSchema().isEmpty()) { break; } - commandGenerator = EsqlQueryGenerator.randomPipeCommandGenerator(); + commandGenerator = randomFrom(pipelineGenerators); desc = commandGenerator.generate(executor.previousCommands(), executor.currentSchema(), schema, queryExecutor); if (desc == CommandGenerator.EMPTY_DESCRIPTION) { continue; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc index d9613f01e0b6b..6ba8e4fe0188f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/README.asciidoc @@ -17,6 +17,10 @@ All you have to do is: *** Have a look at `EsqlQueryGenerator`, it contains many utility methods that will help you generate random expressions. ** Implement `CommandGenerator.validateOutput()` to validate the output of the query. * Add your class to `EsqlQueryGenerator.SOURCE_COMMANDS` (if it's a source command) or `EsqlQueryGenerator.PIPE_COMMANDS` (if it's a pipe command). + These will be used by `GenerativeIT` to pick a random command to append to the query. +** Also consider adding your generators (or a simplified version of them) + to `EsqlQueryGenerator.SIMPLIFIED_SOURCE_COMMANDS` or `EsqlQueryGenerator.SIMPLIFIED_PIPE_COMMANDS`. + These are used to generate queries in contexts when the full complexity of ES|QL is not supported (eg. in `GenerativeCsvIT`). * Run `GenerativeIT` at least a couple of times: these tests can be pretty noisy. * If you get unexpected errors (real bugs in ES|QL), please open an issue and add the error to `GenerativeRestTest.ALLOWED_ERRORS`. Run tests again until everything works fine. diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java index e18748be154f8..2901f45e34f8e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java @@ -54,7 +54,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return VALIDATION_OK; } @@ -92,6 +93,9 @@ CommandDescription generate( * @param previousOutput The output of the original query (without last generated command), as a list (rows) of lists (columns) of values * @param columns The output schema of the full query (WITH last generated command). * @param output The output of the full query (WITH last generated command), as a list (rows) of lists (columns) of values + * @param deterministic True if the query is executed in deterministic mode (eg. in CsvTests), ie. that the + * results (also their order) are stable between multiple executions. + * False if the query is executed in non-deterministic mode (eg. in GenerativeIT, against an ES cluster) * @return The result of the output validation. If the validation succeeds, you should return {@link CommandGenerator#VALIDATION_OK}. * Also, if for some reason you can't validate the output, just return {@link CommandGenerator#VALIDATION_OK}; for a command, having a generator without * validation is much better than having no generator at all. @@ -102,19 +106,19 @@ ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ); static ValidationResult expectSameRowCount( List previousCommands, List> previousOutput, - List> output + List> output, + boolean deterministic ) { - - // ES|QL is quite non-deterministic in this sense, we can't guarantee it for now - // if (output.size() != previousOutput.size()) { - // return new ValidationResult(false, "Expecting [" + previousOutput.size() + "] rows, but got [" + output.size() + "]"); - // } + if (deterministic && previousOutput.size() != output.size()) { + return new ValidationResult(false, "Expecting [" + previousOutput.size() + "] rows, got [" + output.size() + "]"); + } return VALIDATION_OK; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java index 2b04cc9aced80..f905ece9c6e50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ChangePointGenerator.java @@ -49,7 +49,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return CommandGenerator.expectAtLeastSameNumberOfColumns(previousColumns, columns); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java index 6e0f95c7e7846..db065e6922768 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DissectGenerator.java @@ -66,7 +66,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -76,6 +77,6 @@ public ValidationResult validateOutput( return new ValidationResult(false, "Expecting at least [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java index 97b754692f392..0f8023c2c4d12 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java @@ -73,7 +73,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -88,7 +89,8 @@ public ValidationResult validateOutput( } // TODO awaits fix https://github.com/elastic/elasticsearch/issues/120272 // return CommandGenerator.expectSameRowCount(previousOutput, output); - return VALIDATION_OK; + + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java index be3d547cf535c..d9cd8cbb99292 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java @@ -49,7 +49,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -59,6 +60,6 @@ public ValidationResult validateOutput( return new ValidationResult(false, "Expecting at least [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java index 8c2841dda3f04..239b470d8697b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java @@ -69,7 +69,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { List expectedColumns = (List) commandDescription.context().get(NEW_COLUMNS); List resultColNames = columns.stream().map(Column::name).toList(); @@ -87,7 +88,7 @@ public ValidationResult validateOutput( ); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } private static String unquote(String colName) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java index 3dfca414a997a..9f1fd8f309c45 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/ForkGenerator.java @@ -112,13 +112,14 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return VALIDATION_OK; } }; - EsqlQueryGenerator.generatePipeline(3, gen, schema, exec, executor); + EsqlQueryGenerator.generatePipeline(3, gen, EsqlQueryGenerator.PIPE_COMMANDS, schema, exec, executor); if (exec.previousCommands().size() > 1) { String previousCmd = exec.previousCommands() .stream() @@ -143,8 +144,9 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java index 162ccbfc21e15..ff81a1cc1a7e5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/GrokGenerator.java @@ -65,7 +65,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -73,6 +74,6 @@ public ValidationResult validateOutput( if (previousColumns.size() > columns.size()) { return new ValidationResult(false, "Expecting at least [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java index e4cceabb21c9f..58be5c939a0c9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java @@ -69,7 +69,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -79,7 +80,7 @@ public ValidationResult validateOutput( return new ValidationResult(false, "Expecting at most [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return VALIDATION_OK; + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java index bd2f49e4b079b..6ef7151cdfcc3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java @@ -41,7 +41,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { int limit = (int) commandDescription.context().get(LIMIT); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java index 011db8e2f0e9f..b4027e67b4812 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java @@ -88,7 +88,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java index 7be8d0bdff62a..aac31d955d2b4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/MvExpandGenerator.java @@ -43,7 +43,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java index 132c0dc9840a7..6ee53ea50f60b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/RenameGenerator.java @@ -89,7 +89,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { if (commandDescription == EMPTY_DESCRIPTION) { return VALIDATION_OK; @@ -97,7 +98,7 @@ public ValidationResult validateOutput( if (previousColumns.size() < columns.size()) { return new ValidationResult(false, "Expecting at most [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output); + return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java index 4d61c00d1391c..45815d13a0221 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/SortGenerator.java @@ -56,7 +56,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return CommandGenerator.expectSameColumns(previousColumns, columns); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java index 30464c28a8464..1bb1420793337 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java @@ -75,7 +75,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { // TODO validate columns return VALIDATION_OK; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java index 00a7f6cfe76d2..60e554ba69ef9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/WhereGenerator.java @@ -66,7 +66,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return CommandGenerator.expectSameColumns(previousColumns, columns); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java index 13bfe92692f1b..49a15d1858a0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java @@ -49,7 +49,8 @@ public ValidationResult validateOutput( List previousColumns, List> previousOutput, List columns, - List> output + List> output, + boolean deterministic ) { return VALIDATION_OK; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/SimpleFromGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/SimpleFromGenerator.java new file mode 100644 index 0000000000000..b86047204e396 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/SimpleFromGenerator.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.generator.command.source; + +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; + +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.ESTestCase.randomIntBetween; + +public class SimpleFromGenerator implements CommandGenerator { + + public static final SimpleFromGenerator INSTANCE = new SimpleFromGenerator(); + + @Override + public CommandDescription generate( + List previousCommands, + List previousOutput, + QuerySchema schema, + QueryExecutor executor + ) { + List availableIndices = schema.baseIndices(); + String idx = availableIndices.get(randomIntBetween(0, availableIndices.size() - 1)); + + String query = "from " + idx; + return new CommandDescription("from", this, query, Map.of()); + } + + @Override + public ValidationResult validateOutput( + List previousCommands, + CommandDescription commandDescription, + List previousColumns, + List> previousOutput, + List columns, + List> output, + boolean deterministic + ) { + return VALIDATION_OK; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java new file mode 100644 index 0000000000000..9d55692f2d554 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java @@ -0,0 +1,782 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.logging.HeaderWarning; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.ConstantNullBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverCompletionInfo; +import org.elasticsearch.compute.operator.DriverRunner; +import org.elasticsearch.compute.operator.exchange.ExchangeSinkHandler; +import org.elasticsearch.compute.operator.exchange.ExchangeSourceHandler; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.enrich.EnrichPolicy; +import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; +import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; +import org.elasticsearch.xpack.esql.analysis.EnrichResolution; +import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; +import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; +import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.generator.Column; +import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; +import org.elasticsearch.xpack.esql.generator.LookupIdx; +import org.elasticsearch.xpack.esql.generator.LookupIdxColumn; +import org.elasticsearch.xpack.esql.generator.QueryExecuted; +import org.elasticsearch.xpack.esql.generator.QueryExecutor; +import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.inference.InferenceService; +import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanPreOptimizer; +import org.elasticsearch.xpack.esql.optimizer.LogicalPreOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.TestLocalPhysicalPlanOptimizer; +import org.elasticsearch.xpack.esql.parser.EsqlParser; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.physical.ChangePointExec; +import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec; +import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; +import org.elasticsearch.xpack.esql.plan.physical.MergeExec; +import org.elasticsearch.xpack.esql.plan.physical.OutputExec; +import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; +import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner; +import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlan; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders; +import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; +import org.elasticsearch.xpack.esql.plugin.QueryPragmas; +import org.elasticsearch.xpack.esql.session.Configuration; +import org.elasticsearch.xpack.esql.session.EsqlSession; +import org.elasticsearch.xpack.esql.session.EsqlSession.PlanRunner; +import org.elasticsearch.xpack.esql.session.Result; +import org.elasticsearch.xpack.esql.stats.DisabledSearchStats; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.esql.CsvTestUtils.ActualResults; +import static org.elasticsearch.xpack.esql.CsvTestUtils.Type; +import static org.elasticsearch.xpack.esql.CsvTestUtils.loadPageFromCsv; +import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP; +import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_POLICIES; +import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.availableDatasetsForEs; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyInferenceResolution; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; + +//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") +public class GenerativeCsvTests extends ESTestCase implements QueryExecutor { + + private static final Logger LOGGER = LogManager.getLogger(GenerativeCsvTests.class); + + public static final int ITERATIONS = 100; + public static final int MAX_DEPTH = 20; + + public static final Set ALLOWED_ERRORS = Set.of( + "Reference \\[.*\\] is ambiguous", + "Cannot use field \\[.*\\] due to ambiguities", + "cannot sort on .*", + "argument of \\[count.*\\] must", + "Cannot use field \\[.*\\] with unsupported type \\[.*\\]", + "Unbounded SORT not supported yet", + "The field names are too complex to process", // field_caps problem + "must be \\[any type except counter types\\]", // TODO refine the generation of count() + + // Awaiting fixes for query failure + "Unknown column \\[\\]", // https://github.com/elastic/elasticsearch/issues/121741, + // https://github.com/elastic/elasticsearch/issues/125866 + "Plan \\[ProjectExec\\[\\[.* optimized incorrectly due to missing references", + "The incoming YAML document exceeds the limit:", // still to investigate, but it seems to be specific to the test framework + "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072 + "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/131509 + + // Awaiting fixes for correctness + "Expecting at most \\[.*\\] columns, got \\[.*\\]" // https://github.com/elastic/elasticsearch/issues/129561 + ); + + public static final Set ALLOWED_ERROR_PATTERNS = ALLOWED_ERRORS.stream() + .map(x -> ".*" + x + ".*") + .map(x -> Pattern.compile(x, Pattern.DOTALL)) + .collect(Collectors.toSet()); + + private final Configuration configuration = EsqlTestUtils.configuration( + new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()) + ); + private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); + private final EsqlParser parser = new EsqlParser(); + private final Mapper mapper = new Mapper(); + private ThreadPool threadPool; + private Executor executor; + + @Before + public void setUp() throws Exception { + super.setUp(); + if (randomBoolean()) { + int numThreads = randomBoolean() ? 1 : between(2, 16); + threadPool = new TestThreadPool( + "CsvTests", + new FixedExecutorBuilder(Settings.EMPTY, "esql_test", numThreads, 1024, "esql", EsExecutors.TaskTrackingConfig.DEFAULT) + ); + executor = threadPool.executor("esql_test"); + } else { + threadPool = new TestThreadPool(getTestName()); + executor = threadPool.executor(ThreadPool.Names.SEARCH); + } + HeaderWarning.setThreadContext(threadPool.getThreadContext()); + } + + @After + public void teardown() { + HeaderWarning.removeThreadContext(threadPool.getThreadContext()); + } + + @After + public void tearDown() throws Exception { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + super.tearDown(); + } + + private int randomPageSize() { + if (randomBoolean()) { + return between(1, 16); + } else { + return between(1, 16 * 1024); + } + } + + public GenerativeCsvTests() {} + + @Override + protected final boolean enableWarningsCheck() { + return false; // We use our own warnings check + } + + public Set excludedDatasets() { + // these have subfields, ranges and other things that are not properly supported in CSV + return Set.of( + "system_metrics", + "mv_text", + "multi_column_joinable", + "heights", + "firewall_logs", + "app_logs", + "employees_incompatible", + "employees_incompatible_types", + "decades", + "k8s", + "k8s-downsampled", + "ages", + "books" + ); + } + + public void test() throws Exception { + List indices = availableIndices().stream().filter(x -> excludedDatasets().contains(x) == false).toList(); + List lookupIndices = lookupIndices(); + List policies = availableEnrichPolicies(); + CommandGenerator.QuerySchema mappingInfo = new CommandGenerator.QuerySchema(indices, lookupIndices, policies); + + for (int i = 0; i < ITERATIONS; i++) { + var exec = new EsqlQueryGenerator.Executor() { + @Override + public void run(CommandGenerator generator, CommandGenerator.CommandDescription current) { + previousCommands.add(current); + final String command = current.commandString(); + + final QueryExecuted result = previousResult == null + ? execute(command, 0) + : execute(previousResult.query() + command, previousResult.depth()); + previousResult = result; + + logger.trace(() -> "query:" + result.query()); + + final boolean hasException = result.exception() != null; + if (hasException || checkResults(List.of(), generator, current, previousResult, result).success() == false) { + if (hasException) { + checkException(result); + } + continueExecuting = false; + currentSchema = List.of(); + } else { + continueExecuting = true; + currentSchema = result.outputSchema(); + } + } + + @Override + public List previousCommands() { + return previousCommands; + } + + @Override + public boolean continueExecuting() { + return continueExecuting; + } + + @Override + public List currentSchema() { + return currentSchema; + } + + boolean continueExecuting; + List currentSchema; + final List previousCommands = new ArrayList<>(); + QueryExecuted previousResult; + }; + EsqlQueryGenerator.generatePipeline( + MAX_DEPTH, + EsqlQueryGenerator.simplifiedSourceCommand(), + EsqlQueryGenerator.SIMPLIFIED_PIPE_COMMANDS, + mappingInfo, + exec, + this + ); + } + } + + private static CommandGenerator.ValidationResult checkResults( + List previousCommands, + CommandGenerator commandGenerator, + CommandGenerator.CommandDescription commandDescription, + QueryExecuted previousResult, + QueryExecuted result + ) { + CommandGenerator.ValidationResult outputValidation = commandGenerator.validateOutput( + previousCommands, + commandDescription, + previousResult == null ? null : previousResult.outputSchema(), + previousResult == null ? null : previousResult.result(), + result.outputSchema(), + result.result(), + true + ); + if (outputValidation.success() == false) { + for (Pattern allowedError : ALLOWED_ERROR_PATTERNS) { + if (isAllowedError(outputValidation.errorMessage(), allowedError)) { + return outputValidation; + } + } + fail("query: " + result.query() + "\nerror: " + outputValidation.errorMessage()); + } + return outputValidation; + } + + private void checkException(QueryExecuted query) { + for (Pattern allowedError : ALLOWED_ERROR_PATTERNS) { + if (isAllowedError(query.exception().getMessage(), allowedError)) { + return; + } + } + fail("query: " + query.query() + "\nexception: " + query.exception().getMessage()); + } + + /** + * Long lines in exceptions can be split across several lines. When a newline is inserted, the end of the current line and the beginning + * of the new line are marked with a backslash {@code \}; the new line will also have whitespace before the backslash for aligning. + */ + private static final Pattern ERROR_MESSAGE_LINE_BREAK = Pattern.compile("\\\\\n\\s*\\\\"); + + private static boolean isAllowedError(String errorMessage, Pattern allowedPattern) { + String errorWithoutLineBreaks = ERROR_MESSAGE_LINE_BREAK.matcher(errorMessage).replaceAll(""); + return allowedPattern.matcher(errorWithoutLineBreaks).matches(); + } + + private List availableIndices() throws IOException { + return availableDatasetsForEs(false, false, false).stream() + .filter(x -> x.requiresInferenceEndpoint() == false) + .map(x -> x.indexName()) + .toList(); + } + + private List lookupIndices() { + List result = new ArrayList<>(); + // we don't have key info from the dataset loader, let's hardcode it for now + result.add(new LookupIdx("languages_lookup", List.of(new LookupIdxColumn("language_code", "integer")))); + result.add(new LookupIdx("message_types_lookup", List.of(new LookupIdxColumn("message", "keyword")))); + List multiColumnJoinableLookupKeys = List.of( + new LookupIdxColumn("id_int", "integer"), + new LookupIdxColumn("name_str", "keyword"), + new LookupIdxColumn("is_active_bool", "boolean"), + new LookupIdxColumn("ip_addr", "ip"), + new LookupIdxColumn("other1", "keyword"), + new LookupIdxColumn("other2", "integer") + ); + result.add(new LookupIdx("multi_column_joinable_lookup", multiColumnJoinableLookupKeys)); + return result; + } + + List availableEnrichPolicies() { + return ENRICH_POLICIES; + } + + private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset datasets) { + var indexNames = datasets.datasets().stream().map(CsvTestsDataLoader.TestDataset::indexName); + Map indexModes = indexNames.collect(Collectors.toMap(x -> x, x -> IndexMode.STANDARD)); + List mappings = datasets.datasets() + .stream() + .map(ds -> new MappingPerIndex(ds.indexName(), createMappingForIndex(ds))) + .toList(); + var mergedMappings = mergeMappings(mappings); + return IndexResolution.valid( + new EsIndex(datasets.indexPattern(), mergedMappings.mapping, indexModes, mergedMappings.partiallyUnmappedFields) + ); + } + + private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { + var mapping = new TreeMap<>(loadMapping(dataset.mappingFileName())); + if (dataset.typeMapping() == null) { + return mapping; + } + for (var entry : dataset.typeMapping().entrySet()) { + if (mapping.containsKey(entry.getKey())) { + DataType dataType = DataType.fromTypeName(entry.getValue()); + EsField field = mapping.get(entry.getKey()); + EsField editedField = new EsField( + field.getName(), + dataType, + field.getProperties(), + field.isAggregatable(), + field.getTimeSeriesFieldType() + ); + mapping.put(entry.getKey(), editedField); + } + } + return mapping; + } + + record MappingPerIndex(String index, Map mapping) {} + + record MergedResult(Map mapping, Set partiallyUnmappedFields) {} + + private static MergedResult mergeMappings(List mappingsPerIndex) { + int numberOfIndices = mappingsPerIndex.size(); + Map> columnNamesToFieldByIndices = new HashMap<>(); + for (var mappingPerIndex : mappingsPerIndex) { + for (var entry : mappingPerIndex.mapping().entrySet()) { + String columnName = entry.getKey(); + EsField field = entry.getValue(); + columnNamesToFieldByIndices.computeIfAbsent(columnName, k -> new HashMap<>()).put(mappingPerIndex.index(), field); + } + } + + var partiallyUnmappedFields = columnNamesToFieldByIndices.entrySet() + .stream() + .filter(e -> e.getValue().size() < numberOfIndices) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + var mappings = columnNamesToFieldByIndices.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue()))); + return new MergedResult(mappings, partiallyUnmappedFields); + } + + private static EsField mergeFields(String index, Map columnNameToField) { + var indexFields = columnNameToField.values(); + if (indexFields.stream().distinct().count() > 1) { + var typesToIndices = new HashMap>(); + for (var typeToIndex : columnNameToField.entrySet()) { + typesToIndices.computeIfAbsent(typeToIndex.getValue().getDataType().typeName(), k -> new HashSet<>()) + .add(typeToIndex.getKey()); + } + return new InvalidMappedField(index, typesToIndices); + } else { + return indexFields.iterator().next(); + } + } + + private static EnrichResolution loadEnrichPolicies() { + EnrichResolution enrichResolution = new EnrichResolution(); + for (CsvTestsDataLoader.EnrichConfig policyConfig : CsvTestsDataLoader.ENRICH_POLICIES) { + EnrichPolicy policy = loadEnrichPolicyMapping(policyConfig.policyFileName()); + CsvTestsDataLoader.TestDataset sourceIndex = CSV_DATASET_MAP.get(policy.getIndices().get(0)); + // this could practically work, but it's wrong: + // EnrichPolicyResolution should contain the policy (system) index, not the source index + EsIndex esIndex = loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset.of(sourceIndex.withTypeMapping(Map.of()))).get(); + var concreteIndices = Map.of(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0)); + enrichResolution.addResolvedPolicy( + policyConfig.policyName(), + Enrich.Mode.ANY, + new ResolvedEnrichPolicy( + policy.getMatchField(), + policy.getType(), + policy.getEnrichFields(), + concreteIndices, + esIndex.mapping() + ) + ); + } + return enrichResolution; + } + + private static EnrichPolicy loadEnrichPolicyMapping(String policyFileName) { + URL policyMapping = CsvTestsDataLoader.class.getResource("/" + policyFileName); + assertThat(policyMapping, is(notNullValue())); + try { + String fileContent = CsvTestsDataLoader.readTextFile(policyMapping); + return EnrichPolicy.fromXContent(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, fileContent)); + } catch (IOException e) { + throw new IllegalArgumentException("Cannot read resource " + policyFileName); + } + } + + private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.MultiIndexTestDataset datasets) { + var indexResolution = loadIndexResolution(datasets); + var enrichPolicies = loadEnrichPolicies(); + var analyzer = new Analyzer( + new AnalyzerContext(configuration, functionRegistry, indexResolution, enrichPolicies, emptyInferenceResolution()), + TEST_VERIFIER + ); + LogicalPlan plan = analyzer.analyze(parsed); + plan.setAnalyzed(); + LOGGER.debug("Analyzed plan:\n{}", plan); + return plan; + } + + private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) { + var preAnalysis = new PreAnalyzer().preAnalyze(parsed); + if (preAnalysis.index() == null) { + // If the data set doesn't matter we'll just grab one we know works. Employees is fine. + return CsvTestsDataLoader.MultiIndexTestDataset.of(CSV_DATASET_MAP.get("employees")); + } + + String indexName = preAnalysis.index().indexPattern(); + List datasets = new ArrayList<>(); + if (indexName.endsWith("*")) { + String indexPrefix = indexName.substring(0, indexName.length() - 1); + for (var entry : CSV_DATASET_MAP.entrySet()) { + if (entry.getKey().startsWith(indexPrefix)) { + datasets.add(entry.getValue()); + } + } + } else { + for (String index : indexName.split(",")) { + var dataset = CSV_DATASET_MAP.get(index); + if (dataset == null) { + throw new IllegalArgumentException("unknown CSV dataset for table [" + index + "]"); + } + datasets.add(dataset); + } + } + if (datasets.isEmpty()) { + throw new IllegalArgumentException("unknown CSV dataset for table [" + indexName + "]"); + } + return new CsvTestsDataLoader.MultiIndexTestDataset(indexName, datasets); + } + + private static TestPhysicalOperationProviders testOperationProviders( + FoldContext foldCtx, + CsvTestsDataLoader.MultiIndexTestDataset datasets + ) throws Exception { + var indexPages = new ArrayList(); + for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { + var testData = loadPageFromCsv(GenerativeCsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); + Set mappedFields = loadMapping(dataset.mappingFileName()).keySet(); + indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2(), mappedFields)); + } + return TestPhysicalOperationProviders.create(foldCtx, indexPages); + } + + @Override + public QueryExecuted execute(String command, int depth) { + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); + try { + ActualResults result = executePlan(command, bigArrays); + + return new QueryExecuted(command, depth, toColumns(result.columnNames(), result.columnTypes()), data(result), null); + } catch (Exception e) { + return new QueryExecuted(command, depth, null, null, e); + } + } + + private List toColumns(List names, List types) { + List columns = new ArrayList<>(names.size()); + for (int i = 0; i < names.size(); i++) { + columns.add(new Column(names.get(i), types.get(i).name(), List.of())); + } + return columns; + } + + private List> data(ActualResults queryResult) { + List> result = new ArrayList<>(); + if (queryResult.pages().isEmpty()) { + return result; + } + int cols = queryResult.columnNames().size(); + for (Page page : queryResult.pages()) { + for (int p = 0; p < page.getPositionCount(); p++) { + List row = new ArrayList<>(cols); + for (int c = 0; c < cols; c++) { + row.add(value(page.getBlock(c), p)); + } + result.add(row); + } + } + return result; + } + + private Object value(Block block, int position) { + BytesRef spare = new BytesRef(); + int valueCount = block.getValueCount(position); + int idx = block.getFirstValueIndex(position); + if (valueCount == 0) { + return null; + } else if (valueCount == 1) { + return extract(block, idx, spare); + } else { + List values = new ArrayList<>(valueCount); + for (int i = 0; i < valueCount; i++) { + values.add(extract(block, idx + i, spare)); + } + return values; + } + } + + private static Object extract(Block block, int idx, BytesRef spare) { + return switch (block) { + case ConstantNullBlock n -> null; + case BooleanBlock b -> b.getBoolean(idx); + case BytesRefBlock b -> b.getBytesRef(idx, spare); + case IntBlock i -> i.getInt(idx); + case LongBlock l -> l.getLong(idx); + case FloatBlock f -> f.getFloat(idx); + case DoubleBlock d -> d.getDouble(idx); + default -> throw new IllegalArgumentException("Unsupported block type: " + block.getClass()); + }; + } + + private ActualResults executePlan(String query, BigArrays bigArrays) throws Exception { + LogicalPlan parsed = parser.createStatement(query, EsqlTestUtils.TEST_CFG); + var testDatasets = testDatasets(parsed); + LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); + + FoldContext foldCtx = FoldContext.small(); + EsqlSession session = new EsqlSession( + getTestName(), + configuration, + null, + null, + null, + new LogicalPlanPreOptimizer(new LogicalPreOptimizerContext(foldCtx)), + functionRegistry, + new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration, foldCtx)), + mapper, + TEST_VERIFIER, + new PlanTelemetry(functionRegistry), + null, + EsqlTestUtils.MOCK_TRANSPORT_ACTION_SERVICES + ); + TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets); + + PlainActionFuture listener = new PlainActionFuture<>(); + + session.preOptimizedPlan(analyzed, listener.delegateFailureAndWrap((l, preOptimized) -> { + session.executeOptimizedPlan( + new EsqlQueryRequest(), + new EsqlExecutionInfo(randomBoolean()), + planRunner(bigArrays, foldCtx, physicalOperationProviders), + session.optimizedPlan(preOptimized), + listener.delegateFailureAndWrap( + // Wrap so we can capture the warnings in the calling thread + (next, result) -> next.onResponse( + new ActualResults( + result.schema().stream().map(Attribute::name).toList(), + result.schema().stream().map(a -> Type.asType(a.dataType().nameUpper())).toList(), + result.schema().stream().map(Attribute::dataType).toList(), + result.pages(), + threadPool.getThreadContext().getResponseHeaders() + ) + ) + ) + ); + })); + + return listener.get(); + } + + private Settings randomNodeSettings() { + Settings.Builder builder = Settings.builder(); + if (randomBoolean()) { + builder.put(BlockFactory.LOCAL_BREAKER_OVER_RESERVED_SIZE_SETTING, ByteSizeValue.ofBytes(randomIntBetween(0, 4096))); + builder.put(BlockFactory.LOCAL_BREAKER_OVER_RESERVED_MAX_SIZE_SETTING, ByteSizeValue.ofBytes(randomIntBetween(0, 16 * 1024))); + } + return builder.build(); + } + + // Asserts that the serialization and deserialization of the plan creates an equivalent plan. + private void opportunisticallyAssertPlanSerialization(PhysicalPlan plan) { + if (plan.anyMatch( + p -> p instanceof LocalSourceExec || p instanceof HashJoinExec || p instanceof ChangePointExec || p instanceof MergeExec + )) { + return; + } + SerializationTestUtils.assertSerialization(plan, configuration); + } + + PlanRunner planRunner(BigArrays bigArrays, FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders) { + return (physicalPlan, listener) -> executeSubPlan(bigArrays, foldCtx, physicalOperationProviders, physicalPlan, listener); + } + + void executeSubPlan( + BigArrays bigArrays, + FoldContext foldCtx, + TestPhysicalOperationProviders physicalOperationProviders, + PhysicalPlan physicalPlan, + ActionListener listener + ) { + // Keep in sync with ComputeService#execute + opportunisticallyAssertPlanSerialization(physicalPlan); + Tuple coordinatorAndDataNodePlan = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode( + physicalPlan, + configuration + ); + PhysicalPlan coordinatorPlan = coordinatorAndDataNodePlan.v1(); + PhysicalPlan dataNodePlan = coordinatorAndDataNodePlan.v2(); + + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Coordinator plan\n" + coordinatorPlan); + LOGGER.trace("DataNode plan\n" + dataNodePlan); + } + + BlockFactory blockFactory = new BlockFactory( + bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST), + bigArrays, + ByteSizeValue.ofBytes(randomLongBetween(1, BlockFactory.DEFAULT_MAX_BLOCK_PRIMITIVE_ARRAY_SIZE.getBytes() * 2)) + ); + ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), executor); + ExchangeSinkHandler exchangeSink = new ExchangeSinkHandler(blockFactory, between(1, 64), threadPool::relativeTimeInMillis); + + LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner( + getTestName(), + "", + new CancellableTask(1, "transport", "esql", null, TaskId.EMPTY_TASK_ID, Map.of()), + bigArrays, + blockFactory, + randomNodeSettings(), + configuration, + exchangeSource::createExchangeSource, + () -> exchangeSink.createExchangeSink(() -> {}), + mock(EnrichLookupService.class), + mock(LookupFromIndexService.class), + mock(InferenceService.class), + physicalOperationProviders, + List.of() + ); + + List collectedPages = Collections.synchronizedList(new ArrayList<>()); + + // replace fragment inside the coordinator plan + List drivers = new ArrayList<>(); + LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan( + "final", + foldCtx, + new OutputExec(coordinatorPlan, collectedPages::add) + ); + drivers.addAll(coordinatorNodeExecutionPlan.createDrivers(getTestName())); + if (dataNodePlan != null) { + var searchStats = new DisabledSearchStats(); + var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, foldCtx, searchStats) + ); + + var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer); + exchangeSource.addRemoteSink( + exchangeSink::fetchPageAsync, + Randomness.get().nextBoolean(), + () -> {}, + randomIntBetween(1, 3), + ActionListener.noop().delegateResponse((l, e) -> { + throw new AssertionError("expected no failure", e); + }) + ); + LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan("data", foldCtx, csvDataNodePhysicalPlan); + + drivers.addAll(dataNodeExecutionPlan.createDrivers(getTestName())); + Randomness.shuffle(drivers); + } + // Execute the drivers + DriverRunner runner = new DriverRunner(threadPool.getThreadContext()) { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.getThreadContext(), executor, driver, between(1, 1000), driverListener); + } + }; + listener = ActionListener.releaseAfter(listener, () -> Releasables.close(drivers)); + runner.runToCompletion( + drivers, + listener.map(ignore -> new Result(physicalPlan.output(), collectedPages, DriverCompletionInfo.EMPTY, null)) + ); + } +} From e7011a6d4694e107450f7a8bf764e66e0083f957 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 23 Sep 2025 10:37:11 +0200 Subject: [PATCH 3/5] More checks --- .../generator/command/CommandGenerator.java | 19 +++++++++++++++++++ .../generator/command/pipe/DropGenerator.java | 14 +++++++++++--- .../generator/command/pipe/KeepGenerator.java | 12 +++++++++++- .../command/pipe/LimitGenerator.java | 12 +++++++++++- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java index 42263d00f64b5..4a67db5901c6a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/CommandGenerator.java @@ -126,6 +126,25 @@ static ValidationResult expectSameRowCount( return VALIDATION_OK; } + static ValidationResult expectSameData(List> before, int beforeCol, List> after, int afterCol) { + if (before.size() != after.size()) { + return new ValidationResult(false, "Expecting same number of rows, got [" + before.size() + "] and [" + after.size() + "]"); + } + + for (int i = 0; i < before.size(); i++) { + Object v1 = before.get(i).get(beforeCol); + Object v2 = after.get(i).get(afterCol); + if (v1 == null) { + if (v2 != null) { + return new ValidationResult(false, "Expecting null at row [" + i + "], got [" + v2 + "]"); + } + } else if (v1.equals(v2) == false) { + return new ValidationResult(false, "Expecting [" + v1 + "] at row [" + i + "], got [" + v2 + "]"); + } + } + return VALIDATION_OK; + } + static ValidationResult expectSameColumns(List previousColumns, List columns) { if (previousColumns.stream().anyMatch(x -> x.name().contains(""))) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java index 0f8023c2c4d12..a5e871e0865d9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/DropGenerator.java @@ -87,10 +87,18 @@ public ValidationResult validateOutput( return new ValidationResult(false, "Column [" + droppedColumn + "] was not dropped"); } } - // TODO awaits fix https://github.com/elastic/elasticsearch/issues/120272 - // return CommandGenerator.expectSameRowCount(previousOutput, output); - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); + if (deterministic) { + for (int columnIdx = 0; columnIdx < columns.size(); columnIdx++) { + Column c = columns.get(columnIdx); + int previousColumnIdx = previousColumns.indexOf(c); + if (previousColumnIdx == -1) { + return new ValidationResult(false, "Column [" + c + "] not in previous output"); + } + CommandGenerator.expectSameData(previousOutput, previousColumnIdx, output, columnIdx); + } + } + return VALIDATION_OK; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java index 58be5c939a0c9..31bdcd06ecce1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/KeepGenerator.java @@ -80,7 +80,17 @@ public ValidationResult validateOutput( return new ValidationResult(false, "Expecting at most [" + previousColumns.size() + "] columns, got [" + columns.size() + "]"); } - return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output, deterministic); + if (deterministic) { + for (int columnIdx = 0; columnIdx < columns.size(); columnIdx++) { + Column c = columns.get(columnIdx); + int previousColumnIdx = previousColumns.indexOf(columns.get(columnIdx)); + if (previousColumnIdx == -1) { + return new ValidationResult(false, "Column [" + c + "] not in previous output"); + } + CommandGenerator.expectSameData(previousOutput, previousColumnIdx, output, columnIdx); + } + } + return VALIDATION_OK; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java index 6ef7151cdfcc3..604b6c94d733e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LimitGenerator.java @@ -49,6 +49,16 @@ public ValidationResult validateOutput( if (output.size() > limit) { return new ValidationResult(false, "Expecting at most [" + limit + "] records, got [" + output.size() + "]"); } - return CommandGenerator.expectSameColumns(previousColumns, columns); + + ValidationResult result = CommandGenerator.expectSameColumns(previousColumns, columns); + if (result.success() == false) { + return result; + } + if (deterministic) { + for (int i = 0; i < columns.size(); i++) { + CommandGenerator.expectSameData(previousOutput.subList(0, Math.min(previousOutput.size(), limit)), i, output, i); + } + } + return VALIDATION_OK; } } From 1151b503ffaf065cb06e91ca94409aa71b2e8d58 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 08:44:52 +0000 Subject: [PATCH 4/5] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index bf1a90e5be4e9..6e7d51d3d3020 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -index_request_include_tsid,9167000 +security_stats_endpoint,9168000 From f5d085fb8fd434191a3d3dd76c9251881a06c80f Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 23 Sep 2025 12:41:01 +0200 Subject: [PATCH 5/5] Fix compile conflict after merge --- .../org/elasticsearch/xpack/esql/GenerativeCsvTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java index d0023f0832df1..153cd4801acb9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/GenerativeCsvTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.lucene.DataPartitioning; import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverCompletionInfo; import org.elasticsearch.compute.operator.DriverRunner; @@ -94,6 +95,7 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlan; +import org.elasticsearch.xpack.esql.planner.PhysicalSettings; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; @@ -748,8 +750,9 @@ void executeSubPlan( if (dataNodePlan != null) { var searchStats = new DisabledSearchStats(); var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalSettings = new PhysicalSettings(DataPartitioning.AUTO, ByteSizeValue.ofMb(1), 10_000); var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( - new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, foldCtx, searchStats) + new LocalPhysicalOptimizerContext(physicalSettings, new EsqlFlags(true), configuration, foldCtx, searchStats) ); var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer);