diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java index eff31953100ec..ce2807bec22c7 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/error/ErrorPayload.java @@ -28,5 +28,11 @@ public class ErrorPayload { private ErrorDictionary code; + private String subCode; + private String description; + + public ErrorPayload(final ErrorDictionary code, final String description) { + this(code, null, description); + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java index 9af4e05fa49fa..0cdd993a8e17c 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ActionResourceImpl.java @@ -42,6 +42,8 @@ import javax.ws.rs.core.Response; import org.talend.sdk.component.api.exception.ComponentException; +import org.talend.sdk.component.api.exception.ComponentException.ErrorOrigin; +import org.talend.sdk.component.api.exception.DiscoverSchemaException; import org.talend.sdk.component.runtime.manager.ComponentManager; import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; import org.talend.sdk.component.runtime.manager.ServiceMeta; @@ -212,29 +214,34 @@ private CompletableFuture doExecuteLocalAction(final String family, fi private Response onError(final Throwable re) { log.warn(re.getMessage(), re); - if (WebApplicationException.class.isInstance(re.getCause())) { - return WebApplicationException.class.cast(re.getCause()).getResponse(); + if (re.getCause() instanceof WebApplicationException webException) { + return webException.getResponse(); } - if (ComponentException.class.isInstance(re)) { - final ComponentException ce = (ComponentException) re; + final String description = "Action execution failed with: " + ofNullable(re.getMessage()) + .orElseGet(() -> re instanceof NullPointerException + ? "unexpected null" + : "no error message"); + if (re instanceof final DiscoverSchemaException eSchema) { + // we send reason to recognize the error on client side + final String subCode = eSchema.getPossibleHandleErrorWith().toString(); throw new WebApplicationException(Response - .status(ce.getErrorOrigin() == ComponentException.ErrorOrigin.USER ? 400 - : ce.getErrorOrigin() == ComponentException.ErrorOrigin.BACKEND ? 456 : 520, + .status(400, subCode) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, subCode, description)) + .build()); + } else if (re instanceof final ComponentException eComponent) { + throw new WebApplicationException(Response + .status(eComponent.getErrorOrigin() == ErrorOrigin.USER + ? 400 + : eComponent.getErrorOrigin() == ErrorOrigin.BACKEND ? 456 : 520, "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> NullPointerException.class.isInstance(re) ? "unexpected null" - : "no error message"))) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } throw new WebApplicationException(Response .status(520, "Unexpected callback error") - .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, - "Action execution failed with: " + ofNullable(re.getMessage()) - .orElseGet(() -> NullPointerException.class.isInstance(re) ? "unexpected null" - : "no error message"))) + .entity(new ErrorPayload(ErrorDictionary.ACTION_ERROR, description)) .build()); } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index a876e08d3308d..4436d66fb38a0 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import javax.json.bind.Jsonb; +import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.ValidationParameterEnricher; @@ -94,6 +95,9 @@ private Stream buildProperties(final List configuration, final String plugin, - final String family, final String componentName, final String action, final String version) { + final String family, final String componentName, final String version) { this.out = out; this.lineLimit = 50; this.lineCount = -1; @@ -143,7 +120,6 @@ public TaCoKitGuessSchema(final PrintStream out, final Map confi this.plugin = plugin; this.family = family; this.componentName = componentName; - this.action = action; this.columns = new LinkedHashMap<>(); this.keysNoTypeYet = new HashSet<>(); this.javaTypesManager = new JavaTypesManager(); @@ -159,7 +135,7 @@ public TaCoKitGuessSchema() { private void initClass2JavaTypeMap() { class2JavaTypeMap = new HashMap<>(); - JavaType javaTypes[] = javaTypesManager.getJavaTypes(); + JavaType[] javaTypes = javaTypesManager.getJavaTypes(); for (JavaType javaType : javaTypes) { Class nullableClass = javaType.getNullableClass(); if (nullableClass != null) { @@ -194,11 +170,8 @@ private DiscoverSchemaException handleException(final Exception e) throws Except return discoverSchemaException; } - public void guessInputComponentSchema(final Schema schema) throws Exception { + public void guessInputComponentSchema() throws Exception { try { - if (guessSchemaThroughAction(schema)) { - return; - } if (guessInputComponentSchemaThroughResult()) { return; } @@ -208,274 +181,17 @@ public void guessInputComponentSchema(final Schema schema) throws Exception { throw handleException(new Exception(ERROR_NO_AVAILABLE_SCHEMA_FOUND)); } - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch, - final boolean isStartOfJob) throws Exception { + /** + * When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec + */ + public void guessComponentSchemaByLifecycle() throws Exception { try { - executeDiscoverSchemaExtendedAction(incomingSchema, outgoingBranch); - } catch (Exception e) { - final DiscoverSchemaException dse = transformException(e); - // When a processor is the start of a studio job and dev explicitly set the handleError to Lifecycle exec - if (isStartOfJob && EXECUTE_LIFECYCLE == dse.getPossibleHandleErrorWith()) { - try { - guessOutputComponentSchemaThroughResult(); - } catch (Exception er) { - throw handleException(e); - } - } else { - throw handleException(e); - } + guessOutputComponentSchemaThroughResult(); + } catch (Exception er) { + throw handleException(er); } } - public void guessComponentSchema(final Schema incomingSchema, final String outgoingBranch) throws Exception { - guessComponentSchema(incomingSchema, outgoingBranch, false); - } - - private void executeDiscoverSchemaExtendedAction(final Schema schema, final String branch) throws Exception { - final Collection services = getPluginServices(); - ServiceMeta.ActionMeta actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && - a.getType().equals(SCHEMA_EXTENDED_TYPE) && - componentName.equals(a.getAction())) - .findFirst() - .orElse(null); - // did not find action named like componentName, trying to find one matching action... - if (actionRef == null) { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#" + SCHEMA_EXTENDED_TYPE)); - } - final Object schemaResult = - actionRef.getInvoker().apply(buildActionConfig(actionRef, configuration, schema, branch)); - if (schemaResult instanceof Schema) { - final Schema result = (Schema) schemaResult; - if (result.getEntries().isEmpty()) { - throw new DiscoverSchemaException(ERROR_NO_AVAILABLE_SCHEMA_FOUND, EXCEPTION); - } else { - fromSchema(Schema.class.cast(schemaResult)); - } - } - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration, final Schema schema, final String branch) { - final String schemaPath = action.getParameters() - .get() - .stream() - .filter(p -> Schema.class.isAssignableFrom((Class) p.getJavaType())) - .map(p -> p.getPath()) - .findFirst() - .orElse(""); - final String branchPath = action.getParameters() - .get() - .stream() - .filter(p -> String.class.isAssignableFrom((Class) p.getJavaType())) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(""); - - final Map mapped = new HashMap<>(); - if (!schemaPath.isEmpty()) { - try (final Jsonb jsonb = JsonbBuilder.create()) { - mapped.put(schemaPath, jsonb.toJson(schema)); - } catch (final Exception e) { - throw new IllegalStateException(e); - } - } - if (!branchPath.isEmpty()) { - mapped.put(branchPath, branch); - } - if (configuration == null || configuration.isEmpty()) { - return mapped; - } - final String prefix = action - .getParameters() - .get() - .stream() - .filter(s -> !s.getPath().equals(schemaPath) && !s.getPath().equals(branchPath)) - .map(ParameterMeta::getPath) - .findFirst() - .orElse(null); - if (prefix == null) { - return mapped; - } - mapped.putAll(configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), prefix) || prefix.equals(e.getKey())) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); - return mapped; - } - - private Map buildActionConfig(final ServiceMeta.ActionMeta action, - final Map configuration) { - if (configuration == null || configuration.isEmpty()) { - return configuration; // no-mapping - } - - final String prefix = action - .getParameters() - .get() - .stream() - .filter(param -> param.getMetadata().containsKey(TCOMP_CONFIGURATIONTYPE_TYPE) - && DATASET.equals(param.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(ParameterMeta::getPath) - .orElse(null); - - if (prefix == null) { // no mapping to do - return configuration; - } - - final ParameterMeta dataSet = findDataset(action) - .orElseThrow(() -> new IllegalArgumentException("Dataset not found for " + action.getAction())); - - final String dataSetPath = dataSet.getPath(); - return configuration - .entrySet() - .stream() - .filter(e -> isChildParameter(e.getKey(), dataSetPath)) - .collect(toMap(e -> prefix + e.getKey().substring(dataSetPath.length()), Map.Entry::getValue)); - } - - private boolean isChildParameter(final String path, final String parentPath) { - return path.startsWith(parentPath) && path.substring(parentPath.length()).startsWith("."); - } - - private Optional findDataset(final ServiceMeta.ActionMeta action) { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - - // dataset name should be the same as DiscoverSchema action name - final Collection metas = toStream(componentMeta.getParameterMetas().get()).collect(toList()); - return ofNullable(metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE)) - && action.getAction().equals(p.getMetadata().get("tcomp::configurationtype::name"))) - .findFirst() - .orElseGet(() -> { - // find and use single dataset - final Iterator iterator = metas - .stream() - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .iterator(); - if (iterator.hasNext()) { - final ParameterMeta value = iterator.next(); - if (!iterator.hasNext()) { - return value; - } - log - .warn("Multiple potential datasets for {}:{}, ignoring parameters", action.getType(), - action.getAction()); - } - return null; - })); - } - - private ComponentFamilyMeta.BaseMeta findComponent(final ComponentFamilyMeta familyMeta) { - return Stream - .concat(familyMeta.getPartitionMappers().entrySet().stream(), - familyMeta.getProcessors().entrySet().stream()) - .filter(e -> e.getKey().equals(componentName)) - .map(Map.Entry::getValue) - .findFirst() - .orElseThrow(() -> new IllegalStateException(NO_COMPONENT + componentName)); - } - - private ComponentFamilyMeta findFamily() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException("No component family " + plugin)) - .get(ContainerComponentRegistry.class) - .getComponents() - .get(family); - } - - private Stream toStream(final Collection parameterMetas) { - return Stream - .concat(parameterMetas.stream(), - parameterMetas - .stream() - .map(ParameterMeta::getNestedParameters) - .filter(Objects::nonNull) - .flatMap(this::toStream)); - } - - private Optional findFirstComponentDataSetName() { - final ComponentFamilyMeta familyMeta = findFamily(); - final ComponentFamilyMeta.BaseMeta componentMeta = findComponent(familyMeta); - return toStream(componentMeta.getParameterMetas().get()) - .filter(p -> DATASET.equals(p.getMetadata().get(TCOMP_CONFIGURATIONTYPE_TYPE))) - .findFirst() - .map(p -> p.getMetadata().get("tcomp::configurationtype::name")); - } - - public boolean guessSchemaThroughAction(final Schema schema) { - final Collection services = getPluginServices(); - - ServiceMeta.ActionMeta actionRef; - if (action == null || action.isEmpty()) { - // dataset name should be the same as DiscoverSchema action name so let's try to guess from the component - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - if (actionRef == null) { - // let's try DiscoverSchemaExtended action name - actionRef = findFirstComponentDataSetName() - .flatMap(datasetName -> services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getType().equals(SCHEMA_EXTENDED_TYPE)) - .filter(a -> a.getAction().equals(datasetName)) - .findFirst()) - .orElse(null); - } - } else { - actionRef = services - .stream() - .flatMap(s -> s.getActions().stream()) - .filter(a -> a.getFamily().equals(family) && a.getAction().equals(action) - && a.getType().equals(SCHEMA_TYPE)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No action " + family + "#" + SCHEMA_TYPE + "#" + action)); - } - if (actionRef == null) { - return false; - } - final Map actionConfiguration = - SCHEMA_TYPE.equals(actionRef.getType()) ? buildActionConfig(actionRef, configuration) - : buildActionConfig(actionRef, configuration, schema, "INPUT"); - final Object schemaResult = actionRef.getInvoker().apply(actionConfiguration); - - if (schemaResult instanceof Schema) { - return fromSchema(Schema.class.cast(schemaResult)); - - } else { - log.error(ERROR_INSTANCE_SCHEMA); - return false; - } - } - - private Collection getPluginServices() { - return componentManager - .findPlugin(plugin) - .orElseThrow(() -> new IllegalArgumentException(NO_COMPONENT + plugin)) - .get(ContainerComponentRegistry.class) - .getServices(); - } - private boolean fromSchema(final Schema schema) { final Collection entries = schema.getEntries(); if (entries == null || entries.isEmpty()) { @@ -492,8 +208,7 @@ private boolean fromSchema(final Schema schema) { public Collection getFixedSchema(final String execute) { SchemaConverter sc = new SchemaConverter(); Object o = sc.toObjectImpl(execute); - if (o instanceof Schema) { - final Schema schema = Schema.class.cast(o); + if (o instanceof final Schema schema) { final Collection entries = schema.getEntries(); if (entries == null || entries.isEmpty()) { log.info(NO_COLUMN_FOUND_BY_GUESS_SCHEMA); diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java index 054bc1f7c308f..65cfa29bc974b 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TaCoKitGuessSchemaTest.java @@ -22,10 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.talend.sdk.component.api.record.SchemaProperty.IS_KEY; -import static org.talend.sdk.component.api.record.SchemaProperty.PATTERN; import static org.talend.sdk.component.api.record.SchemaProperty.SCALE; import static org.talend.sdk.component.api.record.SchemaProperty.SIZE; -import static org.talend.sdk.component.api.record.SchemaProperty.STUDIO_TYPE; import java.io.ByteArrayOutputStream; import java.io.File; @@ -65,7 +63,6 @@ import org.talend.sdk.component.api.record.Record; import org.talend.sdk.component.api.record.Schema; import org.talend.sdk.component.api.record.Schema.Entry; -import org.talend.sdk.component.api.record.SchemaProperty; import org.talend.sdk.component.api.service.Service; import org.talend.sdk.component.api.service.record.RecordBuilderFactory; import org.talend.sdk.component.api.service.schema.DiscoverSchemaExtended; @@ -116,18 +113,11 @@ void guessSchemaUseVersion(String version) throws Exception { Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", - "inputDi", null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; - guessSchema.guessInputComponentSchema(null); + guessSchema.guessInputComponentSchema(); guessSchema.close(); assertTrue(byteArrayOutputStream.size() > 0); @@ -156,20 +146,13 @@ void guessSchemaUseVersionNOK(final String version) throws Exception { Collections.singletonMap("para1", "bla"), "test-classes", "TaCoKitGuessSchemaTest", - "inputDi", null, version) { - @Override - public boolean guessSchemaThroughAction(final Schema schema) { - // stub to invoke: guessInputComponentSchemaThroughResult - return false; - } - }; final DiscoverSchemaException exception = Assertions.assertThrows(DiscoverSchemaException.class, - () -> guessSchema.guessInputComponentSchema(null)); + () -> guessSchema.guessInputComponentSchema()); assertEquals(EXPECTED_ERROR_MESSAGE, exception.getMessage()); assertEquals(HandleErrorWith.EXCEPTION, exception.getPossibleHandleErrorWith()); } @@ -191,75 +174,18 @@ public boolean guessSchemaThroughAction(final Schema schema) { .withType(Schema.Type.BOOLEAN) .build(); - @Test - void guessProcessorSchema() throws Exception { - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Entry f1 = factory.newEntryBuilder() - .withName("f1") - .withType(Schema.Type.STRING) - .withProp(SchemaProperty.ORIGIN_TYPE, "VARCHAR") - .build(); - final Entry f2 = factory.newEntryBuilder() - .withName("f2") - .withType(Schema.Type.LONG) - .withDefaultValue(11l) - .withProp(SchemaProperty.ORIGIN_TYPE, "LONGINT") - .build(); - final Entry f3 = factory.newEntryBuilder() - .withName("f3") - .withType(Schema.Type.BOOLEAN) - .withProp(SchemaProperty.ORIGIN_TYPE, "BOOLEAN") - .build(); - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .build(); - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = - new TaCoKitGuessSchema(out, config, "test-classes", "TaCoKitGuessSchemaTest", "outputDi", null, - "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"sourceType\":\"VARCHAR\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"sourceType\":\"LONGINT\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"sourceType\":\"BOOLEAN\",\"talendType\":\"id_Boolean\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void guessProcessorSchemaRecordBuilderFactoryImpl() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(null); - } - - @Test - void guessProcessorSchemaAvroRecordBuilderFactory() throws Exception { - guessProcessorSchemaWithRecordBuilderFactory(factory); - } - @Test void guessProcessorSchemaInStartOfJob() throws Exception { try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(byteArrayOutputStream)) { - final Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD).build(); Map config = new HashMap<>(); config.put("configuration.shouldActionFail", "true"); config.put("configuration.failWith", "EXECUTE_LIFECYCLE"); final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); + "TaCoKitGuessSchemaTest", null, "1"); // guess schema action will fail and as start of job is true, it should use processor lifecycle redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", true); + guessSchema.guessComponentSchemaByLifecycle(); guessSchema.close(); restoreStdout(); final String expected = @@ -272,221 +198,6 @@ void guessProcessorSchemaInStartOfJob() throws Exception { } } - @Test - void guessProcessorSchemaInStartWithMockExecution() throws Exception { - final Schema sin = new RecordBuilderFactoryImpl("test-classes").newSchemaBuilder(Schema.Type.RECORD).build(); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Map config = new HashMap<>(); - config.put("configuration.shouldActionFail", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(out, config, "test-classes", - "TaCoKitGuessSchemaTest", "outputDi", null, "1"); - try { - redirectStdout(out); - guessSchema.guessComponentSchema(sin, "out", true); - } catch (Exception e) { - guessSchema.close(); - } - restoreStdout(); - // same transformations as in Studio - final String flattened = flatten(byteArrayOutputStream); - final Matcher errorMatcher = errorPattern.matcher(flattened); - assertFalse(schemaPattern.matcher(flattened).find()); - assertTrue(errorMatcher.find()); - final DiscoverSchemaException de = jsonToException(errorMatcher.group()); - assertNotNull(de); - assertEquals("Cannot execute action.", de.getMessage()); - assertEquals(HandleErrorWith.EXECUTE_MOCK_JOB, de.getPossibleHandleErrorWith()); - } - } - - private void guessProcessorSchemaWithRecordBuilderFactory(RecordBuilderFactory facto) throws Exception { - final RecordBuilderFactory factory = facto == null ? new RecordBuilderFactoryImpl("test") : facto; - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withProp("aprop", "a property!") - .withEntry(f1) - .withEntry(f2) - .withEntry(f3) - .withEntry(factory.newEntryBuilder() - .withName("id") - .withType(Schema.Type.INT) - .withNullable(false) - .withRawName("id") - .withComment("hjk;ljkkj") - .withProp(STUDIO_TYPE, "id_Integer") - .withProp(IS_KEY, "true") - .withProp(SCALE, "0") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("name") - .withComment("hljkjhlk") - .withDefaultValue("toto") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "0") - .withProp(SIZE, "20") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("flag") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("flag") - .withProp(STUDIO_TYPE, "id_Character") - .withProp(SCALE, "0") - .withProp(SIZE, "4") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("female") - .withType(Schema.Type.BOOLEAN) - .withNullable(true) - .withRawName("female") - .withProp(STUDIO_TYPE, "id_Boolean") - .withProp(SCALE, "0") - .withProp(SIZE, "1") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num1") - .withType(Schema.Type.BYTES) - .withNullable(true) - .withRawName("num1") - .withComment("hhhh") - .withProp(STUDIO_TYPE, "id_Byte") - .withProp(SCALE, "0") - .withProp(SIZE, "3") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("num2") - .withType(Schema.Type.INT) - .withNullable(true) - .withRawName("num2") - .withProp(STUDIO_TYPE, "id_Short") - .withProp(SCALE, "0") - .withProp(SIZE, "5") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("age") - .withType(Schema.Type.LONG) - .withNullable(true) - .withRawName("age") - .withProp(STUDIO_TYPE, "id_Long") - .withProp(SCALE, "0") - .withProp(SIZE, "19") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bonus") - .withType(Schema.Type.FLOAT) - .withNullable(true) - .withRawName("bonus") - .withProp(STUDIO_TYPE, "id_Float") - .withProp(SCALE, "2") - .withProp(SIZE, "12") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("salary") - .withType(Schema.Type.DOUBLE) - .withNullable(true) - .withRawName("salary") - .withProp(STUDIO_TYPE, "id_Double") - .withProp(SCALE, "2") - .withProp(SIZE, "22") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("play") - .withType(Schema.Type.STRING) - .withNullable(true) - .withRawName("play") - .withProp(STUDIO_TYPE, "id_String") - .withProp(SCALE, "2") - .withProp(SIZE, "10") - .build()) - .withEntry(factory.newEntryBuilder() - .withName("startdate") - .withType(Schema.Type.DATETIME) - .withNullable(true) - .withRawName("startdate") - .withProp(STUDIO_TYPE, "id_Date") - .withProp(PATTERN, "yyyy-MM-dd") - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.param1", "parameter one"); - config.put("configuration.param2", "parameter two"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"f1\",\"nullable\":false,\"originalDbColumnName\":\"f1\",\"talendType\":\"id_String\"},{\"default\":\"11\",\"defaut\":\"11\",\"label\":\"f2\",\"nullable\":false,\"originalDbColumnName\":\"f2\",\"talendType\":\"id_Long\"},{\"label\":\"f3\",\"nullable\":false,\"originalDbColumnName\":\"f3\",\"talendType\":\"id_Boolean\"},{\"comment\":\"hjk;ljkkj\",\"key\":true,\"label\":\"id\",\"length\":10,\"nullable\":false,\"originalDbColumnName\":\"id\",\"precision\":0,\"talendType\":\"id_Integer\"},{\"comment\":\"hljkjhlk\",\"default\":\"toto\",\"defaut\":\"toto\",\"label\":\"name\",\"length\":20,\"nullable\":true,\"originalDbColumnName\":\"name\",\"precision\":0,\"talendType\":\"id_String\"},{\"label\":\"flag\",\"length\":4,\"nullable\":true,\"originalDbColumnName\":\"flag\",\"precision\":0,\"talendType\":\"id_Character\"},{\"label\":\"female\",\"length\":1,\"nullable\":true,\"originalDbColumnName\":\"female\",\"precision\":0,\"talendType\":\"id_Boolean\"},{\"comment\":\"hhhh\",\"label\":\"num1\",\"length\":3,\"nullable\":true,\"originalDbColumnName\":\"num1\",\"precision\":0,\"talendType\":\"id_Byte\"},{\"label\":\"num2\",\"length\":5,\"nullable\":true,\"originalDbColumnName\":\"num2\",\"precision\":0,\"talendType\":\"id_Short\"},{\"label\":\"age\",\"length\":19,\"nullable\":true,\"originalDbColumnName\":\"age\",\"precision\":0,\"talendType\":\"id_Long\"},{\"label\":\"bonus\",\"length\":12,\"nullable\":true,\"originalDbColumnName\":\"bonus\",\"precision\":2,\"talendType\":\"id_Float\"},{\"label\":\"salary\",\"length\":22,\"nullable\":true,\"originalDbColumnName\":\"salary\",\"precision\":2,\"talendType\":\"id_Double\"},{\"label\":\"play\",\"length\":10,\"nullable\":true,\"originalDbColumnName\":\"play\",\"precision\":2,\"talendType\":\"id_String\"},{\"label\":\"startdate\",\"nullable\":true,\"originalDbColumnName\":\"startdate\",\"pattern\":\"\\\"yyyy-MM-dd\\\"\",\"talendType\":\"id_Date\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - - @Test - void testFromSchema() throws Exception { - final RecordBuilderFactory factory = new RecordBuilderFactoryImpl("test-classes"); - try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(byteArrayOutputStream)) { - Schema schema = factory.newSchemaBuilder(Schema.Type.RECORD) - .withEntry(factory.newEntryBuilder() - .withName("name") - .withType(Schema.Type.STRING) - .withProp(STUDIO_TYPE, StudioTypes.STRING) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("bit") - .withType(Schema.Type.BYTES) - .withProp(STUDIO_TYPE, StudioTypes.BYTE) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("dynamic") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DYNAMIC) - .withProp(PATTERN, "dd/MM/YYYY") - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .withEntry(factory.newEntryBuilder() - .withName("document") - .withType(Schema.Type.RECORD) - .withProp(STUDIO_TYPE, StudioTypes.DOCUMENT) - .withElementSchema(factory.newSchemaBuilder(Schema.Type.RECORD).build()) - .build()) - .build(); - - Map config = new HashMap<>(); - config.put("configuration.skipAssertions", "true"); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema( - out, config, "test-classes", "TaCoKitGuessSchemaTest", - "outputDi", null, "1"); - redirectStdout(out); - guessSchema.guessComponentSchema(schema, "out", false); - guessSchema.close(); - restoreStdout(); - - final String flattened = flatten(byteArrayOutputStream); - final String expected = - "[{\"label\":\"name\",\"nullable\":false,\"originalDbColumnName\":\"name\",\"talendType\":\"id_String\"},{\"label\":\"bit\",\"nullable\":false,\"originalDbColumnName\":\"bit\",\"talendType\":\"id_Byte\"},{\"label\":\"dynamic\",\"nullable\":true,\"originalDbColumnName\":\"dynamic\",\"pattern\":\"\\\"dd/MM/YYYY\\\"\",\"talendType\":\"id_Dynamic\"},{\"label\":\"document\",\"nullable\":true,\"originalDbColumnName\":\"document\",\"talendType\":\"id_Document\"},{\"comment\":\"branch name\",\"label\":\"out\",\"nullable\":false,\"originalDbColumnName\":\"out\",\"talendType\":\"id_String\"}]"; - assertTrue(byteArrayOutputStream.size() > 0); - final Matcher schemaMatcher = schemaPattern.matcher(flattened); - assertFalse(errorPattern.matcher(flattened).find()); - assertTrue(schemaMatcher.find()); - assertEquals(expected, schemaMatcher.group()); - } - } - @Test void serializeDiscoverSchemaException() throws Exception { final DiscoverSchemaException de = new DiscoverSchemaException( diff --git a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java index 6d2982a7bd9d2..fec1789debddd 100644 --- a/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java +++ b/component-studio/component-runtime-di/src/test/java/org/talend/sdk/component/runtime/di/schema/TypeConversionTest.java @@ -31,7 +31,7 @@ class TypeConversionTest { @Test void jsonValueToTalendType() { JavaTypesManager javaTypesManager = new JavaTypesManager(); - final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null, null); + final TaCoKitGuessSchema guessSchema = new TaCoKitGuessSchema(null, null, null, null, null, null); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1))); assertEquals(javaTypesManager.LONG.getId(), guessSchema.getTalendType(jsonProvider.createValue(1L))); assertEquals(javaTypesManager.DOUBLE.getId(), guessSchema.getTalendType(jsonProvider.createValue(1.1d)));