-
Bug
-
Resolution: Done
-
Blocker
-
None
-
All
-
Small
On cloud-engine, the dataset validation works on correct @Option with a @Pattern annotation but when running pipeline with the configuration payload we have an exception during run.
The pulsar configuration datastore:
@Option
@Documentation("Apache pulsar URL access.")
@Pattern("^pulsar(\\+ssl)?://.*")
@Required
private String brokerURL;
The pulsar configuration dataset:
@Option
@Documentation("Pulsar datastore.")
private PulsarDataStore dataStore;
The pulsar connector:
public class PulsarOutput implements Serializable { private static final long serialVersionUID = 2163705745912165039L; private final PulsarOutputConfiguration config;
Injected configuration:
{
"context": {
"variables": {
"jobName": "pipeline for Pulsar Csv output ",
"pluginIds": "pulsar,localio",
"jobType": "fullRun",
"metrics.refresh.interval": "5000",
"origin": "PIPELINE_DESIGNER",
"metrics.enabled": "true",
"pipelineType": "batch",
"requestId": "cbc42ddb-19c7-4dfb-b45b-0e150a4ca956",
"tenantId": "bf1a2826-cdd3-42d6-aa5d-eb6d37d9aeba",
"metrics.sink.http.url": "USE_LIVY_ENV"
}
},
"environment": {
"id": "db936c32-493e-477d-8fec-a1ea2974ea3d",
"label": "",
"description": "",
"properties": {},
"tuningProperties": {}
},
"runners": [
{
"id": "5ee9ed3a9055eb0001eeca21",
"runnerType": "SparkRunner",
"properties": {
"sparkMaster": "local[5]",
"storageLevel": "MEMORY_AND_DISK",
"batchIntervalMillis": "5000"
},
"tuningProperties": {},
"environmentId": "db936c32-493e-477d-8fec-a1ea2974ea3d"
}
],
"authentication": {
"id": "fakeRuntimeAuthenticationId",
"authenticationType": "SIMPLE",
"properties": {
"username": "talend",
"password": "****"
}
},
"id": "fakeUserFlowId",
"executionId": "8dba2232-dd94-478b-9fdc-f7572f8d6160",
"contextVariables": [],
"pipelines": [
{
"components": [
{
"id": "zytbatjopxigvbd6ntgstmjp",
"type": "cHVsc2FyI1B1bHNhciNPdXRwdXQ",
"label": "",
"data": {
"properties": {
"$datasetDefinition": {
"datasetId": "60f23757-c0d8-4b2b-ac02-19f3207a4dd3",
"datasetPath": "config.dataset",
"datastorePath": "config.dataset.dataStore"
},
"$formId": "cHVsc2FyI1B1bHNhciNkYXRhc3RvcmUjUHVsc2FyRGF0YVN0b3Jl",
"configuration": {},
"$componentMetadata": {
"name": "os-pulsar",
"type": "Output",
"version": 1,
"description": "",
"technicalType": "pulsar#Pulsar#Output",
"datasetReferencePaths": [],
"connectionReferencePaths": []
},
"config": {
"sendTimeout": 30000,
"queueFullBehavior": "ERROR",
"$maxBatchSize": 1000,
"chosenTopic": "tpdAuto",
"sendMode": "SIMPLE",
"dataset": {
"contentFormat": {
"jsonConfiguration": {
"forceDouble": "true"
},
"contentFormat": "CSV",
"csvConfiguration": {
"quotedValue": "\"",
"fieldSeparator": {
"fieldSeparatorType": "COMMA"
},
"lineConfiguration": {
"lineSeparatorType": "LF",
"header": {
"size": "1.0",
"active": "false"
},
"lineSeparator": "\n",
"encoding": {
"encodingType": "UTF8"
}
},
"escape": "\\"
}
},
"subscriptionName": "tpdAuto",
"dataStore": {
"brokerURL": "pulsar://34.200.5.75:6650",
"timeout": "60.0",
"useTLS": "false",
"authentication": {
"authenticationType": "NONE"
}
},
"topic": "tpd-tn/tpd-ns/tpd-tp",
"consumerName": "tpdAuto"
},
"compressionMode": "NONE",
"batchConfig": {
"maxBytes": 131872,
"maxDelay": 1,
"maxMessages": 1000
}
}
}
},
"nodeType": "sink"
},
{
"id": "zngbxvfla0qhkuvnrf6r6j7j",
"type": "bG9jYWxpbyNMb2NhbElPI0ZpeGVkRmxvd0lucHV0UnVudGltZQ",
"label": "",
"data": {
"properties": {
"$datasetDefinition": {
"datasetId": "d547eedb-b2ed-4b39-a027-418b2b0f5a07",
"datasetPath": "configuration.dataset",
"datastorePath": "configuration.dataset.datastore"
},
"$formId": "bG9jYWxpbyNMb2NhbElPI2RhdGFzdG9yZSNGaXhlZERhdGFTdG9yZUNvbmZpZ3VyYXRpb24",
"configuration": {
"overrideValues": "",
"repeat": 1,
"overrideValuesAction": "NONE",
"dataset": {
"csvSchema": "id;name;age;birthday;math;phone",
"values": "1;jim Liu;6;2013-11-08;98;+8618618495612\n2;tom Li;5;2013-09-01;99.5;+75123456",
"format": "CSV"
}
},
"$componentMetadata": {
"name": "test-source",
"type": "Test input",
"version": 1,
"description": ""
}
}
},
"nodeType": "source"
}
],
"ports": [
{
"id": "zngbxvfla0qhkuvnrf6r6j7j-__default__",
"nodeId": "zngbxvfla0qhkuvnrf6r6j7j",
"data": {
"flowType": "__default__"
}
},
{
"id": "zytbatjopxigvbd6ntgstmjp-__default__",
"nodeId": "zytbatjopxigvbd6ntgstmjp",
"data": {
"flowType": "__default__"
}
}
],
"steps": [
{
"id": "zngbxvfla0qhkuvnrf6r6j7j-to-zytbatjopxigvbd6ntgstmjp",
"targetId": "zytbatjopxigvbd6ntgstmjp-__default__",
"sourceId": "zngbxvfla0qhkuvnrf6r6j7j-__default__",
"data": {}
}
],
"id": "fakeRuntimePipelineId",
"runner": "5ee9ed3a9055eb0001eeca21"
}
],
"logger": []
}
Log of pipeline run:
[2022-10-24 23:11:37,439] [INFO ] o.a.l.u.LineBufferedStream - stdout: [2022-10-24T23:11:37,439] [ERROR] o.t.d.s.FullRunJob$ - DATA_STREAMS_ERROR_MSG:- 'config.dataset.dataStore.brokerURL' does not match '^pulsar(\+ssl)?://.*'. [2022-10-24 23:11:37,444] [INFO ] o.a.l.u.LineBufferedStream - stdout: [2022-10-24T23:11:37,439] [ERROR] o.t.d.s.FullRunJob$ - Original Throwable: [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: java.lang.IllegalArgumentException: - 'config.dataset.dataStore.brokerURL' does not match '^pulsar(\+ssl)?://.*'. [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.reflect.ReflectionService$PayloadValidator.throwIfFailed(ReflectionService.java:998) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.reflect.ReflectionService$PayloadValidator.access$300(ReflectionService.java:888) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.reflect.ReflectionService.lambda$parameterFactory$12(ReflectionService.java:216) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.lambda$createParametersFactory$26(ComponentManager.java:1022) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager$ComponentMetaBuilder.lambda$onProcessor$27(ComponentManager.java:1814) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager$ComponentMetaBuilder.lambda$onProcessor$28(ComponentManager.java:1824) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.xbean.registry.EnrichedPropertyEditorRegistry.withCache(EnrichedPropertyEditorRegistry.java:146) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager$ComponentMetaBuilder.lambda$onProcessor$29(ComponentManager.java:1824) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentFamilyMeta$BaseMeta.instantiate(ComponentFamilyMeta.java:214) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.lambda$findDeployedInstance$19(ComponentManager.java:788) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.Optional.map(Optional.java:265) ~[?:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.findDeployedInstance(ComponentManager.java:788) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.lambda$findInstance$17(ComponentManager.java:774) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.Optional.orElseGet(Optional.java:369) ~[?:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.findInstance(ComponentManager.java:773) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.lambda$findComponentInternal$16(ComponentManager.java:739) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,445] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:271) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.concurrent.ConcurrentHashMap$ValueSpliterator.tryAdvance(ConcurrentHashMap.java:3613) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:127) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:502) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:488) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:543) ~[?:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.findComponentInternal(ComponentManager.java:741) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.sdk.component.runtime.manager.ComponentManager.createComponent(ComponentManager.java:728) ~[component-runtime-manager-1.50.0-SNAPSHOT.jar:?] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.beam.compiler.runtime.GenericRuntimeBuilder.getRuntime(GenericRuntimeBuilder.java:82) ~[data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.beam.compiler.BeamCompiler.compileComponent(BeamCompiler.java:188) ~[data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.beam.compiler.runtimeflow.RuntimeFlowBeamCompiler.compile(RuntimeFlowBeamCompiler.java:87) ~[data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.beam.compiler.runtimeflow.RuntimeFlowBeamCompiler.compile(RuntimeFlowBeamCompiler.java:32) ~[data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.streamsjob.FullRunJob$.runJob(FullRunJob.scala:233) [data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.streamsjob.FullRunJob$.processInBeamCompiler(FullRunJob.scala:140) [data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.streamsjob.FullRunJob$.main(FullRunJob.scala:56) [data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT] [2022-10-24 23:11:37,446] [INFO ] o.a.l.u.LineBufferedStream - stdout: at org.talend.datastreams.streamsjob.FullRunJob.main(FullRunJob.scala) [data-processing-runtime-streamsjob-2.21.0-PR-912-SNAPSHOT.jar:2.21.0-PR-912-SNAPSHOT]
Exception rose : 'config.dataset.dataStore.brokerURL' does not match '^pulsar(+ssl)?://.*'.
- is related to
-
TCOMP-2256 Replace nashorn javascript engine by rhino
-
- Done
-