From 302e6c0c711761f7b39c2c243d1d75a9a508aaa3 Mon Sep 17 00:00:00 2001 From: jasir Date: Thu, 25 Mar 2021 13:44:10 +0100 Subject: [PATCH 1/3] Added example pipeline for DB Argument Setter. --- ...db-argument-setter-cdap-data-pipeline.json | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json diff --git a/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json new file mode 100644 index 000000000..d23f3c692 --- /dev/null +++ b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json @@ -0,0 +1,135 @@ +{ + "name": "db-argument-setter", + "description": "Data Pipeline Application", + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.4.0", + "scope": "SYSTEM" + }, + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "File", + "to": "Trash" + }, + { + "from": "Database Argument Setter", + "to": "File" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsource", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.6.0", + "scope": "SYSTEM" + }, + "properties": { + "referenceName": "file_source", + "path": "${file_path}", + "format": "text", + "skipHeader": "false", + "filenameOnly": "false", + "recursive": "false", + "ignoreNonExistingFolders": "false", + "fileEncoding": "UTF-8", + "schema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}" + } + }, + "outputSchema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}", + "id": "File", + "type": "batchsource", + "label": "File", + "icon": "icon-file" + }, + { + "name": "Trash", + "plugin": { + "name": "Trash", + "type": "batchsink", + "label": "Trash", + "artifact": { + "name": "trash-plugin", + "version": "1.2.0", + "scope": "USER" + }, + "properties": { + "referenceName": "Trash" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}" + } + ], + "inputSchema": [ + { + "name": "File", + "schema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}" + } + ], + "id": "Trash", + "type": "batchsink", + "label": "Trash", + "icon": "icon-TrashSink" + }, + { + "name": "Database Argument Setter", + "plugin": { + "name": "DatabaseArgumentSetter", + "type": "action", + "label": "Database Argument Setter", + "artifact": { + "name": "db-argument-setter", + "version": "1.5.2", + "scope": "USER" + }, + "properties": { + "jdbcPluginName": "mysql", + "connectionString": "${connection_string}", + "user": "${username}", + "password": "${password}", + "databaseName": "${database}", + "tableName": "${table}", + "argumentSelectionConditions": "${argument_selection_conditions}", + "argumentsColumns": "${arguments_column}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ], + "id": "Database-Argument-Setter", + "type": "action", + "label": "Database Argument Setter", + "icon": "fa-plug" + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "description": "Data Pipeline Application", + "maxConcurrentRuns": 1 + } +} \ No newline at end of file From ff96754f46e61ea7e120cf4a11d39353037692a4 Mon Sep 17 00:00:00 2001 From: jasir Date: Mon, 29 Mar 2021 11:20:26 +0200 Subject: [PATCH 2/3] PR comments addressed. --- ...db-argument-setter-cdap-data-pipeline.json | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json index d23f3c692..1f210859b 100644 --- a/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json +++ b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json @@ -1,11 +1,11 @@ { - "name": "db-argument-setter", - "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", "version": "6.4.0", "scope": "SYSTEM" }, + "description": "Data Pipeline Application", + "name": "example-db-argument-setter", "config": { "resources": { "memoryMB": 2048, @@ -50,15 +50,16 @@ "filenameOnly": "false", "recursive": "false", "ignoreNonExistingFolders": "false", - "fileEncoding": "UTF-8", - "schema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}" + "fileEncoding": "UTF-8" } }, - "outputSchema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}", - "id": "File", - "type": "batchsource", - "label": "File", - "icon": "icon-file" + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ], + "id": "File" }, { "name": "Trash", @@ -84,13 +85,10 @@ "inputSchema": [ { "name": "File", - "schema": "{\"type\":\"record\",\"name\":\"fileRecord\",\"fields\":[{\"name\":\"offset\",\"type\":\"long\"},{\"name\":\"body\",\"type\":\"string\"}]}" + "schema": "" } ], - "id": "Trash", - "type": "batchsink", - "label": "Trash", - "icon": "icon-TrashSink" + "id": "Trash" }, { "name": "Database Argument Setter", @@ -110,8 +108,8 @@ "password": "${password}", "databaseName": "${database}", "tableName": "${table}", - "argumentSelectionConditions": "${argument_selection_conditions}", - "argumentsColumns": "${arguments_column}" + "argumentSelectionConditions": "project=test_project;environment=production;location=US", + "argumentsColumns": "file_path" } }, "outputSchema": [ @@ -120,10 +118,7 @@ "schema": "" } ], - "id": "Database-Argument-Setter", - "type": "action", - "label": "Database Argument Setter", - "icon": "fa-plug" + "id": "Database-Argument-Setter" } ], "schedule": "0 * * * *", From dac993f8015d21da0910470ffdd222eec0a34614 Mon Sep 17 00:00:00 2001 From: jasir Date: Tue, 30 Mar 2021 13:51:01 +0200 Subject: [PATCH 3/3] Added more configurable fields. --- .../example-db-argument-setter-cdap-data-pipeline.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json index 1f210859b..fdbe7b480 100644 --- a/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json +++ b/generic-db-argument-setter/examples/example-db-argument-setter-cdap-data-pipeline.json @@ -44,7 +44,7 @@ }, "properties": { "referenceName": "file_source", - "path": "${file_path}", + "path": "/${path}/${name}/${timestamp}", "format": "text", "skipHeader": "false", "filenameOnly": "false", @@ -109,7 +109,7 @@ "databaseName": "${database}", "tableName": "${table}", "argumentSelectionConditions": "project=test_project;environment=production;location=US", - "argumentsColumns": "file_path" + "argumentsColumns": "timestamp,name,path" } }, "outputSchema": [