Added set data type, plus table-level properties: classification, Ser…

…de parameters, and table properties key/value pairs

adapter/0.1.3.json

@@ -0,0 +1,47 @@
+/**
+ * Copyright © 2016-2020 by IntegrIT S.A. dba Hackolade.  All rights reserved.
+ *
+ * The copyright to the computer software herein is the property of IntegrIT S.A.
+ * The software may be used and/or copied only with the written permission of
+ * IntegrIT S.A. or in accordance with the terms and conditions stipulated in
+ * the agreement/contract under which the software has been supplied.
+ *
+ * {
+ * 		"add": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"remove": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"modify": {
+ *	 		"entity": [
+ *	 			{
+ *					"from": { <properties that identify record> },
+ *					"to": { <properties that need to be changed> }
+ *				}
+ *			],
+ *			"container": [],
+ *			"model": [],
+ *			"view": [],
+ *			"field": []
+ * 		},
+ * }
+ */
+
+{
+	"modify": {
+		"entity": ["transformTablePropertiesFromJSONToKeyValue"]
+	}
+}

forward_engineering/helpers/awsCliScriptHelpers/glueTableHelper.js

@@ -19,7 +19,7 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => {
 				SortColumns: getGlueTableSortingColumns(tableSchema.sortedByKey, tableSchema.properties),
 				StoredAsSubDirectories: tableSchema.StoredAsSubDirectories
 			},
-			Parameters: mapTableParameters(tableSchema.tableProperties),
+			Parameters: mapTableParameters(tableSchema),
 			PartitionKeys: getGluePartitionKeyTableColumns(tableSchema.properties),
 			TableType: tableSchema.externalTable ? 'EXTERNAL_TABLE' : ''
 		}
@@ -30,11 +30,11 @@ const getGlueTableCreateStatement = (tableSchema, databaseName) => {
 };
 
 const mapSerdeInfo = (tableSchema) => {
+	const paths = getSerdePathParams(tableSchema.parameterPaths, tableSchema.properties);
+	const serDeParameters = getSerDeParams(tableSchema.serDeParameters);
 	return {
 		SerializationLibrary: tableSchema.serDeLibrary,
-		Parameters: {
-			paths: getSerdePathParams(tableSchema.parameterPaths, tableSchema.properties)
-		}
+		Parameters: Object.assign({}, { paths }, serDeParameters)
 	};
 }
 
@@ -46,9 +46,23 @@ const getSerdePathParams = (parameterPaths = [], properties = {}) => {
 	}).join(',');
 }
 
-const mapTableParameters = (properties) => {
+const getSerDeParams = (params = []) => {
+	return params.reduce((acc, param) => {
+		acc[param.serDeKey] = param.serDeValue;
+		return acc;
+	}, {});
+}
+
+const mapTableParameters = (tableSchema) => {
 	try {
-		return JSON.parse(properties);
+		const props = (tableSchema.tableProperties || []).reduce((acc, prop) => {
+			acc[prop.tablePropKey] = prop.tablePropValue;
+			return acc;
+		}, {});
+		if (tableSchema.classification) {
+			props.classification = tableSchema.classification.toLowerCase();
+		}
+		return props;
 	} catch(err) {
 		return {};
 	}

forward_engineering/helpers/columnHelper.js

@@ -54,22 +54,24 @@ const getPropertyByType = (type) => {
 	}, childTypeDescriptor.defaultValues || {});
 };
 
-const getArray = getTypeByProperty => property => {
-	let type;
-
+const getTypeForArrayLikeField = (getTypeByProperty, property) => {
 	if (Array.isArray(property.items)) {
-		type = getTypeByProperty(property.items[0]);
+		return getTypeByProperty(property.items[0]);
 	} else if (property.items) {
-		type = getTypeByProperty(property.items);
+		return getTypeByProperty(property.items);
 	} else if (Array.isArray(property.oneOf)) {
 		const unions = getUnionFromOneOf(getTypeByProperty)(property);
 		const name = Object.keys(unions)[0];
-		type = unions[name];
+		return unions[name];
 	} else if (Array.isArray(property.allOf)) {
 		const unions = getUnionFromAllOf(getTypeByProperty)(property);
 		const name = Object.keys(unions)[0];
-		type = unions[name];
+		return unions[name];
 	}
+}
+
+const getArray = getTypeByProperty => property => {
+	let type = getTypeForArrayLikeField(getTypeByProperty, property);
 
 	if (!type) {
 		type = getTypeByProperty(getChildBySubtype('array', property.subtype));
@@ -78,6 +80,16 @@ const getArray = getTypeByProperty => property => {
 	return `array<${type}>`;
 };
 
+const getSet = getTypeByProperty => property => {
+	let type = getTypeForArrayLikeField(getTypeByProperty, property);
+
+	if (!type) {
+		type = getTypeByProperty(getChildBySubtype('set', property.subtype));
+	}
+
+	return `set<${type}>`;
+};
+
 const getMapKey = (property) => {
 	if (['char', 'varchar'].indexOf(property.keySubtype) !== -1) {
 		return property.keySubtype + '(255)';
@@ -215,6 +227,8 @@ const getTypeByProperty = (property) => {
 			return getArray(getTypeByProperty)(property);
 		case 'map':
 			return getMap(getTypeByProperty)(property);
+		case 'set':
+			return getSet(getTypeByProperty)(property);
 		case undefined:
 			return 'string';
 		default:

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "Glue",
-	"version": "0.1.2",
-	"versionDate": "2019-12-13",
+	"version": "0.1.3",
+	"versionDate": "2020-01-11",
 	"author": "hackolade",
 	"engines": {
 		"hackolade": "3.5.5",

properties_pane/entity_level/entityLevelConfig.json

@@ -288,6 +288,24 @@ making sure that you maintain a proper JSON format.
 					"value": "input/output format"
 				}
 			},
+			{
+				"propertyName": "Serde parameters",
+				"propertyType": "group",
+				"propertyKeyword": "serDeParameters",
+				"structure": [
+					{
+						"propertyName": "Key",
+						"propertyKeyword": "serDeKey",
+						"propertyType": "text"
+					},
+					{
+						"propertyName": "Value",
+						"propertyKeyword": "serDeValue",
+						"propertyTooltip": "",
+						"propertyType": "text"
+					}
+				]
+			},
 			{
 				"propertyName": "Stored as sub-directories",
 				"propertyKeyword": "StoredAsSubDirectories",
@@ -352,13 +370,37 @@ making sure that you maintain a proper JSON format.
 				"propertyKeyword": "location",
 				"propertyType": "text"
 			},
+			{
+				"propertyName": "Classification",
+				"propertyKeyword": "classification",
+				"propertyTooltip": "Select from list of options",
+				"propertyType": "select",
+				"options": [
+					"Avro",
+					"CSV",
+					"JSON",
+					"XML",
+					"Parquet",
+					"ORC"
+				]
+			},
 			{
 				"propertyName": "Table properties",
+				"propertyType": "group",
 				"propertyKeyword": "tableProperties",
-				"propertyType": "details",
-				"template": "textarea",
-				"markdown": false,
-				"valueType": "string"
+				"structure": [
+					{
+						"propertyName": "Key",
+						"propertyKeyword": "tablePropKey",
+						"propertyType": "text"
+					},
+					{
+						"propertyName": "Value",
+						"propertyKeyword": "tablePropValue",
+						"propertyTooltip": "",
+						"propertyType": "text"
+					}
+				]
 			},
 			{
 				"propertyName": "Comments",

properties_pane/field_level/fieldLevelConfig.json

@@ -413,6 +413,35 @@ making sure that you maintain a proper JSON format.
 			"primaryKey",
 			"additionalProperties",
 			"comments"
+		],
+		"set": [
+			"name",
+			"code",
+			"sampleName",
+			"schemaId",
+			"refPath",
+			"description",
+			"type",
+			{
+				"propertyName": "Subtype",
+				"propertyKeyword": "subtype",
+				"propertyType": "select",
+				"options": [
+					"set<txt>",
+					"set<num>",
+					"set<ts>",
+					"set<date>",
+					"set<intrvl>",
+					"set<array>",
+					"set<map>",
+					"set<struct>",
+					"set<union>"
+				]
+			},
+			"required",
+			"primaryKey",
+			"dependencies",
+			"comments"
 		]
 	}
 } 

reverse_engineering/api.js

@@ -99,7 +99,7 @@ module.exports = {
 					{ message: err.message, stack: err.stack, error: err },
 					'Retrieving databases and tables information'
 				);
-				cb();
+				cb({ message: err.message, stack: err.stack });
 			}
 		};
 
@@ -108,6 +108,7 @@ module.exports = {
 };
 
 const mapTableData = ({ Table }, dbDescription) => {
+	const classification = getClassification(Table.Parameters);
 	const tableData = {
 		dbName: Table.DatabaseName,
 		collectionName: Table.Name,
@@ -117,7 +118,7 @@ const mapTableData = ({ Table }, dbDescription) => {
 		entityLevel: {
 			description: Table.Description,
 			externalTable: Table.TableType === 'EXTERNAL_TABLE',
-			tableProperties: JSON.stringify(Table.Parameters, null, 2),
+			tableProperties: mapTableProperties(Table.Parameters),
 			compositePartitionKey: Table.PartitionKeys.map(item => item.Name),
 			compositeClusteringKey: Table.StorageDescriptor.BucketColumns,
 			sortedByKey: mapSortColumns(Table.StorageDescriptor.SortColumns),
@@ -129,7 +130,9 @@ const mapTableData = ({ Table }, dbDescription) => {
 			inputFormatClassname: Table.StorageDescriptor.InputFormat,
 			outputFormatClassname: Table.StorageDescriptor.OutputFormat,
 			serDeLibrary: getSerDeLibrary(Table.StorageDescriptor.SerdeInfo),
-			parameterPaths: mapSerDeParameters(Table.StorageDescriptor.SerdeInfo)
+			parameterPaths: mapSerDePaths(Table.StorageDescriptor.SerdeInfo),
+			serDeParameters: mapSerDeParameters(Table.StorageDescriptor.SerdeInfo.Parameters),
+			classification
 		},
 		documents: [],
 		validation: {
@@ -159,13 +162,54 @@ const getSerDeLibrary = (data = {}) => {
 	return data.SerializationLibrary;
 }
 
-const mapSerDeParameters = (data = {}) => {
+const mapSerDePaths = (data = {}) => {
 	return _.get(data, 'Parameters.paths', '').split(',');
 }
 
+const mapSerDeParameters = (parameters = {}) => {
+	return Object.entries(parameters).reduce((acc, [key, value]) => {
+		if (key !== 'paths') {
+			acc.push({ serDeKey: key, serDeValue: value });
+			return acc;
+		}
+		return acc;
+	}, []);
+}
+
 const logInfo = (step, connectionInfo, logger) => {
 	logger.clear();
 	logger.log('info', logHelper.getSystemInfo(connectionInfo.appVersion), step);
 	logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);
 };
 
+const getClassification = (parameters = {}) => {
+	if (parameters.classification) {
+		switch (parameters.classification.toLowerCase()) {
+			case 'avro':
+				return 'Avro';
+			case 'csv':
+				return 'CSV';
+			case 'json':
+				return 'JSON';
+			case 'xml':
+				return 'XML';
+			case 'parquet':
+				return 'Parquet';
+			case 'orc':
+				return 'ORC';
+		}
+	}
+	return {};
+}
+
+const mapTableProperties = (parameters = {}) => {
+	return Object.entries(parameters).reduce((acc, [key, value]) => {
+		if (key === 'classification') {
+			return acc;
+		}
+		return acc.concat({
+			tablePropKey: key,
+			tablePropValue: value
+		});
+	}, []);
+}