From 416cda6b6b5221a70e9346dbc10b13bf0d64d868 Mon Sep 17 00:00:00 2001 From: Bugazelle Date: Tue, 16 Jul 2019 22:00:58 +0800 Subject: [PATCH 1/3] [Fix] Logic enhancements for exporter object --- README.md | 4 +-- src/ExportCsvToInflux/csv_object.py | 13 +++++++-- src/ExportCsvToInflux/exporter_object.py | 27 ++++++++++++++----- ...55\346\226\207\350\257\264\346\230\216.md" | 4 +-- 4 files changed, 36 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/ExportCsvToInflux/csv_object.py mode change 100644 => 100755 src/ExportCsvToInflux/exporter_object.py diff --git a/README.md b/README.md index 3ddeb0f..fd39bde 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ timestamp,url,response_time --tag_columns url \ --field_columns response_time \ --user admin \ - --password test-automation-monitoring-2019 \ + --password admin \ --server 127.0.0.1:8086 \ --drop_database=True \ --force_insert_even_csv_no_update True \ @@ -145,7 +145,7 @@ timestamp,url,response_time --tag_columns url \ --field_columns response_time \ --user admin \ - --password test-automation-monitoring-2019 \ + --password admin \ --server 127.0.0.1:8086 \ --drop_database True \ --force_insert_even_csv_no_update True \ diff --git a/src/ExportCsvToInflux/csv_object.py b/src/ExportCsvToInflux/csv_object.py old mode 100644 new mode 100755 index 8724e93..74a542e --- a/src/ExportCsvToInflux/csv_object.py +++ b/src/ExportCsvToInflux/csv_object.py @@ -238,9 +238,18 @@ def add_columns_to_csv(self, message = 'Error: The data should be list type, the item should be dict. Or the json type as following' \ 'for example: [{"new_header_1": ["new_value_1", "new_value_2", "new_value_3"]}, ' \ '{"new_header_2": ["new_value_1", "new_value_2", "new_value_3"]}]' - if data_type is not list and data_type is not str and data_type is not unicode: + try: + check_data_type = data_type is not list and data_type is not str and data_type is not unicode + except NameError: + check_data_type = data_type is not list and data_type is not str + if check_data_type: raise Exception(message) - if data_type is str or data_type is unicode: + + try: + check_data_type = data_type is str or data_type is unicode + except NameError: + check_data_type = data_type is str + if check_data_type: try: data = json.loads(data) except ValueError: diff --git a/src/ExportCsvToInflux/exporter_object.py b/src/ExportCsvToInflux/exporter_object.py old mode 100644 new mode 100755 index c16ab47..5ad3ba7 --- a/src/ExportCsvToInflux/exporter_object.py +++ b/src/ExportCsvToInflux/exporter_object.py @@ -88,8 +88,8 @@ def __check_match_and_filter(self, return status @staticmethod - def __validate_match_and_filter(csv_headers, check_columns): - """Private Function: validate_match_and_filter """ + def __validate_columns(csv_headers, check_columns): + """Private Function: validate_columns """ if check_columns: validate_check_columns = all(check_column in csv_headers for check_column in check_columns) @@ -221,6 +221,25 @@ def export_csv_to_influx(self, csv_file_length = csv_object.get_csv_lines_count(csv_file_item) csv_file_md5 = csv_object.get_file_md5(csv_file_item) csv_headers = csv_object.get_csv_header(csv_file_item) + + # Validate csv_headers + if not csv_headers: + print('Error: The csv file {0} has no header detected. Exporter stopping...'.format(csv_file_item)) + continue + + # Validate field_columns, tag_columns, match_columns, filter_columns + field_columns = self.__validate_columns(csv_headers, field_columns) + tag_columns = self.__validate_columns(csv_headers, tag_columns) + if not field_columns: + print('Error: The input --field_columns does not expected. ' + 'Please check the fields are in csv headers or not. Exporter stopping...') + if not tag_columns: + print('Error: The input --tag_columns does not expected. ' + 'Please check the fields are in csv headers or not. Exporter stopping...') + match_columns = self.__validate_columns(csv_headers, match_columns) + filter_columns = self.__validate_columns(csv_headers, filter_columns) + + # Validate time_column with open(csv_file_item) as f: csv_reader = csv.DictReader(f, delimiter=delimiter, lineterminator=lineterminator) time_column_exists = True @@ -233,10 +252,6 @@ def export_csv_to_influx(self, time_column_exists = False break - # Validate match_columns, filter_columns - match_columns = self.__validate_match_and_filter(csv_headers, match_columns) - filter_columns = self.__validate_match_and_filter(csv_headers, filter_columns) - # Check the timestamp, and generate the csv with checksum new_csv_file = 'influx.csv' new_csv_file = os.path.join(current_dir, new_csv_file) diff --git "a/\344\270\255\346\226\207\350\257\264\346\230\216.md" "b/\344\270\255\346\226\207\350\257\264\346\230\216.md" index e2bcc00..ed87385 100644 --- "a/\344\270\255\346\226\207\350\257\264\346\230\216.md" +++ "b/\344\270\255\346\226\207\350\257\264\346\230\216.md" @@ -128,7 +128,7 @@ timestamp,url,response_time --tag_columns url \ --field_columns response_time \ --user admin \ - --password test-automation-monitoring-2019 \ + --password admin \ --server 127.0.0.1:8086 \ --drop_database True \ --force_insert_even_csv_no_update True \ @@ -146,7 +146,7 @@ timestamp,url,response_time --tag_columns url \ --field_columns response_time \ --user admin \ - --password test-automation-monitoring-2019 \ + --password admin \ --server 127.0.0.1:8086 \ --drop_database True \ --force_insert_even_csv_no_update True \ From 52ed4ddd6374d5b5501db423a022b3006fc4966e Mon Sep 17 00:00:00 2001 From: Bugazelle Date: Wed, 17 Jul 2019 15:56:28 +0800 Subject: [PATCH 2/3] [Fix] Fix value error if data is something like 1.0 --- src/ExportCsvToInflux/csv_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ExportCsvToInflux/csv_object.py b/src/ExportCsvToInflux/csv_object.py index 74a542e..b8e3c88 100755 --- a/src/ExportCsvToInflux/csv_object.py +++ b/src/ExportCsvToInflux/csv_object.py @@ -203,7 +203,7 @@ def convert_csv_data_to_int_float(self, file_name): value = row[key] int_status = int_type[key] if int_status is True: - row[key] = int(value) if int_type[key] is True else value + row[key] = int(float(value)) if int_type[key] is True else value else: row[key] = float(value) if float_type[key] is True else value yield row @@ -211,7 +211,7 @@ def convert_csv_data_to_int_float(self, file_name): for key in keys: int_status = int_type[key] if int_status is True: - row[key] = int(key) if int_type[key] is True else key + row[key] = int(float(key)) if int_type[key] is True else key else: row[key] = float(key) if float_type[key] is True else key yield row From d3be59aa14a39947d0b5a6a4ca32cdb99c887fa8 Mon Sep 17 00:00:00 2001 From: Bugazelle Date: Wed, 17 Jul 2019 15:57:03 +0800 Subject: [PATCH 3/3] [Feat] Release to v0.1.11 --- src/ExportCsvToInflux/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ExportCsvToInflux/__version__.py b/src/ExportCsvToInflux/__version__.py index 850505a..13b7089 100644 --- a/src/ExportCsvToInflux/__version__.py +++ b/src/ExportCsvToInflux/__version__.py @@ -1 +1 @@ -__version__ = '0.1.10' +__version__ = '0.1.11'