From 1678c8148631230c532cf5a6c1a6029503b55f60 Mon Sep 17 00:00:00 2001 From: hyuksun98 Date: Sat, 7 Sep 2024 13:24:32 +0000 Subject: [PATCH] Adding Feature Group Editing Endpoint Issue-ID: AIMLFW-154 - Implement the function 'feature_group_by_name' in trainingmgr_main.py - Implement 'edit_feature_group_by_name' function and 'edit_featuregroup' function in common/trainingmgr_util.py - Create test code to validate the code implemented above - Add TODO,NOTE comments for code feedback Change-Id: Ie6242be93432cb1224a896581c1828b65a139b31 Signed-off-by: hyuksun98 --- tests/test_tm_apis.py | 170 ++++++++++++++++++++--- tests/test_trainingmgr_util.py | 180 ++++++++++++++++++++++++- trainingmgr/common/trainingmgr_util.py | 112 ++++++++++++++- trainingmgr/db/common_db_fun.py | 26 ++++ trainingmgr/trainingmgr_main.py | 168 +++++++++++------------ 5 files changed, 547 insertions(+), 109 deletions(-) diff --git a/tests/test_tm_apis.py b/tests/test_tm_apis.py index cda30ad6..4e0dea28 100644 --- a/tests/test_tm_apis.py +++ b/tests/test_tm_apis.py @@ -31,7 +31,7 @@ load_dotenv('tests/test.env') from trainingmgr.constants.states import States from threading import Lock -from trainingmgr import trainingmgr_main +from trainingmgr import trainingmgr_main from trainingmgr.common.tmgr_logger import TMLogger from trainingmgr.common.trainingmgr_config import TrainingMgrConfig from trainingmgr.common.exceptions_utls import DBException, TMException @@ -1183,6 +1183,7 @@ def test_neagtive_create_featuregroup_3(self, mock1, mock2): assert response.data==expected_response assert response.status_code==status.HTTP_400_BAD_REQUEST, "Return status code not equal" + class Test_get_feature_group: def setup_method(self): self.client = trainingmgr_main.APP.test_client(self) @@ -1203,42 +1204,169 @@ def test_negative_get_feature_group(self, mock1): assert response.status_code== status.HTTP_500_INTERNAL_SERVER_ERROR, "status code is not equal" assert response.data == expected_data -class Test_get_feature_group_by_name: +class Test_feature_group_by_name: def setup_method(self): self.client = trainingmgr_main.APP.test_client(self) self.logger = trainingmgr_main.LOGGER - result=[('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', '', '', '','')] - @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=result) - def test_get_feature_group_by_name(self, mock1): - expected_data=b'{"featuregroup": [{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": "", "measured_obj_class": "", "dme_port": "", "source_name": ""}]}' - fg_name='testing' - response=self.client.get('/featureGroup/{}'.format(fg_name)) - assert response.status_code == 200 , "status code is not equal" - assert response.data == expected_data + # Test Code for GET endpoint (In the case where dme is disabled) + fg_target = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')] + + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_target) + def test_feature_group_by_name_get_api(self, mock1): + expected_data = b'{"featuregroup": [{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": false, "measured_obj_class": "", "dme_port": "", "source_name": ""}]}' + fg_name = 'testing' + response = self.client.get('/featureGroup/{}'.format(fg_name)) + assert response.status_code == 200, "status code is not equal" + assert response.data == expected_data, response.data - @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', return_value=None) - def test_negative_get_feature_group_by_name(self, mock1): + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=None) + def test_negative_feature_group_by_name_get_api_1(self, mock1): expected_data=b'{"Exception": "Failed to fetch feature group info from db"}' fg_name='testing' response=self.client.get('/featureGroup/{}'.format(fg_name)) assert response.status_code == 404 , "status code is not equal" - assert response.data == expected_data + assert response.data == expected_data, response.data - @patch('trainingmgr.trainingmgr_main.get_feature_group_by_name_db', side_effect=DBException("Failed to execute query in get_feature_groupsDB ERROR")) - def test_negative_get_feature_group_by_name_2(self, mock1): + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', side_effect=DBException("Failed to execute query in get_feature_groupsDB ERROR")) + def test_negative_feature_group_by_name_get_api_2(self, mock1): expected_data=b'{"Exception": "Failed to execute query in get_feature_groupsDB ERROR"}' fg_name='testing' response=self.client.get('/featureGroup/{}'.format(fg_name)) assert response.status_code == 500 , "status code is not equal" - assert response.data == expected_data + assert response.data == expected_data, response.data - def test_negative_get_feature_group_name_for_incorrect_name(self): - featuregroup_name="usecase*" - response=self.client.get('/featureGroup/'.format(featuregroup_name), content_type="application/json") - assert response.status_code==status.HTTP_400_BAD_REQUEST - assert response.data == b'{"Exception":"The trainingjob_name is not correct"}\n' + def test_negative_feature_group_by_name_get_api_with_incorrect_name(self): + expected_data=b'{"Exception": "The featuregroup_name is not correct"}' + fg_name="usecase*" + response=self.client.get('/featureGroup/{}'.format(fg_name)) + assert response.status_code == 400, "status code is not equal" + assert response.data == expected_data, response.data + + + # Test Code for PUT endpoint (In the case where DME is edited from disabled to enabled) + fg_init = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')] + fg_edit = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', True, '', '31823', '')] + + the_response= Response() + the_response.status_code = status.HTTP_201_CREATED + the_response.headers={"content-type": "application/json"} + the_response._content = b'' + mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") + feature_group_data1=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response) + @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data1) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name') + def test_feature_group_by_name_put_api(self, mock1, mock2, mock3, mock4, mock5, mock6): + expected_data = b'{"result": "Feature Group Edited"}' + fg_name='testing' + featuregroup_req = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + response = self.client.put("/featureGroup/{}".format(fg_name), + data=json.dumps(featuregroup_req), + content_type="application/json") + assert response.status_code == 200, "status code is not equal" + assert response.data == expected_data, response.data + the_response1= Response() + the_response1.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + the_response1.headers={"content-type": "application/json"} + the_response1._content = b'' + mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") + feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response1) + @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name') + def test_negative_feature_group_by_name_put_api_1(self, mock1, mock2, mock3, mock4, mock5, mock6): + expected_data = b'{"Exception": "Cannot create dme job"}' + fg_name='testing' + featuregroup_req = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + response = self.client.put("/featureGroup/{}".format(fg_name), + data=json.dumps(featuregroup_req), + content_type="application/json") + assert response.status_code == 400, "status code is not equal" + assert response.data == expected_data, response.data + + the_response2= Response() + the_response2.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + the_response2.headers={"content-type": "application/json"} + the_response2._content = b'' + mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") + feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response2) + @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name') + def test_negative_feature_group_by_name_put_api_2(self, mock1, mock2, mock3, mock4, mock5, mock6): + expected_data= b'{"Exception": "Failed to edit the feature Group "}' + fg_name='testing' + featuregroup_req = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + mock1.side_effect = [DBException("Failed to execute query in delete_feature_groupDB ERROR"), None] + response = self.client.put("/featureGroup/{}".format(fg_name), + data=json.dumps(featuregroup_req), + content_type="application/json") + assert response.data == expected_data, response.data + assert response.status_code == 200, "status code is not equal" + + def test_negative_feature_group_by_name_put_api_with_incorrect_name(self): + expected_data=b'{"Exception": "The featuregroup_name is not correct"}' + fg_name="usecase*" + response=self.client.get('/featureGroup/{}'.format(fg_name)) + assert response.status_code == 400, "status code is not equal" + assert response.data == expected_data, response.data + + # TODO: Test Code for PUT endpoint (In the case where DME is edited from enabled to disabled) + + class Test_delete_list_of_feature_group: @patch('trainingmgr.common.trainingmgr_config.TMLogger', return_value = TMLogger("tests/common/conf_log.yaml")) def setup_method(self,mock1,mock2): diff --git a/tests/test_trainingmgr_util.py b/tests/test_trainingmgr_util.py index 671f2fe8..bfe6cd85 100644 --- a/tests/test_trainingmgr_util.py +++ b/tests/test_trainingmgr_util.py @@ -37,7 +37,7 @@ from trainingmgr.common.trainingmgr_config import TrainingMgrConfig from trainingmgr.common.trainingmgr_util import response_for_training, check_key_in_dictionary,check_trainingjob_data, \ get_one_key, get_metrics, handle_async_feature_engineering_status_exception_case, get_one_word_status, check_trainingjob_data, \ - validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data + validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, get_feature_group_by_name, edit_feature_group_by_name from requests.models import Response from trainingmgr import trainingmgr_main from trainingmgr.common.tmgr_logger import TMLogger @@ -595,3 +595,181 @@ def test_negative_check_feature_group_data(self, mock1): assert False except: assert True + +class Test_get_feature_group_by_name: + fg_target = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')] + + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_target) + @patch('trainingmgr.common.trainingmgr_util.check_trainingjob_name_or_featuregroup_name', return_value=True) + def test_get_feature_group_by_name(self, mock1, mock2): + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='testing' + expected_data = {"featuregroup":[{"featuregroup_name": "testing", "features": "", "datalake": "InfluxSource", "host": "127.0.0.21", "port": "8080", "bucket": "", "token": "", "db_org": "", "measurement": "", "dme": False, "measured_obj_class": "", "dme_port": "", "source_name": ""}]} + json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name) + assert status_code == 200, "status code is not equal" + assert json_data == expected_data, json_data + + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db') + @patch('trainingmgr.common.trainingmgr_util.check_trainingjob_name_or_featuregroup_name') + def test_negative_get_feature_group_by_name(self, mock1, mock2): + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='testing' + + mock1.side_effect = [True, True] + mock2.side_effect = [None, DBException("Failed to execute query in get_feature_groupsDB ERROR")] + + # Case 1 + expected_data = {"Exception": "Failed to fetch feature group info from db"} + json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name) + assert status_code == 404, "status code is not equal" + assert json_data == expected_data, json_data + + # Case 2 + expected_data = {"Exception": "Failed to execute query in get_feature_groupsDB ERROR"} + json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name) + assert status_code == 500, "status code is not equal" + assert json_data == expected_data, json_data + + def test_negative_get_feature_group_by_name_with_incorrect_name(self): + ps_db_obj=() + logger= trainingmgr_main.LOGGER + fg_name='usecase*' + expected_data = {"Exception":"The featuregroup_name is not correct"} + json_data, status_code = get_feature_group_by_name(ps_db_obj, logger, fg_name) + assert status_code == 400, "status code is not equal" + assert json_data == expected_data, json_data + +class Test_edit_feature_group_by_name: + + fg_init = [('testing', '', 'InfluxSource', '127.0.0.21', '8080', '', '', '', '', False, '', '', '')] + + fg_edit = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', False, '', '', '')] + fg_edit_dme = [('testing', 'testing', 'InfluxSource', '127.0.0.21', '8080', 'testing', '', '', '', True, '', '31823', '')] + + # In the case where the feature group is edited while DME is disabled + feature_group_data1=('testing','testing','InfluxSource',False,'127.0.0.1', '8080', '','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data1) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + def test_edit_feature_group_by_name_1(self, mock1, mock2, mock3): + tm_conf_obj=() + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='testing' + expected_data = {"result": "Feature Group Edited"} + json_request = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request) + assert status_code == 200, "status code is not equal" + assert json_data == expected_data, json_data + + # In the case where the feature group is edited, including DME(disabled to enabled) + the_response2= Response() + the_response2.status_code = status.HTTP_201_CREATED + the_response2.headers={"content-type": "application/json"} + the_response2._content = b'' + mocked_TRAININGMGR_CONFIG_OBJ=mock.Mock(name="TRAININGMGR_CONFIG_OBJ") + feature_group_data2=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response2) + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.trainingmgr_main.TRAININGMGR_CONFIG_OBJ', return_value = mocked_TRAININGMGR_CONFIG_OBJ) + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data2) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name') + def test_edit_feature_group_by_name_2(self, mock1, mock2, mock3, mock4, mock5, mock6): + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='testing' + expected_data = {"result": "Feature Group Edited"} + json_request = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + json_data, status_code = edit_feature_group_by_name(self.mocked_TRAININGMGR_CONFIG_OBJ, ps_db_obj, logger, fg_name, json_request) + assert status_code == 200, "status code is not equal" + assert json_data == expected_data, json_data + + the_response3= Response() + the_response3.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + the_response3.headers={"content-type": "application/json"} + the_response3._content = b'' + feature_group_data3=('testing','testing','InfluxSource',True,'127.0.0.1', '8080', '31823','testing','','','','','') + @patch('trainingmgr.common.trainingmgr_util.create_dme_filtered_data_job', return_value=the_response3) + @patch('trainingmgr.common.trainingmgr_util.edit_featuregroup') + @patch('trainingmgr.common.trainingmgr_util.check_feature_group_data', return_value=feature_group_data3) + @patch('trainingmgr.common.trainingmgr_util.get_feature_group_by_name_db', return_value=fg_init) + @patch('trainingmgr.common.trainingmgr_util.delete_feature_group_by_name') + def test_negative_edit_feature_group_by_name(self, mock1, mock2, mock3, mock4, mock5): + tm_conf_obj=() + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='testing' + json_request = { + "featureGroupName": fg_name, + "feature_list": self.fg_edit[0][1], + "datalake_source": self.fg_edit[0][2], + "Host": self.fg_edit[0][3], + "Port": self.fg_edit[0][4], + "bucket": self.fg_edit[0][5], + "token": self.fg_edit[0][6], + "dbOrg": self.fg_edit[0][7], + "_measurement": self.fg_edit[0][8], + "enable_Dme": self.fg_edit[0][9], + "measured_obj_class": self.fg_edit[0][10], + "dmePort": self.fg_edit[0][11], + "source_name": self.fg_edit[0][12] + } + + # Case 1 + mock1.side_effect = [DBException("Failed to execute query in delete_feature_groupDB ERROR"), None] + expected_data={"Exception": "Failed to edit the feature Group "} + json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request) + # NOTE: This part is a test code that deliberately triggers a DBException even when DME is successfully created, so note that the status_code is 200. + assert status_code == 200, "status code is not equal" + assert json_data == expected_data, json_data + + # Case 2 + mock1.side_effect = None + expected_data={"Exception": "Cannot create dme job"} + json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request) + assert status_code == 400, "status code is not equal" + assert json_data == expected_data, json_data + + def test_negative_edit_feature_group_by_name_with_incorrect_name(self): + tm_conf_obj=() + ps_db_obj=() + logger = trainingmgr_main.LOGGER + fg_name='usecase*' + expected_data = {"Exception":"The featuregroup_name is not correct"} + json_request={} + json_data, status_code = edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, fg_name, json_request) + assert status_code == 400, "status code is not equal" + assert json_data == expected_data, json_data + + # TODO: Test Code in the case where DME is edited from enabled to disabled) \ No newline at end of file diff --git a/trainingmgr/common/trainingmgr_util.py b/trainingmgr/common/trainingmgr_util.py index f5db5b33..e350682a 100644 --- a/trainingmgr/common/trainingmgr_util.py +++ b/trainingmgr/common/trainingmgr_util.py @@ -25,9 +25,11 @@ import requests from trainingmgr.db.common_db_fun import change_in_progress_to_failed_by_latest_version, \ get_field_by_latest_version, change_field_of_latest_version, \ - get_latest_version_trainingjob_name,get_all_versions_info_by_name + get_latest_version_trainingjob_name, get_all_versions_info_by_name, get_feature_group_by_name_db, \ + add_featuregroup, edit_featuregroup, delete_feature_group_by_name from trainingmgr.constants.states import States from trainingmgr.common.exceptions_utls import APIException,TMException,DBException +from trainingmgr.common.trainingmgr_operations import create_dme_filtered_data_job ERROR_TYPE_KF_ADAPTER_JSON = "Kf adapter doesn't sends json type response" MIMETYPE_JSON = "application/json" @@ -190,6 +192,114 @@ def check_feature_group_data(json_data): return (feature_group_name, features, datalake_source, enable_dme, host, port,dme_port, bucket, token, source_name,db_org, measured_obj_class, measurement) +def get_feature_group_by_name(ps_db_obj, logger, featuregroup_name): + """ + Function fetching a feature group + + Args in function: + featuregroup_name: str + name of featuregroup_name. + Returns: + api response: dict + info of featuregroup + status code: + HTTP status code 200 + + Exceptions: + all exception are provided with exception message and HTTP status code. + """ + api_response={} + response_code=status.HTTP_500_INTERNAL_SERVER_ERROR + if not check_trainingjob_name_or_featuregroup_name(featuregroup_name): + return {"Exception":"The featuregroup_name is not correct"}, status.HTTP_400_BAD_REQUEST + logger.debug("Request for getting a feature group with name = "+ featuregroup_name) + try: + result= get_feature_group_by_name_db(ps_db_obj, featuregroup_name) + feature_group=[] + if result: + for res in result: + dict_data={ + "featuregroup_name": res[0], + "features": res[1], + "datalake": res[2], + "host": res[3], + "port": res[4], + "bucket":res[5], + "token":res[6], + "db_org":res[7], + "measurement":res[8], + "dme": res[9], + "measured_obj_class":res[10], + "dme_port":res[11], + "source_name":res[12] + } + feature_group.append(dict_data) + api_response={"featuregroup":feature_group} + response_code=status.HTTP_200_OK + else: + response_code=status.HTTP_404_NOT_FOUND + raise TMException("Failed to fetch feature group info from db") + + except Exception as err: + api_response = {"Exception": str(err)} + logger.error(str(err)) + + return api_response, response_code + +def edit_feature_group_by_name(tm_conf_obj, ps_db_obj, logger, featuregroup_name, json_data): + """ + Function fetching a feature group + + Args in function: + featuregroup_name: str + name of featuregroup_name. + json_data: dict + info of changed featuregroup_name + Returns: + api response: dict + response message + status code: + HTTP status code 200 + + Exceptions: + all exception are provided with exception message and HTTP status code. + """ + api_response= {} + response_code = status.HTTP_500_INTERNAL_SERVER_ERROR + if not check_trainingjob_name_or_featuregroup_name(featuregroup_name): + return {"Exception":"The featuregroup_name is not correct"}, status.HTTP_400_BAD_REQUEST + + logger.debug("Request for editing a feature group with name = "+ featuregroup_name) + logger.debug("db info before the edit : %s", get_feature_group_by_name_db(ps_db_obj, featuregroup_name)) + try: + (feature_group_name, features, datalake_source, enable_dme, host, port,dme_port,bucket, token, source_name,db_org, measured_obj_class, measurement)=check_feature_group_data(json_data) + # the features are stored in string format in the db, and has to be passed as list of feature to the dme. Hence the conversion. + features_list = features.split(",") + edit_featuregroup(feature_group_name, features, datalake_source , host, port, bucket, token, db_org, measurement, enable_dme, ps_db_obj, measured_obj_class, dme_port, source_name) + api_response={"result": "Feature Group Edited"} + response_code =status.HTTP_200_OK + # TODO: Implement the process where DME edits from the dashboard are applied to the endpoint + if enable_dme == True: + response= create_dme_filtered_data_job(tm_conf_obj, source_name, features_list, feature_group_name, host, dme_port, measured_obj_class) + if response.status_code != 201: + api_response={"Exception": "Cannot create dme job"} + delete_feature_group_by_name(ps_db_obj, feature_group_name) + response_code=status.HTTP_400_BAD_REQUEST + else: + api_response={"result": "Feature Group Edited"} + response_code =status.HTTP_200_OK + else: + api_response={"result": "Feature Group Edited"} + response_code =status.HTTP_200_OK + except Exception as err: + delete_feature_group_by_name(ps_db_obj, feature_group_name) + err_msg = "Failed to edit the feature Group " + api_response = {"Exception":err_msg} + logger.error(str(err)) + + logger.debug("db info after the edit : %s", get_feature_group_by_name_db(ps_db_obj, featuregroup_name)) + return api_response, response_code + def get_one_key(dictionary): ''' this function finds any one key from dictionary and return it. diff --git a/trainingmgr/db/common_db_fun.py b/trainingmgr/db/common_db_fun.py index 32776fa5..d7b37b1d 100644 --- a/trainingmgr/db/common_db_fun.py +++ b/trainingmgr/db/common_db_fun.py @@ -582,6 +582,32 @@ def add_featuregroup(feature_group_name, feature_list, datalake_source , host, p if conn is not None: conn.close() +def edit_featuregroup(feature_group_name, feature_list, datalake_source , host, port, bucket, token, db_org,_measurement, enable_dme, ps_db_obj, measured_obj_class="", dme_port="", source_name=""): + """ + This function update existing row with given information + """ + + conn = None + conn = ps_db_obj.get_new_conn() + cursor = conn.cursor() + + try: + cursor.execute('''update {} set feature_list = %s, datalake_source = %s, + host = %s, port = %s, bucket = %s, token = %s, db_org = %s, _measurement = %s, + enable_dme = %s, measured_obj_class = %s, dme_port = %s, source_name = %s + where featuregroup_name = %s'''.format(fg_table_name), + (feature_list, datalake_source, host, port, bucket, token, db_org, + _measurement, enable_dme, measured_obj_class, dme_port, source_name, feature_group_name)) + conn.commit() + cursor.close() + except Exception as err: + if conn is not None: + conn.rollback() + raise DBException(DB_QUERY_EXEC_ERROR + "update_featuregroup" + str(err)) + finally: + if conn is not None: + conn.close() + def get_feature_groups_db(ps_db_obj): """ This function returns feature_groups diff --git a/trainingmgr/trainingmgr_main.py b/trainingmgr/trainingmgr_main.py index e4081c29..a600e702 100644 --- a/trainingmgr/trainingmgr_main.py +++ b/trainingmgr/trainingmgr_main.py @@ -40,7 +40,8 @@ check_key_in_dictionary, get_one_key, \ response_for_training, get_metrics, \ handle_async_feature_engineering_status_exception_case, \ - validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, check_trainingjob_name_and_version, check_trainingjob_name_or_featuregroup_name + validate_trainingjob_name, get_all_pipeline_names_svc, check_feature_group_data, check_trainingjob_name_and_version, check_trainingjob_name_or_featuregroup_name, \ + get_feature_group_by_name, edit_feature_group_by_name from trainingmgr.common.exceptions_utls import APIException,TMException from trainingmgr.constants.steps import Steps from trainingmgr.constants.states import States @@ -50,7 +51,7 @@ change_in_progress_to_failed_by_latest_version, change_steps_state_of_latest_version, \ get_info_by_version, \ get_trainingjob_info_by_name, get_latest_version_trainingjob_name, get_all_versions_info_by_name, \ - update_model_download_url, add_update_trainingjob, add_featuregroup, \ + update_model_download_url, add_update_trainingjob, add_featuregroup, edit_featuregroup, \ get_field_of_given_version,get_all_jobs_latest_status_version, get_info_of_latest_version, \ get_feature_groups_db, get_feature_group_by_name_db, delete_feature_group_by_name, delete_trainingjob_version, change_field_value_by_version @@ -1318,6 +1319,85 @@ def get_metadata(trainingjob_name): status=response_code, mimetype=MIMETYPE_JSON) +@APP.route('/featureGroup/', methods=['GET', 'PUT']) +def feature_group_by_name(featuregroup_name): + """ + Rest endpoint to get or update feature group + Precondtion for update : not really necessary. + + Args in function: + featuregroup_name: str + name of featuregroup_name. + + Args in json: + if get/put request is called + json with below fields are given: + featureGroupName: str + description + feature_list: str + feature names + datalake: str + name of datalake + bucket: str + bucket name + host: str + db host + port: str + db port + token: str + token for the bucket + db org: str + db org name + measurement: str + measurement of the influxdb + enable_Dme: boolean + whether to enable dme + source_name: str + name of source + DmePort: str + DME port + measured_obj_class: str + obj class for dme. + datalake_source: str + string indicating datalake source + + Returns: + 1. For get request + json: + api response : str + response message + status code: + HTTP status code 200 + 2. For put request + json: + api response : str + response message + status code: + HTTP status code 200 + + Exceptions: + All exception are provided with exception message and HTTP status code. + The individual exceptions for put and get are handled within each internal function + """ + api_response = {} + response_code = status.HTTP_500_INTERNAL_SERVER_ERROR + LOGGER.debug("Feature Group read/update request(featuregroup name) %s", featuregroup_name) + + try: + if (request.method == 'GET'): + api_response, response_code = get_feature_group_by_name(PS_DB_OBJ, LOGGER, featuregroup_name) + elif (request.method == 'PUT'): + json_data=request.json + api_response, response_code = edit_feature_group_by_name(TRAININGMGR_CONFIG_OBJ, PS_DB_OBJ, LOGGER, featuregroup_name, json_data) + + except Exception as err: + LOGGER.error("Failed to read/update featuregroup, " + str(err) ) + api_response = {"Exception": str(err)} + + return APP.response_class(response= json.dumps(api_response), + status= response_code, + mimetype=MIMETYPE_JSON) + @APP.route('/featureGroup', methods=['POST']) def create_feature_group(): """ @@ -1464,90 +1544,6 @@ def get_feature_group(): status=response_code, mimetype=MIMETYPE_JSON) -@APP.route('/featureGroup/', methods=['GET']) -def get_feature_group_by_name(featuregroup_name): - """ - Rest endpoint to fetch a feature group - - Args in function: - featuregroup_name: str - name of featuregroup_name. - - Returns: - json: - trainingjob: dict - dictionary contains - featuregroup_name: str - name of featuregroup - features: str - features - datalake: str - name of datalake - host: str - db host - port: str - db port - bucket: str - bucket name - token: str - token for the bucket - db_org: str - db org - measurement: str - measurement of the influxdb - dme: str - whether dme enabled or not - measured_obj_class: str - obj class for dme - dme_port: str - dme_port - source_name: dict - source name - status code: - HTTP status code 200 - - Exceptions: - all exception are provided with exception message and HTTP status code. - - """ - api_response={} - response_code=status.HTTP_500_INTERNAL_SERVER_ERROR - if not check_trainingjob_name_or_featuregroup_name(featuregroup_name): - return {"Exception":"The trainingjob_name is not correct"}, status.HTTP_400_BAD_REQUEST - LOGGER.debug("Request for getting a feature group with name = "+ featuregroup_name) - try: - result= get_feature_group_by_name_db(PS_DB_OBJ, featuregroup_name) - feature_group=[] - if result: - for res in result: - dict_data={ - "featuregroup_name": res[0], - "features": res[1], - "datalake": res[2], - "host": res[3], - "port": res[4], - "bucket":res[5], - "token":res[6], - "db_org":res[7], - "measurement":res[8], - "dme": res[9], - "measured_obj_class":res[10], - "dme_port":res[11], - "source_name":res[12] - } - feature_group.append(dict_data) - api_response={"featuregroup":feature_group} - response_code=status.HTTP_200_OK - else: - response_code=status.HTTP_404_NOT_FOUND - raise TMException("Failed to fetch feature group info from db") - except Exception as err: - api_response = {"Exception": str(err)} - LOGGER.error(str(err)) - return APP.response_class(response=json.dumps(api_response), - status=response_code, - mimetype=MIMETYPE_JSON) - @APP.route('/featureGroup', methods=['DELETE']) def delete_list_of_feature_group(): """