From 1095e6eef2874eb6044606b826598b83c5b53df3 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 2 Aug 2019 17:14:47 -0400 Subject: [PATCH 01/26] Added canonical_facts to delete method and HostEvent class --- api/host.py | 2 +- app/events.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/api/host.py b/api/host.py index 2a4606642..0105f793b 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id) + host_ids_to_delete.append(host.id, host.canonical_facts) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index 7279b6b9b..c4c7cf378 100644 --- a/app/events.py +++ b/app/events.py @@ -1,8 +1,9 @@ import logging from datetime import datetime - from marshmallow import fields from marshmallow import Schema +from marshmallow import validate +from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -11,7 +12,18 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() + insights_id = fields.Str(validate=verify_uuid_format) + rhel_machine_id = fields.Str(validate=verify_uuid_format) + subscription_manager_id = fields.Str(validate=verify_uuid_format) + satellite_id = fields.Str(validate=verify_uuid_format) + fqdn = fields.Str(validate=validate.Length(min=1, max=255)) + bios_uuid = fields.Str(validate=verify_uuid_format) + ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + external_id = fields.Str(validate=validate.Length(min=1, max=500)) def delete(id): - return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete"}).data + return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete", "insights_id": id.canonical_facts.insights_id, "rhel_machine_id": id.canonical_facts.rhel_machine_id, + "subscription_manager_id": id.canonical_facts.subscription_manager_id, "satellite_id": id.canonical_facts.satellite_id, + "fqdn": id.canonical_facts.fqdn, "bios_uuid": id.canonical_facts.bios_uuid, "external_id": id.canonical_facts.external_id}).data \ No newline at end of file From 79c48c5d646abf7acf90089c39f2368628d71bae Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 5 Aug 2019 10:11:06 -0400 Subject: [PATCH 02/26] Small Changes to a few files for local development --- api/host.py | 2 +- hack_client.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ run.py | 2 +- 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 hack_client.py diff --git a/api/host.py b/api/host.py index 0105f793b..752863bc6 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id, host.canonical_facts) + host_ids_to_delete.append(host.id, **host.canonical_facts) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/hack_client.py b/hack_client.py new file mode 100644 index 000000000..a3050bb2a --- /dev/null +++ b/hack_client.py @@ -0,0 +1,80 @@ +import base64 +import random +import json +import requests +import uuid + +#URL = "http://localhost:8080/r/insights/platform/inventory/api/v1/hosts" +URL = "http://localhost:8080/api/inventory/v1/hosts" + +#account_number = "0000101" +account_number = "0000001" + +domainnames = ["domain1", "domain2", "domain3", "domain4", "domain5"] +hostnames1 = ["apple", "pear", "orange", "banana", "apricot", "grape"] +hostnames2 = ["coke", "pepsi", "drpepper", "mrpib", "sprite", "7up", "unsweettea", "sweettea"] + +chunk_size = 3 +bulk = False +#bulk = True + +#insights_id = "4225c384324849ea9fa80e481c078a4d" +#insights_id = None +insights_id = "4225c384-3248-49ea-9fa8-0e481c078a4d" + +def build_chunk(): + list_of_hosts = [] + for i in range(chunk_size): + fqdn = random.choice(hostnames1) + "_" + random.choice(hostnames2) + "." + random.choice(domainnames) + ".com" + #fqdn = "fred.flintstone.com" + + if len(list_of_hosts) == 0: + account_number = "00001" + else: + account_number = "00002" + + payload = { + "account": account_number, + #"insights_id": str(uuid.uuid4()), + "insights_id": insights_id, + "fqdn": fqdn, + "display_name": fqdn, + #"ip_addresses": None, + #"ip_addresses": ["1",], + #"mac_addresses": None, + #"subscription_manager_id": "214", + } + + list_of_hosts.append(payload) + + return list_of_hosts + + +payload = build_chunk() + +headers = {'Content-type': 'application/json', + 'x-rh-insights-request-id': '654321', + 'account_number': account_number, + } + +print("Bulk Upload:") +print("payload:", payload) + +#headers["Authorization"] = "Bearer secret" +#headers["Authorization"] = "Bearer 023e4f11185e4c17478bb9c6750d3068eeebe85b" +identity = {'identity': {'account_number': account_number}} +headers["x-rh-identity"] = base64.b64encode(json.dumps(identity).encode()) + +#payload[0]["insights_id"] = "730b8126-3c85-446d-8f85-493bfdaf34d0" + +#del payload[1]["insights_id"] +#del payload[1]["fqdn"] + +json_payload = json.dumps(payload) + +#print(json_payload) + +r = requests.post(URL, data=json_payload, headers=headers) +print("response:", r.text) +print("status_code", r.status_code) +print("test:", r.headers) \ No newline at end of file diff --git a/run.py b/run.py index c239bcc52..824cf75a5 100755 --- a/run.py +++ b/run.py @@ -3,7 +3,7 @@ from app import create_app -config_name = os.getenv("APP_SETTINGS", "development") +config_name = os.getenv("APP_SETTINGS", "testing") application = create_app(config_name) listen_port = os.getenv("LISTEN_PORT", 8080) From 610227ff1c4fd1b777b651ee30a4c6313cfbdc78 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 5 Aug 2019 12:42:00 -0400 Subject: [PATCH 03/26] changed hack_client.py to not upload multiple chunks --- hack_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hack_client.py b/hack_client.py index a3050bb2a..78dc62d1d 100644 --- a/hack_client.py +++ b/hack_client.py @@ -14,7 +14,7 @@ hostnames1 = ["apple", "pear", "orange", "banana", "apricot", "grape"] hostnames2 = ["coke", "pepsi", "drpepper", "mrpib", "sprite", "7up", "unsweettea", "sweettea"] -chunk_size = 3 +chunk_size = 1 bulk = False #bulk = True @@ -27,11 +27,12 @@ def build_chunk(): for i in range(chunk_size): fqdn = random.choice(hostnames1) + "_" + random.choice(hostnames2) + "." + random.choice(domainnames) + ".com" #fqdn = "fred.flintstone.com" - + ''' if len(list_of_hosts) == 0: - account_number = "00001" + account_number = "000001" else: - account_number = "00002" + account_number = "000002" + ''' payload = { "account": account_number, From 5038067a754f7de45746baaeec273b0188b6ae7a Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 5 Aug 2019 14:16:28 -0400 Subject: [PATCH 04/26] More small changes to try and setup the local dev enviroment --- api/host.py | 2 +- app/events.py | 6 +++-- hack_client.py => utils/hack_client.py | 0 utils/kafka_consumer.py | 37 ++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) rename hack_client.py => utils/hack_client.py (100%) create mode 100644 utils/kafka_consumer.py diff --git a/api/host.py b/api/host.py index 752863bc6..7010dc167 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id, **host.canonical_facts) + host_ids_to_delete.append(host.id) # **host.canonical_facts except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index c4c7cf378..464d990ff 100644 --- a/app/events.py +++ b/app/events.py @@ -12,6 +12,7 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() + ''' insights_id = fields.Str(validate=verify_uuid_format) rhel_machine_id = fields.Str(validate=verify_uuid_format) subscription_manager_id = fields.Str(validate=verify_uuid_format) @@ -21,9 +22,10 @@ class HostEvent(Schema): ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) external_id = fields.Str(validate=validate.Length(min=1, max=500)) + ''' def delete(id): - return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete", "insights_id": id.canonical_facts.insights_id, "rhel_machine_id": id.canonical_facts.rhel_machine_id, + return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete"}).data, ''' "insights_id": id.canonical_facts.insights_id, "rhel_machine_id": id.canonical_facts.rhel_machine_id, "subscription_manager_id": id.canonical_facts.subscription_manager_id, "satellite_id": id.canonical_facts.satellite_id, - "fqdn": id.canonical_facts.fqdn, "bios_uuid": id.canonical_facts.bios_uuid, "external_id": id.canonical_facts.external_id}).data \ No newline at end of file + "fqdn": id.canonical_facts.fqdn, "bios_uuid": id.canonical_facts.bios_uuid, "external_id": id.canonical_facts.external_id}).data ''' \ No newline at end of file diff --git a/hack_client.py b/utils/hack_client.py similarity index 100% rename from hack_client.py rename to utils/hack_client.py diff --git a/utils/kafka_consumer.py b/utils/kafka_consumer.py new file mode 100644 index 000000000..d98d968ff --- /dev/null +++ b/utils/kafka_consumer.py @@ -0,0 +1,37 @@ +import json +import logging +import os + +from kafka import KafkaConsumer + +# from app.models import Host, SystemProfileSchema + + +TOPIC = os.environ.get("KAFKA_TOPIC", "platform.inventory.events") +KAFKA_GROUP = os.environ.get("KAFKA_GROUP", "inventory") +BOOTSTRAP_SERVERS = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "kafka:29092") + +api_version = (0,10) + +def msg_handler(parsed): + print("inside msg_handler()") + print("type(parsed):", type(parsed)) + print("parsed:", parsed) + # id_ = parsed["id"] + # profile = SystemProfileSchema(strict=True).load(parsed["system_profile"]) + # host = Host.query.get(id_) + # host._update_system_profile(profile) + # host.save() + + +consumer = KafkaConsumer(TOPIC, group_id=KAFKA_GROUP, bootstrap_servers=BOOTSTRAP_SERVERS) + +logging.basicConfig(level=logging.INFO) + +print("TOPIC:", TOPIC) +print("KAFKA_GROUP:", KAFKA_GROUP) +print("BOOTSTRAP_SERVERS:", BOOTSTRAP_SERVERS) + +for msg in consumer: + print("calling msg_handler()") + msg_handler(json.loads(msg.value)) \ No newline at end of file From b0d8808f9d37323315956193ff7378f9227be21e Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 7 Aug 2019 13:07:04 -0400 Subject: [PATCH 05/26] added insights_id to delete event and all canonical facts to HostEvent class --- api/host.py | 2 +- app/events.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/api/host.py b/api/host.py index 7010dc167..7a8ab820c 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id) # **host.canonical_facts + host_ids_to_delete.append([host.id, host.canonical_facts]) # **host.canonical_facts except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index 464d990ff..cb5548c58 100644 --- a/app/events.py +++ b/app/events.py @@ -12,7 +12,6 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - ''' insights_id = fields.Str(validate=verify_uuid_format) rhel_machine_id = fields.Str(validate=verify_uuid_format) subscription_manager_id = fields.Str(validate=verify_uuid_format) @@ -22,10 +21,7 @@ class HostEvent(Schema): ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) external_id = fields.Str(validate=validate.Length(min=1, max=500)) - ''' def delete(id): - return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete"}).data, ''' "insights_id": id.canonical_facts.insights_id, "rhel_machine_id": id.canonical_facts.rhel_machine_id, - "subscription_manager_id": id.canonical_facts.subscription_manager_id, "satellite_id": id.canonical_facts.satellite_id, - "fqdn": id.canonical_facts.fqdn, "bios_uuid": id.canonical_facts.bios_uuid, "external_id": id.canonical_facts.external_id}).data ''' \ No newline at end of file + return HostEvent(strict=True).dumps({"id": id[0], "insights_id": id[1]['insights_id'],"timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file From 4c950793022d34c22bfdf20aecdc17d22a865108 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 7 Aug 2019 13:10:17 -0400 Subject: [PATCH 06/26] deleted unecessary comments --- api/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/host.py b/api/host.py index 7a8ab820c..b50ebad9c 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append([host.id, host.canonical_facts]) # **host.canonical_facts + host_ids_to_delete.append([host.id, host.canonical_facts]) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " From c046158e929a99f500fb31728e3d327e6e040cb6 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 7 Aug 2019 13:12:56 -0400 Subject: [PATCH 07/26] removed utils folder containing unecessary tests --- utils/hack_client.py | 81 ----------------------------------------- utils/kafka_consumer.py | 37 ------------------- 2 files changed, 118 deletions(-) delete mode 100644 utils/hack_client.py delete mode 100644 utils/kafka_consumer.py diff --git a/utils/hack_client.py b/utils/hack_client.py deleted file mode 100644 index 78dc62d1d..000000000 --- a/utils/hack_client.py +++ /dev/null @@ -1,81 +0,0 @@ -import base64 -import random -import json -import requests -import uuid - -#URL = "http://localhost:8080/r/insights/platform/inventory/api/v1/hosts" -URL = "http://localhost:8080/api/inventory/v1/hosts" - -#account_number = "0000101" -account_number = "0000001" - -domainnames = ["domain1", "domain2", "domain3", "domain4", "domain5"] -hostnames1 = ["apple", "pear", "orange", "banana", "apricot", "grape"] -hostnames2 = ["coke", "pepsi", "drpepper", "mrpib", "sprite", "7up", "unsweettea", "sweettea"] - -chunk_size = 1 -bulk = False -#bulk = True - -#insights_id = "4225c384324849ea9fa80e481c078a4d" -#insights_id = None -insights_id = "4225c384-3248-49ea-9fa8-0e481c078a4d" - -def build_chunk(): - list_of_hosts = [] - for i in range(chunk_size): - fqdn = random.choice(hostnames1) + "_" + random.choice(hostnames2) + "." + random.choice(domainnames) + ".com" - #fqdn = "fred.flintstone.com" - ''' - if len(list_of_hosts) == 0: - account_number = "000001" - else: - account_number = "000002" - ''' - - payload = { - "account": account_number, - #"insights_id": str(uuid.uuid4()), - "insights_id": insights_id, - "fqdn": fqdn, - "display_name": fqdn, - #"ip_addresses": None, - #"ip_addresses": ["1",], - #"mac_addresses": None, - #"subscription_manager_id": "214", - } - - list_of_hosts.append(payload) - - return list_of_hosts - - -payload = build_chunk() - -headers = {'Content-type': 'application/json', - 'x-rh-insights-request-id': '654321', - 'account_number': account_number, - } - -print("Bulk Upload:") -print("payload:", payload) - -#headers["Authorization"] = "Bearer secret" -#headers["Authorization"] = "Bearer 023e4f11185e4c17478bb9c6750d3068eeebe85b" -identity = {'identity': {'account_number': account_number}} -headers["x-rh-identity"] = base64.b64encode(json.dumps(identity).encode()) - -#payload[0]["insights_id"] = "730b8126-3c85-446d-8f85-493bfdaf34d0" - -#del payload[1]["insights_id"] -#del payload[1]["fqdn"] - -json_payload = json.dumps(payload) - -#print(json_payload) - -r = requests.post(URL, data=json_payload, headers=headers) -print("response:", r.text) -print("status_code", r.status_code) -print("test:", r.headers) \ No newline at end of file diff --git a/utils/kafka_consumer.py b/utils/kafka_consumer.py deleted file mode 100644 index d98d968ff..000000000 --- a/utils/kafka_consumer.py +++ /dev/null @@ -1,37 +0,0 @@ -import json -import logging -import os - -from kafka import KafkaConsumer - -# from app.models import Host, SystemProfileSchema - - -TOPIC = os.environ.get("KAFKA_TOPIC", "platform.inventory.events") -KAFKA_GROUP = os.environ.get("KAFKA_GROUP", "inventory") -BOOTSTRAP_SERVERS = os.environ.get("KAFKA_BOOTSTRAP_SERVERS", "kafka:29092") - -api_version = (0,10) - -def msg_handler(parsed): - print("inside msg_handler()") - print("type(parsed):", type(parsed)) - print("parsed:", parsed) - # id_ = parsed["id"] - # profile = SystemProfileSchema(strict=True).load(parsed["system_profile"]) - # host = Host.query.get(id_) - # host._update_system_profile(profile) - # host.save() - - -consumer = KafkaConsumer(TOPIC, group_id=KAFKA_GROUP, bootstrap_servers=BOOTSTRAP_SERVERS) - -logging.basicConfig(level=logging.INFO) - -print("TOPIC:", TOPIC) -print("KAFKA_GROUP:", KAFKA_GROUP) -print("BOOTSTRAP_SERVERS:", BOOTSTRAP_SERVERS) - -for msg in consumer: - print("calling msg_handler()") - msg_handler(json.loads(msg.value)) \ No newline at end of file From 08a275ca9f2c8f8ee06c15935ba2c618ba2f0c1e Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 7 Aug 2019 13:29:59 -0400 Subject: [PATCH 08/26] Added instructions about adding kafka to hosts file to README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index af4783ecf..c9f234a62 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,14 @@ _prometheus_multiproc_dir_ environment variable. This is done automatically. python run_gunicorn.py ``` +Should issues occur with the server's connection to the docker service, it might +be necessary to add the following to your hosts file: + +``` +kafka 127.0.0.1 +``` + + ## Configuration environment variables From addc78e52a406ac84d628dfc4242c4a94fdc6af3 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 8 Aug 2019 11:26:28 -0400 Subject: [PATCH 09/26] Changed original host_id_to_delete.append() into a dictionary & added all canonical facts to delete event --- api/host.py | 2 +- app/events.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/host.py b/api/host.py index b50ebad9c..3797db846 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append([host.id, host.canonical_facts]) + host_ids_to_delete.append({"id": host.id, **host.canonical_facts}) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index cb5548c58..f6f45bb7e 100644 --- a/app/events.py +++ b/app/events.py @@ -23,5 +23,9 @@ class HostEvent(Schema): external_id = fields.Str(validate=validate.Length(min=1, max=500)) -def delete(id): - return HostEvent(strict=True).dumps({"id": id[0], "insights_id": id[1]['insights_id'],"timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file +def delete(host): + return HostEvent(strict=True).dumps({"id": host['id'], "insights_id": host['insights_id'], "rhel_machine_id": host['rhel_machine_id'], + "subscription_manager_id": host['subscription_manager_id'], "satellite_id": host['satellite_id'], + "fqdn": host['fqdn'], "bios_uuid": host['bios_uuid'], "ip_addresses": host['ip_addresses'], + "mac_addresses:": host['mac_addresses'], "external_id": host['external_id'], + "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file From 40f6db1118c7d37ddfe70161469e5ceadf32d20f Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 8 Aug 2019 11:47:46 -0400 Subject: [PATCH 10/26] added additional conditions to DeleteHostsTestCase to test changes to delete_by_id method --- test_api.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test_api.py b/test_api.py index ab8d914e4..6e6a6b8f9 100755 --- a/test_api.py +++ b/test_api.py @@ -1110,7 +1110,14 @@ def create_hosts(self): host_wrapper.account = ACCOUNT host_wrapper.display_name = host[0] host_wrapper.insights_id = host[1] + host_wrapper.rhel_machine_id = host[1] + host_wrapper.subscription_manager_id = host[1] + host_wrapper.satellite_id = host[1] + host_wrapper.bios_uuid = host[1] + host_wrapper.ip_addresses = ["10.0.0.2"] host_wrapper.fqdn = host[2] + host_wrapper.mac_addresses = ["c2:00:d0:c8:61:01"] + host_wrapper.external_id = host[1] host_wrapper.facts = [{"namespace": "ns1", "facts": {"key1": "value1"}}] response_data = self.post(HOST_URL, [host_wrapper.data()], 207) @@ -1210,6 +1217,15 @@ def test_invalid_host_id(self): class DeleteHostsTestCase(PreCreatedHostsBaseTestCase): def test_create_then_delete(self): original_id = self.added_hosts[0].id + insights_id = self.added_hosts[0].insights_id + rhel_machine_id = self.added_hosts[0].rhel_machine_id + subscription_manager_id = self.added_hosts[0].subscription_manager_id + satellite_id = self.added_hosts[0].satellite_id + bios_uuid = self.added_hosts[0].bios_uuid + ip_addresses = self.added_hosts[0].ip_addresses + fqdn = self.added_hosts[0].fqdn + mac_addresses = self.added_hosts[0].mac_addresses + external_id = self.added_hosts[0].external_id url = HOST_URL + "/" + original_id @@ -1227,6 +1243,15 @@ def __call__(self, e): with unittest.mock.patch("api.host.emit_event", new=MockEmitEvent()) as m: self.delete(url, 200, return_response_as_json=False) assert original_id in m.events[0] + assert insights_id in m.events[0] + assert rhel_machine_id in m.events[0] + assert subscription_manager_id in m.events[0] + assert satellite_id in m.events[0] + assert bios_uuid in m.events[0] + assert ip_addresses in m.events[0] + assert fqdn in m.events[0] + assert mac_addresses in m.events[0] + assert external_id in m.events[0] # Try to get the host again response = self.get(url, 200) From e6de4f2f5a9092edbf2b31f8b6ba1522aead211d Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 8 Aug 2019 14:52:23 -0400 Subject: [PATCH 11/26] Fine tuned tests to work with mac_addresses and ip_addresses --- api/host.py | 2 +- app/events.py | 20 +++++++++++++++++--- test_api.py | 6 +++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/api/host.py b/api/host.py index 2a4606642..3797db846 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id) + host_ids_to_delete.append({"id": host.id, **host.canonical_facts}) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index 7279b6b9b..f6f45bb7e 100644 --- a/app/events.py +++ b/app/events.py @@ -1,8 +1,9 @@ import logging from datetime import datetime - from marshmallow import fields from marshmallow import Schema +from marshmallow import validate +from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -11,7 +12,20 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() + insights_id = fields.Str(validate=verify_uuid_format) + rhel_machine_id = fields.Str(validate=verify_uuid_format) + subscription_manager_id = fields.Str(validate=verify_uuid_format) + satellite_id = fields.Str(validate=verify_uuid_format) + fqdn = fields.Str(validate=validate.Length(min=1, max=255)) + bios_uuid = fields.Str(validate=verify_uuid_format) + ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + external_id = fields.Str(validate=validate.Length(min=1, max=500)) -def delete(id): - return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete"}).data +def delete(host): + return HostEvent(strict=True).dumps({"id": host['id'], "insights_id": host['insights_id'], "rhel_machine_id": host['rhel_machine_id'], + "subscription_manager_id": host['subscription_manager_id'], "satellite_id": host['satellite_id'], + "fqdn": host['fqdn'], "bios_uuid": host['bios_uuid'], "ip_addresses": host['ip_addresses'], + "mac_addresses:": host['mac_addresses'], "external_id": host['external_id'], + "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file diff --git a/test_api.py b/test_api.py index 6e6a6b8f9..aa29bd4d6 100755 --- a/test_api.py +++ b/test_api.py @@ -1116,7 +1116,7 @@ def create_hosts(self): host_wrapper.bios_uuid = host[1] host_wrapper.ip_addresses = ["10.0.0.2"] host_wrapper.fqdn = host[2] - host_wrapper.mac_addresses = ["c2:00:d0:c8:61:01"] + host_wrapper.mac_addresses = ["c2:00:d0:c8:61:01", "Bananna"] host_wrapper.external_id = host[1] host_wrapper.facts = [{"namespace": "ns1", "facts": {"key1": "value1"}}] @@ -1222,9 +1222,9 @@ def test_create_then_delete(self): subscription_manager_id = self.added_hosts[0].subscription_manager_id satellite_id = self.added_hosts[0].satellite_id bios_uuid = self.added_hosts[0].bios_uuid - ip_addresses = self.added_hosts[0].ip_addresses + ip_addresses = self.added_hosts[0].ip_addresses[0] + mac_addresses = self.added_hosts[0].mac_addresses[0] fqdn = self.added_hosts[0].fqdn - mac_addresses = self.added_hosts[0].mac_addresses external_id = self.added_hosts[0].external_id url = HOST_URL + "/" + original_id From a789023e26b5491ceae5ebab72fd61cc35214915 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 9 Aug 2019 08:43:56 -0400 Subject: [PATCH 12/26] added account to delete method to help with callback to legacy api --- api/host.py | 2 +- app/events.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/host.py b/api/host.py index 3797db846..fe2184841 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append({"id": host.id, **host.canonical_facts}) + host_ids_to_delete.append({"id": host.id, **host.canonical_facts, "account": host.account}) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index f6f45bb7e..8fca6e5ff 100644 --- a/app/events.py +++ b/app/events.py @@ -12,6 +12,7 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() + account = fields.Str(validate=validate.Length(min=1, max=255)) insights_id = fields.Str(validate=verify_uuid_format) rhel_machine_id = fields.Str(validate=verify_uuid_format) subscription_manager_id = fields.Str(validate=verify_uuid_format) @@ -28,4 +29,4 @@ def delete(host): "subscription_manager_id": host['subscription_manager_id'], "satellite_id": host['satellite_id'], "fqdn": host['fqdn'], "bios_uuid": host['bios_uuid'], "ip_addresses": host['ip_addresses'], "mac_addresses:": host['mac_addresses'], "external_id": host['external_id'], - "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file + "account": host['account'], "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file From d6f84761a2ab8aa1f58640f03b03314f4ee03a1e Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 12 Aug 2019 08:58:46 -0400 Subject: [PATCH 13/26] tested changes with mock data and removed mock data from events.py --- api/host.py | 2 +- app/events.py | 15 +-------------- test_api.py | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/api/host.py b/api/host.py index 3797db846..2a4606642 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): host_ids_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append({"id": host.id, **host.canonical_facts}) + host_ids_to_delete.append(host.id) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index f6f45bb7e..84cdae017 100644 --- a/app/events.py +++ b/app/events.py @@ -12,20 +12,7 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - insights_id = fields.Str(validate=verify_uuid_format) - rhel_machine_id = fields.Str(validate=verify_uuid_format) - subscription_manager_id = fields.Str(validate=verify_uuid_format) - satellite_id = fields.Str(validate=verify_uuid_format) - fqdn = fields.Str(validate=validate.Length(min=1, max=255)) - bios_uuid = fields.Str(validate=verify_uuid_format) - ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - external_id = fields.Str(validate=validate.Length(min=1, max=500)) def delete(host): - return HostEvent(strict=True).dumps({"id": host['id'], "insights_id": host['insights_id'], "rhel_machine_id": host['rhel_machine_id'], - "subscription_manager_id": host['subscription_manager_id'], "satellite_id": host['satellite_id'], - "fqdn": host['fqdn'], "bios_uuid": host['bios_uuid'], "ip_addresses": host['ip_addresses'], - "mac_addresses:": host['mac_addresses'], "external_id": host['external_id'], - "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file + return HostEvent(strict=True).dumps({"id": host['id'], "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file diff --git a/test_api.py b/test_api.py index aa29bd4d6..754e03b3c 100755 --- a/test_api.py +++ b/test_api.py @@ -1116,7 +1116,7 @@ def create_hosts(self): host_wrapper.bios_uuid = host[1] host_wrapper.ip_addresses = ["10.0.0.2"] host_wrapper.fqdn = host[2] - host_wrapper.mac_addresses = ["c2:00:d0:c8:61:01", "Bananna"] + host_wrapper.mac_addresses = ["apple"] host_wrapper.external_id = host[1] host_wrapper.facts = [{"namespace": "ns1", "facts": {"key1": "value1"}}] @@ -1222,10 +1222,11 @@ def test_create_then_delete(self): subscription_manager_id = self.added_hosts[0].subscription_manager_id satellite_id = self.added_hosts[0].satellite_id bios_uuid = self.added_hosts[0].bios_uuid - ip_addresses = self.added_hosts[0].ip_addresses[0] - mac_addresses = self.added_hosts[0].mac_addresses[0] + ip_addresses = self.added_hosts[0].ip_addresses + mac_addresses = self.added_hosts[0].mac_addresses fqdn = self.added_hosts[0].fqdn external_id = self.added_hosts[0].external_id + account = self.added_hosts[0].account url = HOST_URL + "/" + original_id @@ -1248,11 +1249,16 @@ def __call__(self, e): assert subscription_manager_id in m.events[0] assert satellite_id in m.events[0] assert bios_uuid in m.events[0] - assert ip_addresses in m.events[0] assert fqdn in m.events[0] - assert mac_addresses in m.events[0] assert external_id in m.events[0] - + assert account in m.events[0] + + for ip in ip_addresses: + assert ip in m.events[0] + + for mac in mac_addresses: + assert mac in m.events[0] + # Try to get the host again response = self.get(url, 200) From 8a768975b6c6ebda0698eb09c39348813229b454 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Tue, 13 Aug 2019 14:29:56 -0400 Subject: [PATCH 14/26] made suggested changes, used serialization method for not defined canonical facts --- README.md | 9 --------- api/host.py | 12 ++++++------ app/events.py | 6 +----- run.py | 2 +- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c9f234a62..b824d6f5f 100644 --- a/README.md +++ b/README.md @@ -92,15 +92,6 @@ _prometheus_multiproc_dir_ environment variable. This is done automatically. python run_gunicorn.py ``` -Should issues occur with the server's connection to the docker service, it might -be necessary to add the following to your hosts file: - -``` -kafka 127.0.0.1 -``` - - - ## Configuration environment variables ``` diff --git a/api/host.py b/api/host.py index fe2184841..5e0259e02 100644 --- a/api/host.py +++ b/api/host.py @@ -283,28 +283,28 @@ def find_hosts_by_hostname_or_id(account_number, hostname): def delete_by_id(host_id_list): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) - host_ids_to_delete = [] + hosts_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append({"id": host.id, **host.canonical_facts, "account": host.account}) + hosts_to_delete.append(host.to_json()) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " "already deleted." ) - if not host_ids_to_delete: + if not hosts_to_delete: return flask.abort(status.HTTP_404_NOT_FOUND) with metrics.delete_host_processing_time.time(): query.delete(synchronize_session="fetch") db.session.commit() - metrics.delete_host_count.inc(len(host_ids_to_delete)) + metrics.delete_host_count.inc(len(hosts_to_delete)) - logger.debug("Deleted hosts: %s", host_ids_to_delete) + logger.debug("Deleted hosts: %s", hosts_to_delete) - for deleted_host_id in host_ids_to_delete: + for deleted_host_id in hosts_to_delete: emit_event(events.delete(deleted_host_id)) diff --git a/app/events.py b/app/events.py index 8fca6e5ff..5bc535c3c 100644 --- a/app/events.py +++ b/app/events.py @@ -25,8 +25,4 @@ class HostEvent(Schema): def delete(host): - return HostEvent(strict=True).dumps({"id": host['id'], "insights_id": host['insights_id'], "rhel_machine_id": host['rhel_machine_id'], - "subscription_manager_id": host['subscription_manager_id'], "satellite_id": host['satellite_id'], - "fqdn": host['fqdn'], "bios_uuid": host['bios_uuid'], "ip_addresses": host['ip_addresses'], - "mac_addresses:": host['mac_addresses'], "external_id": host['external_id'], - "account": host['account'], "timestamp": datetime.utcnow(), "type": "delete"}).data \ No newline at end of file + return HostEvent().dumps({"timestamp": datetime.utcnow(), **host, "type": "delete"}).data \ No newline at end of file diff --git a/run.py b/run.py index 824cf75a5..c239bcc52 100755 --- a/run.py +++ b/run.py @@ -3,7 +3,7 @@ from app import create_app -config_name = os.getenv("APP_SETTINGS", "testing") +config_name = os.getenv("APP_SETTINGS", "development") application = create_app(config_name) listen_port = os.getenv("LISTEN_PORT", 8080) From 4612bf4173aa22652cd5d73dd4dfe887a9d9cfb2 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 14 Aug 2019 09:21:38 -0400 Subject: [PATCH 15/26] ran changes through pre-commit screening & implemented suggestions --- api/host.py | 2 +- app/events.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api/host.py b/api/host.py index 5e0259e02..34d114f17 100644 --- a/api/host.py +++ b/api/host.py @@ -286,7 +286,7 @@ def delete_by_id(host_id_list): hosts_to_delete = [] for host in query.all(): try: - hosts_to_delete.append(host.to_json()) + hosts_to_delete.append(host) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " diff --git a/app/events.py b/app/events.py index 5bc535c3c..7c54d838c 100644 --- a/app/events.py +++ b/app/events.py @@ -1,8 +1,11 @@ import logging from datetime import datetime + from marshmallow import fields from marshmallow import Schema from marshmallow import validate + +from app.models import CanonicalFacts from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -25,4 +28,15 @@ class HostEvent(Schema): def delete(host): - return HostEvent().dumps({"timestamp": datetime.utcnow(), **host, "type": "delete"}).data \ No newline at end of file + return ( + HostEvent() + .dumps( + { + "timestamp": datetime.utcnow(), + **CanonicalFacts.to_json(host.canonical_facts), + "account": host.account, + "type": "delete", + } + ) + .data + ) From a698d7f5cf771d31f2550cc604556c93ebdc26ab Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 14 Aug 2019 13:46:49 -0400 Subject: [PATCH 16/26] applied suggestions made, added test for timestamp and type, generated unique UUIDs for each variable that needed it --- test_api.py | 80 +++++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/test_api.py b/test_api.py index 754e03b3c..9164062c3 100755 --- a/test_api.py +++ b/test_api.py @@ -1107,17 +1107,18 @@ def create_hosts(self): for host in hosts_to_create: host_wrapper = HostWrapper() + host_wrapper.id = generate_uuid() host_wrapper.account = ACCOUNT host_wrapper.display_name = host[0] - host_wrapper.insights_id = host[1] - host_wrapper.rhel_machine_id = host[1] - host_wrapper.subscription_manager_id = host[1] - host_wrapper.satellite_id = host[1] - host_wrapper.bios_uuid = host[1] + host_wrapper.insights_id = generate_uuid() + host_wrapper.rhel_machine_id = generate_uuid() + host_wrapper.subscription_manager_id = generate_uuid() + host_wrapper.satellite_id = generate_uuid() + host_wrapper.bios_uuid = generate_uuid() host_wrapper.ip_addresses = ["10.0.0.2"] host_wrapper.fqdn = host[2] - host_wrapper.mac_addresses = ["apple"] - host_wrapper.external_id = host[1] + host_wrapper.mac_addresses = ["aa:bb:cc:dd:ee:ff"] + host_wrapper.external_id = generate_uuid() host_wrapper.facts = [{"namespace": "ns1", "facts": {"key1": "value1"}}] response_data = self.post(HOST_URL, [host_wrapper.data()], 207) @@ -1216,19 +1217,8 @@ def test_invalid_host_id(self): class DeleteHostsTestCase(PreCreatedHostsBaseTestCase): def test_create_then_delete(self): - original_id = self.added_hosts[0].id - insights_id = self.added_hosts[0].insights_id - rhel_machine_id = self.added_hosts[0].rhel_machine_id - subscription_manager_id = self.added_hosts[0].subscription_manager_id - satellite_id = self.added_hosts[0].satellite_id - bios_uuid = self.added_hosts[0].bios_uuid - ip_addresses = self.added_hosts[0].ip_addresses - mac_addresses = self.added_hosts[0].mac_addresses - fqdn = self.added_hosts[0].fqdn - external_id = self.added_hosts[0].external_id - account = self.added_hosts[0].account - - url = HOST_URL + "/" + original_id + + url = HOST_URL + "/" + self.added_hosts[0].id # Get the host self.get(url, 200) @@ -1243,22 +1233,40 @@ def __call__(self, e): # Delete the host with unittest.mock.patch("api.host.emit_event", new=MockEmitEvent()) as m: self.delete(url, 200, return_response_as_json=False) - assert original_id in m.events[0] - assert insights_id in m.events[0] - assert rhel_machine_id in m.events[0] - assert subscription_manager_id in m.events[0] - assert satellite_id in m.events[0] - assert bios_uuid in m.events[0] - assert fqdn in m.events[0] - assert external_id in m.events[0] - assert account in m.events[0] - - for ip in ip_addresses: - assert ip in m.events[0] - - for mac in mac_addresses: - assert mac in m.events[0] - + event = json.loads(m.events[0]) + + self.assertIsInstance(event, dict) + expected_keys = { + "timestamp", + "type", + "id", + "account", + "insights_id", + "rhel_machine_id", + "subscription_manager_id", + "satellite_id", + "bios_uuid", + "ip_addresses", + "fqdn", + "mac_addresses", + "external_id", + } + self.assertEqual(set(event.keys()), expected_keys) + + self.assertEqual(self.added_hosts[0].timestamp, event["timestamp"]) + self.assertEqual(self.added_hosts[0].type, event["type"]) + self.assertEqual(self.added_hosts[0].id, event["id"]) + self.assertEqual(self.added_hosts[0].insights_id, event["insights_id"]) + self.assertEqual(self.added_hosts[0].rhel_machine_id, event["rhel_machine_id"]) + self.assertEqual(self.added_hosts[0].subscription_manager_id, event["subscription_manager_id"]) + self.assertEqual(self.added_hosts[0].satellite_id, event["satellite_id"]) + self.assertEqual(self.added_hosts[0].bios_uuid, event["bios_uuid"]) + self.assertEqual(self.added_hosts[0].fqdn, event["fqdn"]) + self.assertEqual(self.added_hosts[0].external_id, event["external_id"]) + self.assertEqual(self.added_hosts[0].account, event["account"]) + self.assertEqual(self.added_hosts[0].ip_addresses, event["ip_addresses"]) + self.assertEqual(self.added_hosts[0].mac_addresses, event["mac_addresses"]) + # Try to get the host again response = self.get(url, 200) From d1b46a26f8d662fb8e7f4eafa887d9cb6902a8ef Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 15 Aug 2019 08:37:45 -0400 Subject: [PATCH 17/26] made changes based on feedback --- api/host.py | 4 ++-- app/events.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/host.py b/api/host.py index 34d114f17..239a3df0d 100644 --- a/api/host.py +++ b/api/host.py @@ -304,8 +304,8 @@ def delete_by_id(host_id_list): logger.debug("Deleted hosts: %s", hosts_to_delete) - for deleted_host_id in hosts_to_delete: - emit_event(events.delete(deleted_host_id)) + for deleted_host in hosts_to_delete: + emit_event(events.delete(deleted_host)) @api_operation diff --git a/app/events.py b/app/events.py index 7c54d838c..87780c911 100644 --- a/app/events.py +++ b/app/events.py @@ -33,6 +33,7 @@ def delete(host): .dumps( { "timestamp": datetime.utcnow(), + "id": host.id, **CanonicalFacts.to_json(host.canonical_facts), "account": host.account, "type": "delete", From 1af93ce22fd420cde563f0316ecfec4976d04e98 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 15 Aug 2019 09:08:08 -0400 Subject: [PATCH 18/26] made suggested changes to test_api.py --- test_api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test_api.py b/test_api.py index 9164062c3..70a03edb7 100755 --- a/test_api.py +++ b/test_api.py @@ -1216,7 +1216,8 @@ def test_invalid_host_id(self): class DeleteHostsTestCase(PreCreatedHostsBaseTestCase): - def test_create_then_delete(self): + @unittest.mock.patch("app.events.datetime", **{"utcnow.return_value": datetime.utcnow()}) + def test_create_then_delete(self, datetime_mock): url = HOST_URL + "/" + self.added_hosts[0].id @@ -1234,6 +1235,7 @@ def __call__(self, e): with unittest.mock.patch("api.host.emit_event", new=MockEmitEvent()) as m: self.delete(url, 200, return_response_as_json=False) event = json.loads(m.events[0]) + timestamp_iso = datetime_mock.utcnow.return_value.isoformat() self.assertIsInstance(event, dict) expected_keys = { @@ -1253,8 +1255,8 @@ def __call__(self, e): } self.assertEqual(set(event.keys()), expected_keys) - self.assertEqual(self.added_hosts[0].timestamp, event["timestamp"]) - self.assertEqual(self.added_hosts[0].type, event["type"]) + self.assertEqual(f"{timestamp_iso}+00:00", event["timestamp"]) + self.assertEqual("delete", event["type"]) self.assertEqual(self.added_hosts[0].id, event["id"]) self.assertEqual(self.added_hosts[0].insights_id, event["insights_id"]) self.assertEqual(self.added_hosts[0].rhel_machine_id, event["rhel_machine_id"]) From 8e1892eab0ac992f51e84cc05544d84dc2a155d7 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 2 Aug 2019 17:14:47 -0400 Subject: [PATCH 19/26] Updated changes based on feedback --- README.md | 1 - api/host.py | 14 +++++++------- app/events.py | 31 ++++++++++++++++++++++++++++--- test_api.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index af4783ecf..b824d6f5f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,6 @@ _prometheus_multiproc_dir_ environment variable. This is done automatically. python run_gunicorn.py ``` - ## Configuration environment variables ``` diff --git a/api/host.py b/api/host.py index 2a4606642..239a3df0d 100644 --- a/api/host.py +++ b/api/host.py @@ -283,29 +283,29 @@ def find_hosts_by_hostname_or_id(account_number, hostname): def delete_by_id(host_id_list): query = _get_host_list_by_id_list(current_identity.account_number, host_id_list) - host_ids_to_delete = [] + hosts_to_delete = [] for host in query.all(): try: - host_ids_to_delete.append(host.id) + hosts_to_delete.append(host) except sqlalchemy.orm.exc.ObjectDeletedError: logger.exception( "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " "already deleted." ) - if not host_ids_to_delete: + if not hosts_to_delete: return flask.abort(status.HTTP_404_NOT_FOUND) with metrics.delete_host_processing_time.time(): query.delete(synchronize_session="fetch") db.session.commit() - metrics.delete_host_count.inc(len(host_ids_to_delete)) + metrics.delete_host_count.inc(len(hosts_to_delete)) - logger.debug("Deleted hosts: %s", host_ids_to_delete) + logger.debug("Deleted hosts: %s", hosts_to_delete) - for deleted_host_id in host_ids_to_delete: - emit_event(events.delete(deleted_host_id)) + for deleted_host in hosts_to_delete: + emit_event(events.delete(deleted_host)) @api_operation diff --git a/app/events.py b/app/events.py index 7279b6b9b..f5c832713 100644 --- a/app/events.py +++ b/app/events.py @@ -1,8 +1,11 @@ import logging from datetime import datetime - from marshmallow import fields from marshmallow import Schema +from marshmallow import validate + +from app.models import CanonicalFacts +from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -11,7 +14,29 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() + account = fields.Str(validate=validate.Length(min=1, max=255)) + insights_id = fields.Str(validate=verify_uuid_format) + rhel_machine_id = fields.Str(validate=verify_uuid_format) + subscription_manager_id = fields.Str(validate=verify_uuid_format) + satellite_id = fields.Str(validate=verify_uuid_format) + fqdn = fields.Str(validate=validate.Length(min=1, max=255)) + bios_uuid = fields.Str(validate=verify_uuid_format) + ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + external_id = fields.Str(validate=validate.Length(min=1, max=500)) -def delete(id): - return HostEvent(strict=True).dumps({"id": id, "timestamp": datetime.utcnow(), "type": "delete"}).data +def delete(host): + return ( + HostEvent() + .dumps( + { + "timestamp": datetime.utcnow(), + "id": host.id, + **CanonicalFacts.to_json(host.canonical_facts), + "account": host.account, + "type": "delete", + } + ) + .data + ) diff --git a/test_api.py b/test_api.py index ab8d914e4..70a03edb7 100755 --- a/test_api.py +++ b/test_api.py @@ -1107,10 +1107,18 @@ def create_hosts(self): for host in hosts_to_create: host_wrapper = HostWrapper() + host_wrapper.id = generate_uuid() host_wrapper.account = ACCOUNT host_wrapper.display_name = host[0] - host_wrapper.insights_id = host[1] + host_wrapper.insights_id = generate_uuid() + host_wrapper.rhel_machine_id = generate_uuid() + host_wrapper.subscription_manager_id = generate_uuid() + host_wrapper.satellite_id = generate_uuid() + host_wrapper.bios_uuid = generate_uuid() + host_wrapper.ip_addresses = ["10.0.0.2"] host_wrapper.fqdn = host[2] + host_wrapper.mac_addresses = ["aa:bb:cc:dd:ee:ff"] + host_wrapper.external_id = generate_uuid() host_wrapper.facts = [{"namespace": "ns1", "facts": {"key1": "value1"}}] response_data = self.post(HOST_URL, [host_wrapper.data()], 207) @@ -1208,10 +1216,10 @@ def test_invalid_host_id(self): class DeleteHostsTestCase(PreCreatedHostsBaseTestCase): - def test_create_then_delete(self): - original_id = self.added_hosts[0].id + @unittest.mock.patch("app.events.datetime", **{"utcnow.return_value": datetime.utcnow()}) + def test_create_then_delete(self, datetime_mock): - url = HOST_URL + "/" + original_id + url = HOST_URL + "/" + self.added_hosts[0].id # Get the host self.get(url, 200) @@ -1226,7 +1234,40 @@ def __call__(self, e): # Delete the host with unittest.mock.patch("api.host.emit_event", new=MockEmitEvent()) as m: self.delete(url, 200, return_response_as_json=False) - assert original_id in m.events[0] + event = json.loads(m.events[0]) + timestamp_iso = datetime_mock.utcnow.return_value.isoformat() + + self.assertIsInstance(event, dict) + expected_keys = { + "timestamp", + "type", + "id", + "account", + "insights_id", + "rhel_machine_id", + "subscription_manager_id", + "satellite_id", + "bios_uuid", + "ip_addresses", + "fqdn", + "mac_addresses", + "external_id", + } + self.assertEqual(set(event.keys()), expected_keys) + + self.assertEqual(f"{timestamp_iso}+00:00", event["timestamp"]) + self.assertEqual("delete", event["type"]) + self.assertEqual(self.added_hosts[0].id, event["id"]) + self.assertEqual(self.added_hosts[0].insights_id, event["insights_id"]) + self.assertEqual(self.added_hosts[0].rhel_machine_id, event["rhel_machine_id"]) + self.assertEqual(self.added_hosts[0].subscription_manager_id, event["subscription_manager_id"]) + self.assertEqual(self.added_hosts[0].satellite_id, event["satellite_id"]) + self.assertEqual(self.added_hosts[0].bios_uuid, event["bios_uuid"]) + self.assertEqual(self.added_hosts[0].fqdn, event["fqdn"]) + self.assertEqual(self.added_hosts[0].external_id, event["external_id"]) + self.assertEqual(self.added_hosts[0].account, event["account"]) + self.assertEqual(self.added_hosts[0].ip_addresses, event["ip_addresses"]) + self.assertEqual(self.added_hosts[0].mac_addresses, event["mac_addresses"]) # Try to get the host again response = self.get(url, 200) From 05367ff897a9c4cfcd3f6f79221338088665f61e Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 16 Aug 2019 16:44:16 -0400 Subject: [PATCH 20/26] removed validations from HostEvent(Schema) --- app/events.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/app/events.py b/app/events.py index f5c832713..9e03b6fd6 100644 --- a/app/events.py +++ b/app/events.py @@ -1,11 +1,10 @@ import logging from datetime import datetime + from marshmallow import fields from marshmallow import Schema -from marshmallow import validate from app.models import CanonicalFacts -from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -14,16 +13,16 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - account = fields.Str(validate=validate.Length(min=1, max=255)) - insights_id = fields.Str(validate=verify_uuid_format) - rhel_machine_id = fields.Str(validate=verify_uuid_format) - subscription_manager_id = fields.Str(validate=verify_uuid_format) - satellite_id = fields.Str(validate=verify_uuid_format) - fqdn = fields.Str(validate=validate.Length(min=1, max=255)) - bios_uuid = fields.Str(validate=verify_uuid_format) - ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - external_id = fields.Str(validate=validate.Length(min=1, max=500)) + account = fields.Str() + insights_id = fields.Str() + rhel_machine_id = fields.Str() + subscription_manager_id = fields.Str() + satellite_id = fields.Str() + fqdn = fields.Str() + bios_uuid = fields.Str() + ip_addresses = fields.List(fields.Str()) + mac_addresses = fields.List(fields.Str()) + external_id = fields.Str() def delete(host): From 355ba890da6cea01a06e14dd220c3c17e5f1a958 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Wed, 21 Aug 2019 09:22:26 -0400 Subject: [PATCH 21/26] Added validate back in to match create/update schema --- app/events.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/events.py b/app/events.py index 9e03b6fd6..3ce0ad7e5 100644 --- a/app/events.py +++ b/app/events.py @@ -3,8 +3,10 @@ from marshmallow import fields from marshmallow import Schema +from marshmallow import validate from app.models import CanonicalFacts +from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -13,16 +15,16 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - account = fields.Str() - insights_id = fields.Str() - rhel_machine_id = fields.Str() - subscription_manager_id = fields.Str() - satellite_id = fields.Str() - fqdn = fields.Str() - bios_uuid = fields.Str() - ip_addresses = fields.List(fields.Str()) - mac_addresses = fields.List(fields.Str()) - external_id = fields.Str() + account = fields.Str(required=True, validate=validate.Length(min=1, max=10)) + insights_id = fields.Str(validate=verify_uuid_format) + rhel_machine_id = fields.Str(validate=verify_uuid_format) + subscription_manager_id = fields.Str(validate=verify_uuid_format) + satellite_id = fields.Str(validate=verify_uuid_format) + fqdn = fields.Str(validate=validate.Length(min=1, max=255)) + bios_uuid = fields.Str(validate=verify_uuid_format) + ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) + external_id = fields.Str(validate=validate.Length(min=1, max=500)) def delete(host): From fd38720e42dc92893c46b8e77f1bc543dba741ce Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Thu, 22 Aug 2019 15:38:06 -0400 Subject: [PATCH 22/26] Fix 500 ERROR and adapt ObjectDeletedError to new code --- api/host.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/host.py b/api/host.py index 239a3df0d..2a5a7793e 100644 --- a/api/host.py +++ b/api/host.py @@ -285,13 +285,7 @@ def delete_by_id(host_id_list): hosts_to_delete = [] for host in query.all(): - try: - hosts_to_delete.append(host) - except sqlalchemy.orm.exc.ObjectDeletedError: - logger.exception( - "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " - "already deleted." - ) + hosts_to_delete.append(host) if not hosts_to_delete: return flask.abort(status.HTTP_404_NOT_FOUND) @@ -305,7 +299,13 @@ def delete_by_id(host_id_list): logger.debug("Deleted hosts: %s", hosts_to_delete) for deleted_host in hosts_to_delete: - emit_event(events.delete(deleted_host)) + try: + emit_event(events.delete(deleted_host)) + except sqlalchemy.orm.exc.ObjectDeletedError: + logger.exception( + "Encountered sqlalchemy.orm.exc.ObjectDeletedError exception during delete_by_id operation. Host was " + "already deleted." + ) @api_operation From 60025fb8ae806317509ca117970a7bb448dcdad8 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 23 Aug 2019 09:07:38 -0400 Subject: [PATCH 23/26] Added request_id to the delete event --- app/events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/events.py b/app/events.py index 3ce0ad7e5..55cb34edf 100644 --- a/app/events.py +++ b/app/events.py @@ -5,6 +5,7 @@ from marshmallow import Schema from marshmallow import validate +from app.logging import threadctx from app.models import CanonicalFacts from app.validators import verify_uuid_format @@ -25,6 +26,7 @@ class HostEvent(Schema): ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) external_id = fields.Str(validate=validate.Length(min=1, max=500)) + request_id = fields.Str(validate=validate.Length(min=1, max=500)) def delete(host): @@ -36,6 +38,7 @@ def delete(host): "id": host.id, **CanonicalFacts.to_json(host.canonical_facts), "account": host.account, + "request_id": threadctx.request_id, "type": "delete", } ) From a2b0c4d876cf76b9d521b3086c87a150024522ab Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Fri, 23 Aug 2019 14:28:24 -0400 Subject: [PATCH 24/26] removing validations from Schema --- app/events.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/app/events.py b/app/events.py index 55cb34edf..1c7f6b041 100644 --- a/app/events.py +++ b/app/events.py @@ -3,11 +3,9 @@ from marshmallow import fields from marshmallow import Schema -from marshmallow import validate from app.logging import threadctx from app.models import CanonicalFacts -from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -16,17 +14,17 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - account = fields.Str(required=True, validate=validate.Length(min=1, max=10)) - insights_id = fields.Str(validate=verify_uuid_format) - rhel_machine_id = fields.Str(validate=verify_uuid_format) - subscription_manager_id = fields.Str(validate=verify_uuid_format) - satellite_id = fields.Str(validate=verify_uuid_format) - fqdn = fields.Str(validate=validate.Length(min=1, max=255)) - bios_uuid = fields.Str(validate=verify_uuid_format) - ip_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - mac_addresses = fields.List(fields.Str(validate=validate.Length(min=1, max=255))) - external_id = fields.Str(validate=validate.Length(min=1, max=500)) - request_id = fields.Str(validate=validate.Length(min=1, max=500)) + account = fields.Str() + insights_id = fields.Str() + rhel_machine_id = fields.Str() + subscription_manager_id = fields.Str() + satellite_id = fields.Str() + fqdn = fields.Str() + bios_uuid = fields.Str() + ip_addresses = fields.List(fields.Str()) + mac_addresses = fields.List(fields.Str()) + external_id = fields.Str() + request_id = fields.Str() def delete(host): From c4c01efffb8f4eadb932eaddc76d6f75b2bec650 Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 26 Aug 2019 08:26:02 -0400 Subject: [PATCH 25/26] simplified validations to include specified attributes --- app/events.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/app/events.py b/app/events.py index 1c7f6b041..58eb81fa8 100644 --- a/app/events.py +++ b/app/events.py @@ -6,6 +6,7 @@ from app.logging import threadctx from app.models import CanonicalFacts +from app.validators import verify_uuid_format logger = logging.getLogger(__name__) @@ -14,16 +15,8 @@ class HostEvent(Schema): id = fields.UUID() timestamp = fields.DateTime(format="iso8601") type = fields.Str() - account = fields.Str() - insights_id = fields.Str() - rhel_machine_id = fields.Str() - subscription_manager_id = fields.Str() - satellite_id = fields.Str() - fqdn = fields.Str() - bios_uuid = fields.Str() - ip_addresses = fields.List(fields.Str()) - mac_addresses = fields.List(fields.Str()) - external_id = fields.Str() + account = fields.Str(required=True) + insights_id = fields.Str(validate=verify_uuid_format) request_id = fields.Str() From b586b700ef74917a77d00b26adf145a966fe13ec Mon Sep 17 00:00:00 2001 From: Alec Cohan Date: Mon, 26 Aug 2019 17:06:29 -0400 Subject: [PATCH 26/26] fixed Delete event tests in test_api.py --- test_api.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/test_api.py b/test_api.py index 70a03edb7..f8f44bb2c 100755 --- a/test_api.py +++ b/test_api.py @@ -1238,36 +1238,14 @@ def __call__(self, e): timestamp_iso = datetime_mock.utcnow.return_value.isoformat() self.assertIsInstance(event, dict) - expected_keys = { - "timestamp", - "type", - "id", - "account", - "insights_id", - "rhel_machine_id", - "subscription_manager_id", - "satellite_id", - "bios_uuid", - "ip_addresses", - "fqdn", - "mac_addresses", - "external_id", - } + expected_keys = {"timestamp", "type", "id", "account", "insights_id", "request_id"} self.assertEqual(set(event.keys()), expected_keys) self.assertEqual(f"{timestamp_iso}+00:00", event["timestamp"]) self.assertEqual("delete", event["type"]) self.assertEqual(self.added_hosts[0].id, event["id"]) self.assertEqual(self.added_hosts[0].insights_id, event["insights_id"]) - self.assertEqual(self.added_hosts[0].rhel_machine_id, event["rhel_machine_id"]) - self.assertEqual(self.added_hosts[0].subscription_manager_id, event["subscription_manager_id"]) - self.assertEqual(self.added_hosts[0].satellite_id, event["satellite_id"]) - self.assertEqual(self.added_hosts[0].bios_uuid, event["bios_uuid"]) - self.assertEqual(self.added_hosts[0].fqdn, event["fqdn"]) - self.assertEqual(self.added_hosts[0].external_id, event["external_id"]) - self.assertEqual(self.added_hosts[0].account, event["account"]) - self.assertEqual(self.added_hosts[0].ip_addresses, event["ip_addresses"]) - self.assertEqual(self.added_hosts[0].mac_addresses, event["mac_addresses"]) + self.assertEqual("-1", event["request_id"]) # Try to get the host again response = self.get(url, 200)