Merge pull request #336 from asmorodskyi/dump_state

Create dedicated command to collect public cloud stats

ocw/lib/azure.py

@@ -10,8 +10,7 @@
 from dateutil.parser import parse
 from webui.PCWConfig import PCWConfig
 from .provider import Provider
-from ..models import Instance, ProviderChoice
-from .influx import influxwrite, Influx
+from ..models import Instance
 
 
 class Azure(Provider):
@@ -82,7 +81,6 @@ def get_storage_key(self, storage_account: str) -> str:
         storage_keys = [v.value for v in storage_keys.keys]
         return storage_keys[0]
 
-    @influxwrite(Influx.VMS_QUANTITY, ProviderChoice.AZURE)
     def list_instances(self) -> list:
         return list(self.compute_mgmt_client().virtual_machines.list_all())
 
@@ -111,11 +109,9 @@ def delete_resource(self, resource_id: str) -> None:
             self.log_info(f"Deleting of resource group {resource_id}")
             self.resource_mgmt_client().resource_groups.begin_delete(resource_id)
 
-    @influxwrite(Influx.IMAGES_QUANTITY, ProviderChoice.AZURE)
     def list_images(self):
         return self.list_resource(filters="resourceType eq 'Microsoft.Compute/images'")
 
-    @influxwrite(Influx.DISK_QUANTITY, ProviderChoice.AZURE)
     def list_disks(self):
         return self.list_resource(filters="resourceType eq 'Microsoft.Compute/disks'")
 

ocw/lib/dump_state.py

@@ -0,0 +1,21 @@
+import logging
+import traceback
+from webui.PCWConfig import PCWConfig
+from ocw.lib.azure import Azure
+from ocw.enums import ProviderChoice
+from ocw.lib.influx import Influx
+
+logger = logging.getLogger(__name__)
+
+
+def dump_state():
+    for namespace in PCWConfig.get_namespaces_for('influxdb'):
+        try:
+            providers = PCWConfig.get_providers_for('influxdb', namespace)
+            logger.info("[%s] Dump state %s", namespace, ','.join(providers))
+            if ProviderChoice.AZURE in providers:
+                Influx().dump_resource(ProviderChoice.AZURE.value, Influx.VMS_QUANTITY, Azure(namespace).list_instances)
+                Influx().dump_resource(ProviderChoice.AZURE.value, Influx.IMAGES_QUANTITY, Azure(namespace).list_images)
+                Influx().dump_resource(ProviderChoice.AZURE.value, Influx.DISK_QUANTITY, Azure(namespace).list_disks)
+        except Exception:
+            logger.exception("[%s] Dump state failed!: \n %s", namespace, traceback.format_exc())

ocw/lib/gce.py

@@ -3,10 +3,10 @@
 from os.path import basename
 from datetime import timezone
 from dateutil.parser import parse
-from webui.PCWConfig import ConfigFile
 import googleapiclient.discovery
 from googleapiclient.errors import HttpError
 from google.oauth2 import service_account
+from webui.PCWConfig import ConfigFile
 from .provider import Provider
 
 

ocw/lib/influx.py

@@ -1,27 +1,17 @@
+from typing import Callable
 import os
 import logging
 from influxdb_client import InfluxDBClient, Point
 from influxdb_client.client.write_api import SYNCHRONOUS, WriteApi
 from influxdb_client.client.exceptions import InfluxDBError
-from urllib3.exceptions import HTTPError, TimeoutError
+from urllib3.exceptions import HTTPError
 
 
 from webui.PCWConfig import PCWConfig
-from ocw.enums import ProviderChoice
 
 logger = logging.getLogger(__name__)
 
 
-def influxwrite(influx_type: str, provider: ProviderChoice):
-    def decorator(func):
-        def wrapper(*args, **kwargs):
-            objects = func(*args, **kwargs)
-            Influx().write(provider.value, influx_type, len(objects))
-            return objects
-        return wrapper
-    return decorator
-
-
 class Influx:
     __client: WriteApi | None = None
     VMS_QUANTITY: str = "vms_quantity"
@@ -53,5 +43,10 @@ def write(self, measurement: str, field: str, value: int) -> None:
             point = Point(measurement).field(field, value)
             try:
                 self.__client.write(bucket=self.bucket, org=self.org, record=point)
-            except (InfluxDBError, HTTPError, TimeoutError) as exception:
-                logger.warning(f"Failed to write to influxdb(record={point}): {exception}")
+            except (InfluxDBError, HTTPError) as exception:
+                logger.warning("Failed to write to influxdb(record=%s): %s", point, exception)
+
+    def dump_resource(self, provider: str, field: str, dump_method: Callable) -> None:
+        items_cnt = len(dump_method())
+        logger.debug("%d instances found in %s", items_cnt, provider)
+        self.write(provider, field, items_cnt)

ocw/management/commands/dumpstate.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+from ocw.lib.dump_state import dump_state
+
+
+class Command(BaseCommand):
+    help = 'Dump current state (amount of all entities \
+        tracked by pcw) of all providers (according to pcw.ini)'
+
+    def handle(self, *args, **options):
+        dump_state()