diff --git a/ocw/lib/azure.py b/ocw/lib/azure.py index 86135200..5e5a6583 100644 --- a/ocw/lib/azure.py +++ b/ocw/lib/azure.py @@ -10,8 +10,7 @@ from dateutil.parser import parse from webui.PCWConfig import PCWConfig from .provider import Provider -from ..models import Instance, ProviderChoice -from .influx import influxwrite, Influx +from ..models import Instance class Azure(Provider): @@ -82,7 +81,6 @@ def get_storage_key(self, storage_account: str) -> str: storage_keys = [v.value for v in storage_keys.keys] return storage_keys[0] - @influxwrite(Influx.VMS_QUANTITY, ProviderChoice.AZURE) def list_instances(self) -> list: return list(self.compute_mgmt_client().virtual_machines.list_all()) @@ -111,11 +109,9 @@ def delete_resource(self, resource_id: str) -> None: self.log_info(f"Deleting of resource group {resource_id}") self.resource_mgmt_client().resource_groups.begin_delete(resource_id) - @influxwrite(Influx.IMAGES_QUANTITY, ProviderChoice.AZURE) def list_images(self): return self.list_resource(filters="resourceType eq 'Microsoft.Compute/images'") - @influxwrite(Influx.DISK_QUANTITY, ProviderChoice.AZURE) def list_disks(self): return self.list_resource(filters="resourceType eq 'Microsoft.Compute/disks'") diff --git a/ocw/lib/dump_state.py b/ocw/lib/dump_state.py new file mode 100644 index 00000000..362ed002 --- /dev/null +++ b/ocw/lib/dump_state.py @@ -0,0 +1,21 @@ +import logging +import traceback +from webui.PCWConfig import PCWConfig +from ocw.lib.azure import Azure +from ocw.enums import ProviderChoice +from ocw.lib.influx import Influx + +logger = logging.getLogger(__name__) + + +def dump_state(): + for namespace in PCWConfig.get_namespaces_for('influxdb'): + try: + providers = PCWConfig.get_providers_for('influxdb', namespace) + logger.info("[%s] Dump state %s", namespace, ','.join(providers)) + if ProviderChoice.AZURE in providers: + Influx().dump_resource(ProviderChoice.AZURE.value, Influx.VMS_QUANTITY, Azure(namespace).list_instances) + Influx().dump_resource(ProviderChoice.AZURE.value, Influx.IMAGES_QUANTITY, Azure(namespace).list_images) + Influx().dump_resource(ProviderChoice.AZURE.value, Influx.DISK_QUANTITY, Azure(namespace).list_disks) + except Exception: + logger.exception("[%s] Dump state failed!: \n %s", namespace, traceback.format_exc()) diff --git a/ocw/lib/gce.py b/ocw/lib/gce.py index 8ee7603d..d6eb56a2 100644 --- a/ocw/lib/gce.py +++ b/ocw/lib/gce.py @@ -3,10 +3,10 @@ from os.path import basename from datetime import timezone from dateutil.parser import parse -from webui.PCWConfig import ConfigFile import googleapiclient.discovery from googleapiclient.errors import HttpError from google.oauth2 import service_account +from webui.PCWConfig import ConfigFile from .provider import Provider diff --git a/ocw/lib/influx.py b/ocw/lib/influx.py index f8ed8228..2a75f096 100644 --- a/ocw/lib/influx.py +++ b/ocw/lib/influx.py @@ -1,27 +1,17 @@ +from typing import Callable import os import logging from influxdb_client import InfluxDBClient, Point from influxdb_client.client.write_api import SYNCHRONOUS, WriteApi from influxdb_client.client.exceptions import InfluxDBError -from urllib3.exceptions import HTTPError, TimeoutError +from urllib3.exceptions import HTTPError from webui.PCWConfig import PCWConfig -from ocw.enums import ProviderChoice logger = logging.getLogger(__name__) -def influxwrite(influx_type: str, provider: ProviderChoice): - def decorator(func): - def wrapper(*args, **kwargs): - objects = func(*args, **kwargs) - Influx().write(provider.value, influx_type, len(objects)) - return objects - return wrapper - return decorator - - class Influx: __client: WriteApi | None = None VMS_QUANTITY: str = "vms_quantity" @@ -53,5 +43,10 @@ def write(self, measurement: str, field: str, value: int) -> None: point = Point(measurement).field(field, value) try: self.__client.write(bucket=self.bucket, org=self.org, record=point) - except (InfluxDBError, HTTPError, TimeoutError) as exception: - logger.warning(f"Failed to write to influxdb(record={point}): {exception}") + except (InfluxDBError, HTTPError) as exception: + logger.warning("Failed to write to influxdb(record=%s): %s", point, exception) + + def dump_resource(self, provider: str, field: str, dump_method: Callable) -> None: + items_cnt = len(dump_method()) + logger.debug("%d instances found in %s", items_cnt, provider) + self.write(provider, field, items_cnt) diff --git a/ocw/management/commands/dumpstate.py b/ocw/management/commands/dumpstate.py new file mode 100644 index 00000000..f6f323c3 --- /dev/null +++ b/ocw/management/commands/dumpstate.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand +from ocw.lib.dump_state import dump_state + + +class Command(BaseCommand): + help = 'Dump current state (amount of all entities \ + tracked by pcw) of all providers (according to pcw.ini)' + + def handle(self, *args, **options): + dump_state()