diff --git a/.gitignore b/.gitignore index c5c345c4..edd4b23f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,9 @@ dist/ .ruff_cache .pytest_cache .idea/ +/facade_charm/mocks/provide/ +/facade_charm/mocks/require/ +/facade_charm/charmcraft.yaml + venv -*.charm \ No newline at end of file +*.charm diff --git a/facade_charm/README.md b/facade_charm/README.md new file mode 100644 index 00000000..5e509fa4 --- /dev/null +++ b/facade_charm/README.md @@ -0,0 +1,69 @@ +# Facade charm + +The facade charm is somewhat similar to the any-charm, but is less generic and has a sharper focus on relation data. + +Usage: +`juju deploy facade` +`juju integrate facade:provide-your-interface <your charm>` + +## update databags with jhack eval + +`jhack eval facade/0 "self.set('provide-your-interface', app_data={'hello': 'world'}, unit_data={'lets foo': 'those bars'})"` + +## update databags with actions + +```yaml +# in params.yaml +endpoint: provide-tempo_cluster +app_data: '{"foo":"bar"}' +unit_data: '{"foo":"baz"}' +``` + +`juju run facade/0 update --params params.yaml` + +## update databags from mock files + +`cd facade-charm` +`jhack sync -S facade/0 --include-files=".*\.yaml" --source mocks` + +`cd facade-charm/mocks/provide` + +edit `provide-your-interface.yaml` + +`juju run facade update` + +# using custom interfaces + +Your interface isn't in `charm-relation-interfaces` or on charmhub (yet)? No problem. +Add it to `custom_interfaces.yaml`, and it shall be picked up when you `tox -e pack`. + + +## interface conflicts +The operator framework translates `"-"` to `"_"` when generating event names. +So if you have an endpoint called `foo-bar`, you'll listen to relation events like: `self.on.foo_bar_relation_changed`. + +So if you have two endpoints, one called `foo-bar` and another called `foo_bar`, the operator framework will complain that the event is already defined. + +It turns out, there are some conflicts of this kind. +In this case, instead of generating `provides-foo-bar` and `provides-foo_bar`, our approach is to map `foo-bar` to `foo__bar`. + +So the facade charm will have: + +```yaml +provides: + provides-foo__bar: + interface: foo-bar + provides-foo_bar: + interface: foo_bar +``` + +And same for `requires`. + +The list of such conflicts, and how they've been resolved, at the time of writing, is: + +- `vault-kv & vault_kv` --> **vault__kv** +- `nginx-route & nginx_route` --> **nginx__route** +- `grafana-dashboard & grafana_dashboard` --> **grafana__dashboard** +- `tls-certificates & tls_certificates` --> **tls__certificates** + + diff --git a/facade_charm/charmhub_interfaces.yaml b/facade_charm/charmhub_interfaces.yaml new file mode 100644 index 00000000..e970c30c --- /dev/null +++ b/facade_charm/charmhub_interfaces.yaml @@ -0,0 +1,239 @@ +interfaces: +- ephemeral-backend +- nova-vgpu +- secrets +- glance +- pgbouncer-extra-config +- ceph-mds +- elasticsearch-datastore +- gnocchi +- ubuntu +- lxd-bgp +- local-monitors +- keystone-domain-backend +- ceph-admin +- ldap +- docker-registry +- aar +- jenkins-slave +- rest +- lldp +- kapacitor +- script-provider +- tokens +- grafana_auth +- quantum +- user-group +- temporal +- swift +- oauth +- pod-defaults +- jenkins-extension +- hydra_endpoints +- elastic-beats +- block-storage +- tls-certificates +- ceph-osd +- prometheus +- db +- squid-auth-helper +- ovsdb-subordinate +- dockerhost +- keystone-middleware +- grafana-source +- radosgw-user +- kubeflow_dashboard_links +- nova-ceilometer +- swift-gw +- karma_dashboard +- shards +- fluentbit +- object-storage +- websso-fid-service-provider +- ovsdb-manager +- glance-simplestreams-sync +- anbox-stream-gateway +- openstack-loadbalancer +- ingress_per_unit +- monitors +- logs +- kafka +- pgsql +- telegraf-exec +- neutron-load-balancer +- smtp +- redis +- slurmdbd +- statistics +- lxd +- prometheus_scrape +- grafana_datasource +- cinder +- keystone-fid-service-provider +- loadbalancer +- service-mesh +- certificate_transfer +- manila-plugin +- magma-orchestrator +- gcp-integration +- elasticsearch +- lxd-https +- httpd +- grafana_cloud_config +- cos_agent +- ceph-radosgw +- mount +- prometheus_remote_write +- keystone-admin +- external_cloud_provider +- apache-vhost-config +- rabbitmq +- giraph +- mysql-root +- vsd-rest-api +- mongodb +- mongodb_client +- ceph-iscsi-admin-access +- keystone-notifications +- swift-global-cluster +- vsphere-integration +- s3 +- ntp +- monitor +- kratos_info +- neutron-api +- zuul +- http +- mysql-monitor +- kratos_endpoints +- external_provider +- baremetal +- juju-info +- ovsdb-cms +- barbican-hsm +- xlc-compiler +- containers +- prolog-epilog +- mysql-router +- postgresql_client +- slurmrestd +- istio-gateway-info +- service-control +- register-application +- glauth_auxiliary +- odl-controller-api +- cinder-backend +- keystone-credentials +- sdn-plugin +- alertmanager_remote_configuration +- lxd-metrics +- prometheus-manual +- nova-cell +- oidc-client +- placement +- event-service +- radosgw-multisite +- prometheus-rules +- ceph-bootstrap +- slurmd +- ranger_client +- openstack-integration +- bind-rndc +- arangodb +- thruk-agent +- etcd +- glance-backend +- nova +- mysql-async-replication +- mysql_client +- landscape-hosted +- bgp +- websso-trusted-dashboard +- mysql-shared +- untrusted-container-runtime +- ftn-compiler +- ovsdb-cluster +- vault-kv +- kubernetes-cni +- login_ui_endpoints +- cinder-gw +- infoblox +- neutron-plugin +- loki_push_api +- swift-proxy +- ceph-client +- mysql +- keystone +- neutron-plugin-api-subordinate +- jenkins_agent_v0 +- stun-server +- autoscaling +- dashboard-plugin +- openfga +- zookeeper +- barbican-secrets +- sentry-metrics +- nats +- aws-iam +- ingress-auth +- syslog +- nova-vmware +- azure-integration +- grafana_dashboard +- hacluster +- agent-auth +- memcache +- cassandra +- pacemaker-remote +- apache-website +- guacd +- nginx-route +- config-server +- ingress +- nova-compute +- ceph-dashboard +- grafana-dashboard +- ssl-termination +- juju-dashboard +- lxd-dns +- kube-dns +- oathkeeper_info +- udldap-userdata +- kafka_client +- cinder-backup +- aws-integration +- heat-plugin-subordinate +- etcd-proxy +- nrpe +- ceph-rbd-mirror +- saml +- postfix-metrics +- db2 +- munin-node +- cinder-nedge +- kube-control +- lte-core +- grpc +- forward_auth +- auth_proxy +- catalogue +- nrpe-external-master +- dashboard +- neutron-plugin-api +- web-publish +- logstash-client +- midonet +- traefik_route +- container-runtime +- public-address +- ceilometer +- influxdb-api +- generic-ip-port-user-pass +- java +- nedge +- ovsdb +- k8s-service +- cinder-ceph-key +- designate +- tracing +- alertmanager_dispatch diff --git a/facade_charm/custom_interfaces.yaml b/facade_charm/custom_interfaces.yaml new file mode 100644 index 00000000..9de1c16c --- /dev/null +++ b/facade_charm/custom_interfaces.yaml @@ -0,0 +1,3 @@ +interfaces: + - your-interface-here + - tempo_cluster diff --git a/facade_charm/fetch_interfaces_from_charmhub.py b/facade_charm/fetch_interfaces_from_charmhub.py new file mode 100644 index 00000000..2eae09f7 --- /dev/null +++ b/facade_charm/fetch_interfaces_from_charmhub.py @@ -0,0 +1,61 @@ +"""Scrape charmhub to grab all interfaces for all registered charms. +""" +import asyncio +import json +import logging +from pathlib import Path + +import aiohttp +import requests +import tenacity +import yaml + +logger = logging.getLogger("update-endpoints") + +FACADE_CHARM_ROOT = Path(__file__).parent +CH_INTERFACES_PATH = FACADE_CHARM_ROOT / 'charmhub_interfaces.yaml' + + +def _get_all_registered_charms(): + print('fetching store...') + resp = requests.get("https://charmhub.io/packages.json") + return resp.json()['packages'] + + +@tenacity.retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(1)) +async def get_endpoints(charm_name, session: aiohttp.ClientSession): + url = f"https://charmhub.io/{charm_name}/integrations.json" + async with session.get(url=url, timeout=4) as response: + raw = await response.read() + return json.loads(raw)['grouped_relations'] + + +async def get_all_integrations(charms_pkg_info): + async with aiohttp.ClientSession() as session: + ret = await asyncio.gather(*(get_endpoints(charm['name'], session) for charm in charms_pkg_info)) + return ret + + +def _gather_interfaces(charms_pkg_info): + logger.info(f"gathering interfaces from {len(charms_pkg_info)} charms...") + all_integrations = asyncio.run(get_all_integrations(charms_pkg_info)) + + interfaces = set() + # discard any failed ones + for integrations in filter(None, all_integrations): + provides = integrations.get('provides', []) + requires = integrations.get('requires', []) + interfaces.update(endpoint['interface'] for endpoint in provides + requires) + + logger.info(f"gathered {len(interfaces)} interfaces") + return interfaces + + +def main(): + charms_pkg_info = _get_all_registered_charms() + interfaces = _gather_interfaces(charms_pkg_info) + CH_INTERFACES_PATH.write_text(yaml.safe_dump({"interfaces": list(interfaces)})) + + +if __name__ == '__main__': + main() diff --git a/facade_charm/mocks/__DO_NOT_TOUCH.md b/facade_charm/mocks/__DO_NOT_TOUCH.md new file mode 100644 index 00000000..49cef1a3 --- /dev/null +++ b/facade_charm/mocks/__DO_NOT_TOUCH.md @@ -0,0 +1 @@ +the contents of this directory may be overridden by the update_endpoints script. \ No newline at end of file diff --git a/facade_charm/requirements.txt b/facade_charm/requirements.txt new file mode 100644 index 00000000..618ba75e --- /dev/null +++ b/facade_charm/requirements.txt @@ -0,0 +1 @@ +ops ~= 2.5 diff --git a/facade_charm/src/charm.py b/facade_charm/src/charm.py new file mode 100755 index 00000000..70548bc0 --- /dev/null +++ b/facade_charm/src/charm.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import json +import logging +from itertools import chain +from pathlib import Path +from typing import Dict, Optional + +import ops +import yaml +from ops import ActiveStatus + +logger = logging.getLogger(__name__) +MOCKS_ROOT = Path(__file__).parent.parent / 'mocks' + + +class FacadeCharm(ops.CharmBase): + """Charming facade.""" + + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.collect_unit_status, self._on_collect_unit_status) + self.framework.observe(self.on.update_action, self._on_update_action) + + for relation in self.meta.relations: + on_relation = self.on[relation] + for evt in ['changed', 'created', 'joined']: + self.framework.observe(getattr(on_relation, "relation_" + evt), self._on_update_relation) + + def _on_update_action(self, e: ops.ActionEvent): + updated = [] + if endpoint := e.params.get("endpoint"): + app_data = json.loads(e.params.get("app_data") or "null") + unit_data = json.loads(e.params.get("unit_data") or "null") + if app_data or unit_data: + e.log(f"updating mock data for {endpoint}") + self._write_mock( + endpoint, + app_data, + unit_data + ) + + e.log(f"updating endpoint {endpoint}") + if relations := self.model.relations.get(endpoint): + updated = self._update(*relations) + else: + e.log(f"no bindings on {endpoint}") + + else: + e.log("updating all endpoints") + updated = self._update() + e.set_results({"updated": updated}) + + def _on_clear_action(self, e: ops.ActionEvent): + updated = [] + if endpoint := e.params.get("endpoint"): + e.log(f"clearing endpoint {endpoint}") + if relations := self.model.relations[endpoint]: + updated = self._update(*relations, clear=True) + else: + e.log(f"no bindings on {endpoint}") + else: + e.log("clearing all endpoints") + updated = self._update(clear=True) + e.set_results({"cleared": updated}) + + def _on_update_relation(self, e: ops.RelationEvent): + self._update(e.relation) + + def _update(self, *relation: ops.Relation, + clear=False): + to_update = list(relation) if relation else list(chain(*self.model.relations.values())) + updated = [] + for relation in to_update: + if self._update_relation(relation, clear=clear): + updated.append(relation.name) + return updated + + def _update_relation(self, relation: ops.Relation, + clear=False, replace=False): + changed = False + app_databag = relation.data[self.app] + unit_databag = relation.data[self.unit] + if replace or clear: + if app_databag.keys(): + app_databag.clear() + changed = True + if unit_databag.keys(): + unit_databag.clear() + changed = True + + if not clear: + app_data, unit_data = self._load_mock(relation.name) + if app_data and dict(app_databag) != app_data: + app_databag.update(app_data) + changed = True + if unit_data and dict(unit_databag) != unit_data: + unit_databag.update(unit_data) + changed = True + return changed + + def _get_mock_file(self, endpoint: str): + if endpoint.startswith("provide"): + pth = MOCKS_ROOT / "provide" / (endpoint + ".yaml") + else: + pth = MOCKS_ROOT / "require" / (endpoint + ".yaml") + + if not pth.exists(): + logger.info(f"mock not found for {endpoint} ({pth}): creating file...") + pth.parent.mkdir(parents=True, exist_ok=True) + pth.touch() + self._write_mock(endpoint, {}, {}) + + return pth + + def _load_mock(self, endpoint: str): + pth = self._get_mock_file(endpoint) + yml = yaml.safe_load(pth.read_text()) or {} + app_data = yml.get("app_data", {}) + unit_data = yml.get("unit_data", {}) + return app_data, unit_data + + def _write_mock(self, endpoint: str, + app_data: Optional[dict] = None, + unit_data: Optional[dict] = None): + pth = self._get_mock_file(endpoint) + yml = yaml.safe_load(pth.read_text()) or {} + + if app_data == {}: + _app_data = {} + else: + _app_data = yml.get("app_data") or {} + if app_data: + _app_data.update(app_data) + + if unit_data == {}: + _unit_data = {} + else: + _unit_data = yml.get("unit_data") or {} + if unit_data: + _unit_data.update(unit_data) + + logger.info(f"updating mock with {_app_data}, {_unit_data}") + pth.write_text( + yaml.safe_dump( + { + "endpoint": endpoint, + "app_data": _app_data, + "unit_data": _unit_data} + ) + ) + + def _on_collect_unit_status(self, e: ops.CollectStatusEvent): + e.add_status(ActiveStatus("facade ready")) + + # target for jhack eval + def set(self, + endpoint: str, + relation_id: Optional[int] = None, + app_data: Optional[Dict[str, str]] = None, + unit_data: Optional[Dict[str, str]] = None, + ): + """Target for jhack eval. + + Writes app and unit databags for one or multiple relations. + """ + # keep mocks in sync + self._write_mock(endpoint, app_data, unit_data) + + rel = self.model.get_relation(endpoint, relation_id) + + if app_data: + rel.data[self.app].update(app_data) + elif app_data is not None: # user passed {} + rel.data[self.app].clear() + + if unit_data: + rel.data[self.unit].update(unit_data) + elif unit_data is not None: # user passed {} + rel.data[self.unit].clear() + + +if __name__ == "__main__": # pragma: nocover + ops.main(FacadeCharm) # type: ignore diff --git a/facade_charm/tests/unit/test_charm.py b/facade_charm/tests/unit/test_charm.py new file mode 100644 index 00000000..69fe3322 --- /dev/null +++ b/facade_charm/tests/unit/test_charm.py @@ -0,0 +1,15 @@ +# Copyright 2024 pietro +# See LICENSE file for licensing details. +# +# Learn more about testing at: https://juju.is/docs/sdk/testing + +import scenario + +from facade_charm.src.charm import FacadeCharm + + +def test_smoke(): + scenario.Context(FacadeCharm).run( + "update-status", + scenario.State() + ) diff --git a/facade_charm/tox.ini b/facade_charm/tox.ini new file mode 100644 index 00000000..5dfe7604 --- /dev/null +++ b/facade_charm/tox.ini @@ -0,0 +1,64 @@ +# Copyright 2024 canonical +# See LICENSE file for licensing details. + +[tox] +no_package = True +skip_missing_interpreters = True +env_list = pack, release +min_version = 4.0.0 + +[vars] +src_path = {tox_root}/src +tests_path = {tox_root}/tests +all_path = {[vars]src_path} {[vars]tests_path} + +[testenv] +set_env = + PYTHONPATH = {tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=pdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS + +[testenv:pack] +description = Pack the facade charm. +deps = + -r {tox_root}/requirements.txt + # required by fetch_interfaces_from_charmhub + aiohttp + requests + tenacity +allowlist_externals = + charmcraft + rm +commands = + python ./fetch_interfaces_from_charmhub.py + python ./update_endpoints.py + charmcraft pack + rm charmcraft.yaml + rm -rf ./mocks/require + rm -rf ./mocks/provide + +[testenv:fmt] +description = Format. +deps = + -r {tox_root}/requirements.txt + black + ruff + isort +commands = + ruff check --fix . + isort --profile black . + + +[testenv:release] +description = Release a new revision of the facade charm. +allowlist_externals = + charmcraft +commands = + charmcraft upload ./facade_ubuntu-22.04-amd64.charm --format json + # TODO grab revision from ^^ + charmcraft release facade -r 5 --channel edge + diff --git a/facade_charm/update_endpoints.py b/facade_charm/update_endpoints.py new file mode 100644 index 00000000..dfd342c6 --- /dev/null +++ b/facade_charm/update_endpoints.py @@ -0,0 +1,187 @@ +# Copyright 2024 canonical +# See LICENSE file for licensing details. + +import logging +from pathlib import Path +from shutil import rmtree + +import yaml + +logger = logging.getLogger("update-endpoints") +FACADE_CHARM_ROOT = Path(__file__).parent +MOCKS_ROOT = FACADE_CHARM_ROOT / "mocks" +CRI_ROOT = FACADE_CHARM_ROOT.parent +INTERFACES_ROOT = CRI_ROOT / 'interfaces' +CH_INTERFACES_PATH = FACADE_CHARM_ROOT / 'charmhub_interfaces.yaml' + +CHARMCRAFT_YAML_TEMPLATE = """name: facade +type: charm +title: Facade charm +summary: Charm meant for manual and integration testing of charm interfaces. +description: | + A programmable charm that allows you to mock relation data. +bases: + - build-on: + - name: ubuntu + channel: "22.04" + run-on: + - name: ubuntu + channel: "22.04" +actions: + update: + description: updates all databags or the specified one + params: + endpoint: + type: string + default: "" + description: name of the endpoint + relation_id: + type: number + default: -1 + description: | + Relation ID. If not given, + all relations over this endpoint will be updated. + app_data: + type: string + default: "" + description: | + Json-encoded application databag. + If ``{}``, the databag will be cleared. + unit_data: + type: string + default: "" + description: | + Json-encoded unit databag. + If ``{}``, the databag will be cleared. + + + clear: + description: clears all databags or the specified one + params: + endpoint: + type: string + default: "" + description: name of the endpoint + + # set: + # params: + # app: + # type: boolean + # description: target the application databag + # endpoint: + # type: string + # description: name of the endpoint + # relation_id: + # type: number + # default: -1 + # description: relation id + # contents: + # type: string + # description: comma-separated, key=value mapping. Example: foo=bar,baz=qux + # description: writes to a databag + +# requires and provides is populated by tox -e update-endpoints +""" + +DATABAG_TEMPLATE = """ +# databag template for {interface} {role}: +# this data will be put in the application/unit databag of any relation bound on {endpoint} + +# must be str:str +app_data: + # foo: bar + +# must be str:str +units_data: + # baz: qux +""" + +reserved_interfaces = { + "juju-info", + "juju-dashboard" +} + + +def _load_custom_interfaces() -> list: + return yaml.safe_load((FACADE_CHARM_ROOT / 'custom_interfaces.yaml').read_text())['interfaces'] + + +def main(): + logger.info("loading custom interfaces...") + interfaces = _load_custom_interfaces() + + logger.info("collecting built-in interfaces...") + for interface_path in INTERFACES_ROOT.glob("*"): + interface = interface_path.name + + if interface.startswith("__"): + logger.info(f"skipping {interface}") + continue + + interfaces.append(interface) + + # add interfaces from charmhub if file is found + if CH_INTERFACES_PATH.exists(): + logger.info(f"loading from {CH_INTERFACES_PATH}...") + interfaces.extend(yaml.safe_load(CH_INTERFACES_PATH.read_text())['interfaces']) + + def _underscore(s: str): + return s.replace("-", "_") + + def _dunderscore(s: str): + return s.replace("-", "__") + + deduped = set() + for intf in interfaces: + if '-' in intf and _underscore(intf) in interfaces: + # of will replace - with _ on event register, so we replace - with __ + # instead to avoid endpoint name conflicts at runtime, + logger.warning( + f"{intf!r} conflicts with {_underscore(intf)!r}: " + f"endpoint will be named {_dunderscore(intf)!r} instead" + ) + deduped.add(_dunderscore(intf)) + else: + deduped.add(intf) + + # remove reserved interfaces, and sort them all + sorted_interfaces = sorted(deduped.difference(reserved_interfaces)) + + endpoints = { + "provides": {f"provide-{intf}": {"interface": intf} for intf in sorted_interfaces}, + "requires": {f"require-{intf}": {"interface": intf} for intf in sorted_interfaces} + } + + logger.info("writing charmcraft.yaml...") + charmcraft_yaml = FACADE_CHARM_ROOT / 'charmcraft.yaml' + post = yaml.safe_dump(endpoints) + charmcraft_yaml.write_text(CHARMCRAFT_YAML_TEMPLATE + post) + + logger.info("cleaning up old mocks...") + for mock in MOCKS_ROOT.glob("*"): + if mock.name.startswith("__"): + logger.debug(f"skipping {mock}") + continue + rmtree(mock) + + logger.info("generating mocks...") + mocks_root_provide = MOCKS_ROOT / "provide" + mocks_root_require = MOCKS_ROOT / "require" + mocks_root_provide.mkdir(exist_ok=True) + mocks_root_require.mkdir(exist_ok=True) + + for intf in interfaces: + for prefix, prefixed_path in ( + ("provide", mocks_root_provide), ("require", mocks_root_require) + ): + endpoint = f"{prefix}-{intf}" + mock_path = prefixed_path / (endpoint + ".yaml") + mock_path.write_text(DATABAG_TEMPLATE.format( + role=prefix + "r", + interface=intf, + endpoint=endpoint) + ) + + +if __name__ == '__main__': + main()