add more company reconciliation endpoints

reconcile/api/base.py

@@ -26,6 +26,7 @@ class Reconcile:
     suggest: list[str] = ["entity", "type", "property"]
     extend = True
     preview = True
+    base_type = "Organization"
 
     def reconcile_query(self, *args, **kwargs):
         return do_reconcile_query(*args, **kwargs)
@@ -209,7 +210,7 @@ def suggest_property(
         if not prefix:
             raise Http404("Prefix must be supplied")
 
-        properties = self.propose_properties(request, "Organization")["properties"]
+        properties = self.propose_properties(request, self.base_type)["properties"]
 
         return {
             "result": [
@@ -224,8 +225,9 @@ def suggest_property(
         }
 
     def propose_properties(self, request, type_, limit=500):
-        if type_ != "Organization":
-            raise Http404("type must be Organization")
+        if type_ != self.base_type:
+            msg = f"type must be {self.base_type}"
+            raise Http404(msg)
 
         organisation_properties = Organisation.get_fields_as_properties()
 

reconcile/api/reconcile_all.py

@@ -36,11 +36,18 @@
 def get_service_spec(request, queries: Query[ReconciliationQueryBatchForm]):
     if queries.queries:
         queries_parsed = ReconciliationQueryBatch(queries=json.loads(queries.queries))
-        return reconcile.reconcile(request, queries_parsed, orgtypes="all")
+        return {
+            k: ReconciliationResult(**v)
+            for k, v in reconcile.reconcile(
+                request, queries_parsed, orgtypes="all"
+            ).items()
+        }
     elif queries.extend:
         queries_parsed = DataExtensionQuery(**json.loads(queries.extend))
-        return reconcile.data_extension(request, queries_parsed)
-    return reconcile.get_service_spec(request, orgtypes="all")
+        return DataExtensionQueryResponse(
+            **reconcile.data_extension(request, queries_parsed)
+        )
+    return ServiceSpec(**reconcile.get_service_spec(request, orgtypes="all"))
 
 
 @api.post(
@@ -53,10 +60,17 @@ def get_service_spec(request, queries: Query[ReconciliationQueryBatchForm]):
 def reconcile_entities(request, queries: Form[ReconciliationQueryBatchForm]):
     if queries.queries:
         queries_parsed = ReconciliationQueryBatch(queries=json.loads(queries.queries))
-        return reconcile.reconcile(request, queries_parsed, orgtypes="all")
+        return {
+            k: ReconciliationResult(**v)
+            for k, v in reconcile.reconcile(
+                request, queries_parsed, orgtypes="all"
+            ).items()
+        }
     elif queries.extend:
         queries_parsed = DataExtensionQuery(**json.loads(queries.extend))
-        return reconcile.data_extension(request, queries_parsed)
+        return DataExtensionQueryResponse(
+            **reconcile.data_extension(request, queries_parsed)
+        )
 
 
 @api.get("/preview")

reconcile/api/reconcile_company.py

@@ -1,16 +1,23 @@
 import json
 from typing import Dict
 
+from charity_django.companies.models import CompanyTypeChoices
+from django.http import Http404
 from ninja import Form, Query, Router
 
+from ftc.documents import CompanyDocument
 from reconcile.companies import COMPANY_RECON_TYPE, do_reconcile_query
 
 from .base import Reconcile
 from .schema import (
+    DataExtensionPropertyProposalQuery,
+    DataExtensionPropertyProposalResponse,
     ReconciliationQueryBatch,
     ReconciliationQueryBatchForm,
     ReconciliationResult,
     ServiceSpec,
+    SuggestResponse,
+    SuggestTypeQuery,
 )
 
 api = Router(tags=["Reconciliation (registered companies)"])
@@ -20,13 +27,55 @@ class CompanyReconcile(Reconcile):
     name = "Find that Charity Company Reconciliation API"
     view_url = "company_detail"
     view_url_args = {"company_number": "{{id}}"}
-    suggest = []
-    extend = False
+    suggest = ["property", "type"]
+    extend = True
     preview = False
+    base_type = "Company"
 
     def reconcile_query(self, *args, **kwargs):
         return do_reconcile_query(*args, **kwargs)
 
+    def propose_properties(self, request, type_, limit=500):
+        if type_ != self.base_type:
+            msg = f"type must be {self.base_type}"
+            raise Http404(msg)
+
+        mapping = CompanyDocument._index.get_mapping()
+
+        internal_fields = ["scrape", "spider", "id", "priority"]
+        company_properties = [
+            {"id": f, "name": f}
+            for f in mapping["companies"]["mappings"]["properties"].keys()
+            if f not in internal_fields
+        ]
+
+        return {"limit": limit, "type": type_, "properties": company_properties}
+
+    def suggest_type(
+        self,
+        request,
+        prefix: str,
+        cursor: int = 0,
+    ):
+        if not prefix:
+            raise Http404("Prefix must be supplied")
+
+        results = [
+            ("registered-company", "Registered Company")
+        ] + CompanyTypeChoices.choices
+
+        return {
+            "result": [
+                {
+                    "id": id,
+                    "name": name,
+                    "notable": [],
+                }
+                for id, name in results
+                if prefix.lower() in id.lower() or prefix.lower() in name.lower()
+            ]
+        }
+
 
 reconcile = CompanyReconcile()
 
@@ -43,7 +92,9 @@ def get_company_service_spec(
     if queries.queries:
         queries_parsed = ReconciliationQueryBatch(queries=json.loads(queries.queries))
         return reconcile.reconcile(request, queries_parsed)
-    return reconcile.get_service_spec(request, defaultTypes=[COMPANY_RECON_TYPE])
+    return ServiceSpec(
+        **reconcile.get_service_spec(request, defaultTypes=[COMPANY_RECON_TYPE])
+    )
 
 
 @api.post(
@@ -59,3 +110,22 @@ def company_reconcile_entities(
 ):
     queries_parsed = ReconciliationQueryBatch(queries=json.loads(queries.queries))
     return reconcile.reconcile(request, queries_parsed)
+
+
+@api.get("/suggest/type", response={200: SuggestResponse}, exclude_none=True)
+def suggest_type(request, query: Query[SuggestTypeQuery]):
+    return reconcile.suggest_type(request, query.prefix, query.cursor)
+
+
+@api.get("/suggest/property", response={200: SuggestResponse}, exclude_none=True)
+def suggest_property(request, query: Query[SuggestTypeQuery]):
+    return reconcile.suggest_property(request, query.prefix, query.cursor)
+
+
+@api.get(
+    "/extend/propose",
+    response={200: DataExtensionPropertyProposalResponse},
+    exclude_none=True,
+)
+def propose_properties(request, query: Query[DataExtensionPropertyProposalQuery]):
+    return reconcile.propose_properties(request, type_=query.type, limit=query.limit)

reconcile/api/schema.py

@@ -42,7 +42,7 @@ class UrlSchema(Schema):
 
 
 class PreviewMetadata(Schema):
-    url: str = None
+    url: Optional[str] = None
     width: int = 430
     height: int = 300
 
@@ -63,8 +63,8 @@ class DataExtensionPropertySetting(Schema):
 
 class DataExtensionProperty(Schema):
     id: str
-    name: str = None
-    settings: DataExtensionPropertySetting = None
+    name: Optional[str] = None
+    settings: Optional[DataExtensionPropertySetting] = None
 
 
 class DataExtensionPropertyProprosal(Schema):
@@ -106,8 +106,8 @@ class DataExtensionQueryResponse(Schema):
 class SuggestMetadata(Schema):
     service_url: str
     service_path: str
-    flyout_service_url: str = None
-    flyout_service_path: str = None
+    flyout_service_url: Optional[str] = None
+    flyout_service_path: Optional[str] = None
 
 
 class ServiceSpecSuggest(Schema):
@@ -173,8 +173,8 @@ class ReconciliationQueryBatch(Schema):
 
 
 class ReconciliationQueryBatchForm(Schema):
-    queries: str = None
-    extend: str = None
+    queries: Optional[str] = None
+    extend: Optional[str] = None
 
 
 class MatchingFeature(Schema):
@@ -194,7 +194,7 @@ class ReconciliationCandidate(Schema):
 
 
 class ReconciliationResult(Schema):
-    result: List[ReconciliationCandidate]
+    result: List[ReconciliationCandidate] = []
 
 
 class ReconciliationResultBatch(RootModel[Dict[str, Dict]], Schema):
@@ -215,7 +215,7 @@ class SuggestTypeQuery(Schema):
 class SuggestResult(Schema):
     id: str
     name: str
-    description: str = None
+    description: Optional[str] = None
     notable: Optional[Union[List[EntityType], List[str]]] = None
 
 

reconcile/companies.py

@@ -17,7 +17,7 @@ def do_reconcile_query(
     result_key="result",
 ):
     if not query:
-        return []
+        return {result_key: []}
 
     properties = {p["pid"]: p["v"] for p in properties} if properties else {}
 

reconcile/query.py

@@ -31,7 +31,7 @@ def do_reconcile_query(
     result_key="result",
 ):
     if not query:
-        return []
+        return {result_key: []}
 
     if type_:
         orgtypes = type_ + orgtypes

reconcile/tests/conftest.py

@@ -3,8 +3,11 @@
 import re
 import warnings
 
+from referencing import Registry
 from referencing.jsonschema import DRAFT7
 
+from ftc.tests import TestCase
+
 SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "specs")
 SUPPORTED_API_VERSIONS = ["0.2"]
 
@@ -28,7 +31,6 @@ def get_schema(
     return schemas
 
 
-# set up jsonschema registry
 def retrieve_schema_from_filesystem(uri: str):
     recon_schema = re.match(
         r"https://reconciliation-api\.github\.io/specs/(.*)/schemas/(.*\.json)",
@@ -44,3 +46,33 @@ def retrieve_schema_from_filesystem(uri: str):
         )
 
     raise ValueError(f"Unknown URI {uri}")
+
+
+class ReconTestCase(TestCase):
+    methods = ["GET", "POST"]
+
+    def setUp(self):
+        super().setUp()
+        # set up jsonschema registry
+        self.registry = Registry(retrieve=retrieve_schema_from_filesystem)
+
+    def do_request(self, method, *args, **kwargs):
+        if method == "GET":
+            return self.client.get(*args, **kwargs)
+        elif method == "POST":
+            return self.client.post(*args, **kwargs)
+        else:
+            raise ValueError(f"Unknown method {method}")
+
+    def get_test_cases(
+        self,
+        schema_file,
+        urls: list[tuple[str, str]],
+        methods: list[str] = ["GET"],
+    ):
+        for base_url, base_url_schema_version in urls:
+            for schema_version, schema in get_schema(
+                schema_file, supported_api_versions=base_url_schema_version
+            ).items():
+                for method in methods:
+                    yield base_url, schema_version, schema, method

reconcile/tests/data/company_mapping_response.json

@@ -0,0 +1,62 @@
+{
+    "companies": {
+        "mappings": {
+            "properties": {
+                "CompanyCategory": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "CompanyName": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "CompanyNumber": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "CompanyStatus": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "PreviousNames": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                },
+                "RegAddress_PostCode": {
+                    "type": "text",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+                }
+            }
+        }
+    }
+}