Merge pull request #2427 from GSA-TTS/main

2023-10-05 main -> prod

backend/.profile

@@ -58,8 +58,6 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
     echo 'Starting seed_cog_baseline' &&
     python manage.py seed_cog_baseline &&
     echo 'Finished seed_cog_baseline'
-    python manage.py update_oldformat_reportids &&
-    echo 'Finished rewriting old-format report_id values'
 fi
 
 # Make psql usable by scripts, for debugging, etc.

backend/api/test_uei.py

@@ -123,7 +123,7 @@
 
 invalid_uei_results_dict = {
     "totalRecords": 1,
-    "entityData": [
+    "deliberatelymisnamedkeyentityData": [
         {
             "entityRegistration": {
                 "samRegistered": "Yes",
@@ -282,6 +282,198 @@
 }
 inactive_uei_results = json.dumps(inactive_uei_results_dict)
 
+multiple_uei_results_dict = {
+    "totalRecords": 2,
+    "entityData": [
+        {
+            "entityRegistration": {
+                "samRegistered": "Yes",
+                "ueiSAM": "TESTLZR5JNQ8",
+                "entityEFTIndicator": "0686",
+                "cageCode": "7TEST",
+                "dodaac": None,
+                "legalBusinessName": "SOME VILLAGE",
+                "dbaName": None,
+                "purposeOfRegistrationCode": "Z1",
+                "purposeOfRegistrationDesc": "Federal Assistance Awards",
+                "registrationStatus": "Active",
+                "evsSource": "E&Y",
+                "registrationDate": "2011-01-18",
+                "lastUpdateDate": "2022-12-22",
+                "registrationExpirationDate": "2023-12-16",
+                "activationDate": "2022-12-21",
+                "ueiStatus": "Active",
+                "ueiExpirationDate": None,
+                "ueiCreationDate": "2020-05-01",
+                "publicDisplayFlag": "Y",
+                "exclusionStatusFlag": "N",
+                "exclusionURL": None,
+                "dnbOpenData": None,
+            },
+            "coreData": {
+                "entityInformation": {
+                    "entityURL": "www.example.gov",
+                    "entityDivisionName": None,
+                    "entityDivisionNumber": None,
+                    "entityStartDate": "1879-06-11",
+                    "fiscalYearEndCloseDate": "04/30",
+                    "submissionDate": "2022-12-16",
+                },
+                "physicalAddress": {
+                    "addressLine1": "53 SOME RD",
+                    "addressLine2": None,
+                    "city": "SOME",
+                    "stateOrProvinceCode": "IL",
+                    "zipCode": "60525",
+                    "zipCodePlus4": "2469",
+                    "countryCode": "USA",
+                },
+                "mailingAddress": {
+                    "addressLine1": "53 SOME RD",
+                    "addressLine2": None,
+                    "city": "SOME",
+                    "stateOrProvinceCode": "IL",
+                    "zipCode": "60525",
+                    "zipCodePlus4": "2469",
+                    "countryCode": "USA",
+                },
+                "congressionalDistrict": "03",
+                "generalInformation": {
+                    "entityStructureCode": "2A",
+                    "entityStructureDesc": "U.S. Government Entity",
+                    "entityTypeCode": "12",
+                    "entityTypeDesc": "US Local Government",
+                    "profitStructureCode": None,
+                    "profitStructureDesc": None,
+                    "organizationStructureCode": None,
+                    "organizationStructureDesc": None,
+                    "stateOfIncorporationCode": None,
+                    "stateOfIncorporationDesc": None,
+                    "countryOfIncorporationCode": None,
+                    "countryOfIncorporationDesc": None,
+                },
+                "businessTypes": {
+                    "businessTypeList": [
+                        {
+                            "businessTypeCode": "12",
+                            "businessTypeDesc": "US Local Government",
+                        },
+                        {"businessTypeCode": "C6", "businessTypeDesc": "Municipality"},
+                    ],
+                    "sbaBusinessTypeList": [
+                        {
+                            "sbaBusinessTypeCode": None,
+                            "sbaBusinessTypeDesc": None,
+                            "certificationEntryDate": None,
+                            "certificationExitDate": None,
+                        }
+                    ],
+                },
+                "financialInformation": {
+                    "creditCardUsage": "N",
+                    "debtSubjectToOffset": "N",
+                },
+            },
+        },
+        {
+            "entityRegistration": {
+                "samRegistered": "Yes",
+                "ueiSAM": "TESTLZR5JNQ8",
+                "entityEFTIndicator": None,
+                "cageCode": "6TEST",
+                "dodaac": None,
+                "legalBusinessName": "SOME VILLAGE",
+                "dbaName": None,
+                "purposeOfRegistrationCode": "Z1",
+                "purposeOfRegistrationDesc": "Federal Assistance Awards",
+                "registrationStatus": "Active",
+                "evsSource": "E&Y",
+                "registrationDate": "2011-01-18",
+                "lastUpdateDate": "2022-12-22",
+                "registrationExpirationDate": "2023-12-16",
+                "activationDate": "2022-12-21",
+                "ueiStatus": "Active",
+                "ueiExpirationDate": None,
+                "ueiCreationDate": "2020-05-01",
+                "publicDisplayFlag": "Y",
+                "exclusionStatusFlag": "N",
+                "exclusionURL": None,
+                "dnbOpenData": None,
+            },
+            "coreData": {
+                "entityInformation": {
+                    "entityURL": "www.example.gov",
+                    "entityDivisionName": None,
+                    "entityDivisionNumber": None,
+                    "entityStartDate": "1879-06-11",
+                    "fiscalYearEndCloseDate": "04/30",
+                    "submissionDate": "2022-12-16",
+                },
+                "physicalAddress": {
+                    "addressLine1": "53 SOME RD",
+                    "addressLine2": None,
+                    "city": "SOME",
+                    "stateOrProvinceCode": "IL",
+                    "zipCode": "60525",
+                    "zipCodePlus4": "2469",
+                    "countryCode": "USA",
+                },
+                "mailingAddress": {
+                    "addressLine1": "53 SOME RD",
+                    "addressLine2": None,
+                    "city": "SOME",
+                    "stateOrProvinceCode": "IL",
+                    "zipCode": "60525",
+                    "zipCodePlus4": "2469",
+                    "countryCode": "USA",
+                },
+                "congressionalDistrict": "03",
+                "generalInformation": {
+                    "entityStructureCode": "2A",
+                    "entityStructureDesc": "U.S. Government Entity",
+                    "entityTypeCode": "12",
+                    "entityTypeDesc": "US Local Government",
+                    "profitStructureCode": None,
+                    "profitStructureDesc": None,
+                    "organizationStructureCode": None,
+                    "organizationStructureDesc": None,
+                    "stateOfIncorporationCode": None,
+                    "stateOfIncorporationDesc": None,
+                    "countryOfIncorporationCode": None,
+                    "countryOfIncorporationDesc": None,
+                },
+                "businessTypes": {
+                    "businessTypeList": [
+                        {
+                            "businessTypeCode": "12",
+                            "businessTypeDesc": "US Local Government",
+                        },
+                        {"businessTypeCode": "C6", "businessTypeDesc": "Municipality"},
+                    ],
+                    "sbaBusinessTypeList": [
+                        {
+                            "sbaBusinessTypeCode": None,
+                            "sbaBusinessTypeDesc": None,
+                            "certificationEntryDate": None,
+                            "certificationExitDate": None,
+                        }
+                    ],
+                },
+                "financialInformation": {
+                    "creditCardUsage": "N",
+                    "debtSubjectToOffset": "N",
+                },
+            },
+        },
+    ],
+    "links": {
+        "selfLink": "https://api.sam.gov/entity-information/v3/entities?api_key=REPLACE_WITH_API_KEY&ueiSAM=TESTLZR5JNQ8&samRegistered=Yes&includeSections=entityRegistration%2CcoreData&page=0&size=10"
+    },
+}
+
+
+multiple_uei_results = json.dumps(multiple_uei_results_dict)
+
 
 class UtilsTesting(TestCase):
     def test_get_uei_info_from_sam_gov(self):
@@ -299,7 +491,7 @@ def test_get_uei_info_from_sam_gov(self):
             results = get_uei_info_from_sam_gov(uei=test_uei)
 
             self.assertTrue(results["valid"])
-            self.assertRaises(KeyError, lambda: results["errors"])
+            self.assertTrue("errors" not in results)
             self.assertEqual(
                 results["response"],
                 json.loads(valid_uei_results)["entityData"][0],
@@ -398,9 +590,12 @@ def bad_reqexception(*args, **kwds):
 
             self.assertFalse(results["valid"])
             self.assertTrue(results["errors"])
-            self.assertEquals(
+            self.assertEqual(
                 results["errors"],
-                ["SAM.gov invalid number of entries"],
+                ["UEI was not found in SAM.gov"],
+                # 2023_10_05: we now return the above error if the number of actual
+                # entries is zero. The old error is in the next comment line.
+                # ["SAM.gov invalid number of entries"],
             )
 
         # Invalid number of SAM.gov entries
@@ -419,18 +614,38 @@ def bad_reqexception(*args, **kwds):
                 ["SAM.gov unexpected JSON shape"],
             )
 
-        # Inactive entry
+    def test_get_uei_info_from_sam_gov_inactive_result(self):
+        """
+        Tests that we can handle inactive entries.
+        """
+        test_uei = "ZQGGHJH74DW7"
+
         with patch("api.uei.SESSION.get") as mock_get:
-            mock_get.return_value.status_code = 200  # Mock the status code
-            mock_get.return_value.json.return_value = json.loads(
-                inactive_uei_results
-            )  # Mock the json
+            mock_results = json.loads(inactive_uei_results)
+            expected = mock_results["entityData"][0]
+            mock_get.return_value.status_code = 200
+            mock_get.return_value.json.return_value = mock_results
 
             results = get_uei_info_from_sam_gov(uei=test_uei)
 
-            self.assertFalse(results["valid"])
-            self.assertTrue(results["errors"])
-            self.assertEquals(
-                results["errors"],
-                ["UEI is not listed as active from SAM.gov response data"],
-            )
+            self.assertTrue(results["valid"])
+            self.assertTrue("errors" not in results)
+            self.assertEqual(results["response"], expected)
+
+    def test_get_uei_info_from_sam_gov_multiple_results(self):
+        """
+        Tests that we can handle multiple UEIs.
+        """
+        test_uei = "ZQGGHJH74DW7"
+
+        with patch("api.uei.SESSION.get") as mock_get:
+            mock_results = json.loads(multiple_uei_results)
+            expected = mock_results["entityData"][0]
+            mock_get.return_value.status_code = 200
+            mock_get.return_value.json.return_value = mock_results
+
+            results = get_uei_info_from_sam_gov(uei=test_uei)
+
+            self.assertTrue(results["valid"])
+            self.assertTrue("errors" not in results)
+            self.assertEqual(results["response"], expected)

backend/api/uei.py

@@ -17,7 +17,7 @@ def proxy_manager_for(self, *args, **kwargs):
         kwargs["ssl_context"] = self.ssl_context
         return super().proxy_manager_for(*args, **kwargs)
 
-    def init_poolmanager(self, connections, maxsize, block=False):
+    def init_poolmanager(self, connections, maxsize, *_args, block=False, **_kwds):
         self.poolmanager = urllib3.poolmanager.PoolManager(
             num_pools=connections,
             maxsize=maxsize,
@@ -52,8 +52,8 @@ def call_sam_api(
         error = "SAM.gov API timeout"
     except requests.exceptions.TooManyRedirects:
         error = "SAM.gov API error - too many redirects"
-    except requests.exceptions.RequestException as e:
-        error = f"Unable to make SAM.gov API request, error: {str(e)}"
+    except requests.exceptions.RequestException as err:
+        error = f"Unable to make SAM.gov API request, error: {str(err)}"
     return None, error
 
 
@@ -73,33 +73,45 @@ def parse_sam_uei_json(response: dict) -> dict:
         return {"valid": False, "errors": ["UEI was not found in SAM.gov"]}
 
     # Ensure there's only one entry:
+    # Nope! 2023-10-05: turns out you can get multiple valid entries.
     entries = response.get("entityData", [])
-    if len(entries) != 1:
-        return {
-            "valid": False,
-            "errors": ["SAM.gov invalid number of entries"],
-        }
+    if len(entries) > 1:
+
+        def is_active(entry):
+            return entry["entityRegistration"]["registrationStatus"] == "Active"
+
+        actives = list(filter(is_active, entries))
+        if actives:
+            entry = actives[0]
+        else:
+            entry = entries[0]
+    elif len(entries) == 1:
+        entry = entries[0]
+    else:
+        return {"valid": False, "errors": ["UEI was not found in SAM.gov"]}
 
     # Get the ueiStatus and catch errors if the JSON shape is unexpected:
     entry = entries[0]
     try:
-        status = entry.get("entityRegistration", {}).get("ueiStatus", "").upper()
+        _ = entry.get("entityRegistration", {}).get("ueiStatus", "").upper()
     except AttributeError:
         return {
             "valid": False,
             "errors": ["SAM.gov unexpected JSON shape"],
         }
 
+    # 2023-10-05 comment out the following, as checking for this creates more
+    # problems for our users than it's worth.
     # Ensure the status is active:
-    if status != "ACTIVE":
-        return {
-            "valid": False,
-            "errors": ["UEI is not listed as active from SAM.gov response data"],
-        }
+    # if status != "ACTIVE":
+    #     return {
+    #         "valid": False,
+    #         "errors": ["UEI is not listed as active from SAM.gov response data"],
+    #     }
 
     # Get the fiscalYearEndCloseDate and catch errors if the JSON shape is unexpected:
     try:
-        status = (
+        _ = (
             entry.get("coreData", {})
             .get("entityInformation", {})
             .get("fiscalYearEndCloseDate", "")
@@ -130,6 +142,22 @@ def get_uei_info_from_sam_gov(uei: str) -> dict:
     # SAM API headers
     api_headers = {"X-Api-Key": SAM_API_KEY}
 
+    # Call the SAM API
+    resp, error = call_sam_api(SAM_API_URL, api_params, api_headers)
+    if resp is None:
+        return {"valid": False, "errors": [error]}
+
+    # Get the response status code
+    if resp.status_code != 200:
+        error = f"SAM.gov API response status code invalid: {resp.status_code}"
+        return {"valid": False, "errors": [error]}
+
+    results = parse_sam_uei_json(resp.json())
+    if results["valid"] and (not results.get("errors")):
+        return results
+
+    # Try again with samRegistered set to No:
+    api_params = api_params | {"samRegistered": "No"}
     # Call the SAM API
     resp, error = call_sam_api(SAM_API_URL, api_params, api_headers)
     if resp is None: