[chore] small changes - solar pv, nyserda contract kdm completed/date…

…, tendering of interconnection agreement kdm rename
calblueprint · Nov 15, 2024 · a63e806 · a63e806
1 parent 6b0ddad
commit a63e806
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 52 deletions.
diff --git a/api/webscraper/__pycache__/database_constants.cpython-312.pyc b/api/webscraper/__pycache__/database_constants.cpython-312.pyc
diff --git a/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc b/api/webscraper/__pycache__/nyserda_scraper.cpython-312.pyc
diff --git a/api/webscraper/database.py b/api/webscraper/database.py
@@ -31,7 +31,7 @@
 
 def nyserda_large_to_database():
     database = []
-    database.extend(query_nyserda_large())
+    database.extend(query_nyserda_large()[:20])
     for project in database:
         if project.get("proposed_cod", None) is not None:
             ymd = datetime.strptime(project.get("proposed_cod"), "%Y").strftime(
@@ -58,18 +58,12 @@ def nyserda_large_to_database():
                 ]
 
             # update key development milestones
-            if project.get("year_of_delivery_start_date", None) is not None:
-                nyserda_contract_date = datetime.strptime(
-                    project.get("year_of_delivery_start_date"), "%Y"
-                )
-                completed = nyserda_contract_date < datetime.now()
-                nyserda_contract_date = nyserda_contract_date.strftime("%Y-%m-%d")
-                update_object["key_development_milestones"] = update_kdm(
-                    "Winning a contract award from NYSERDA",
-                    date=nyserda_contract_date,
-                    completed=completed,
-                    kdm=update_object["key_development_milestones"],
-                )
+            update_object["key_development_milestones"] = update_kdm(
+                "Winning a contract award from NYSERDA",
+                date=project.get("nyserda_contract_date"),
+                completed=True,
+                kdm=update_object["key_development_milestones"],
+            )
 
             try:
                 response = (
@@ -108,20 +102,12 @@ def nyserda_large_to_database():
                     project["town"] = town
 
             # append key development milestones
-            if project.get("year_of_delivery_start_date", None) is not None:
-                nyserda_contract_date = datetime.strptime(
-                    project.get("year_of_delivery_start_date"), "%Y"
-                )
-                completed = nyserda_contract_date < datetime.now()
-                nyserda_contract_date = nyserda_contract_date.strftime("%Y-%m-%d")
-                project["key_development_milestones"] = update_kdm(
-                    "Winning a contract award from NYSERDA",
-                    date=nyserda_contract_date,
-                    completed=False,
-                    kdm=initial_kdm_dict,
-                )
-            if "year_of_delivery_start_date" in project:
-                del project["year_of_delivery_start_date"]
+            project["key_development_milestones"] = update_kdm(
+                "Winning a contract award from NYSERDA",
+                date=project.get("nyserda_contract_date"),
+                completed=True,
+                kdm=initial_kdm_dict,
+            )
             try:
                 response = (
                     supabase.table("Projects_duplicate").insert(project).execute()
@@ -246,7 +232,7 @@ def nyiso_to_database():
                 completed = ia_date < current_date
                 ia_date = ia_date.strftime("%Y-%m-%d")
                 update_object["key_development_milestones"] = update_kdm(
-                    milestoneTitle="Execution of an Interconnection Agreement (IA)",
+                    milestoneTitle="Tendering of an Interconnection Agreement (IA)",
                     completed=completed,
                     date=ia_date,
                     kdm=update_object["key_development_milestones"],
@@ -282,7 +268,7 @@ def nyiso_to_database():
                 completed = ia_date < current_date
                 ia_date = ia_date.strftime("%Y-%m-%d")
                 project["key_development_milestones"] = update_kdm(
-                    milestoneTitle="Execution of an Interconnection Agreement (IA)",
+                    milestoneTitle="Tendering of an Interconnection Agreement (IA)",
                     completed=completed,
                     date=ia_date,
                     kdm=project["key_development_milestones"],
@@ -354,7 +340,7 @@ def nyiso_in_service_to_database():
                 completed = ia_date < current_date
                 ia_date = ia_date.strftime("%Y-%m-%d")
                 update_object["key_development_milestones"] = update_kdm(
-                    milestoneTitle="Execution of an Interconnection Agreement (IA)",
+                    milestoneTitle="Tendering of an Interconnection Agreement (IA)",
                     completed=completed,
                     date=ia_date,
                     kdm=update_object["key_development_milestones"],
@@ -390,7 +376,7 @@ def nyiso_in_service_to_database():
                 completed = ia_date < current_date
                 ia_date = ia_date.strftime("%Y-%m-%d")
                 project["key_development_milestones"] = update_kdm(
-                    milestoneTitle="Execution of an Interconnection Agreement (IA)",
+                    milestoneTitle="Tendering of an Interconnection Agreement (IA)",
                     completed=completed,
                     date=ia_date,
                     kdm=project["key_development_milestones"],

diff --git a/api/webscraper/database_constants.py b/api/webscraper/database_constants.py
@@ -1,16 +1,18 @@
+# maps renewable energy strings to our app's renewable energy names
 renewable_energy_map = {
     "Hydroelectric": "Hydroelectric",
     "Land Based Wind": "Land-Based Wind",
     "Offshore Wind": "Offshore Wind",
-    "Solar": "Solar",
+    "Solar": "Solar PV",
     "Geothermal": "Geothermal",
     "Energy Storage": "Energy Storage",
     "Pumped Storage": "Pumped Storage",
 }
 
+# maps NYISO abbreviations to renewable energy strings
 renewable_energy_abbreviations = {
     "H": "Hydroelectric",
-    "S": "Solar",
+    "S": "Solar PV",
     "ES": "Energy Storage",
     "PS": "Pumped Storage",
     "W": "Land-Based Wind",
@@ -35,7 +37,7 @@
         "date": None,
     },
     {
-        "milestoneTitle": "Execution of an Interconnection Agreement (IA)",
+        "milestoneTitle": "Tendering of an Interconnection Agreement (IA)",
         "completed": False,
         "date": None,
     },
@@ -51,3 +53,26 @@
         "date": None,
     },
 ]
+
+project_fields = [
+    "project_name",
+    "renewable_energy_technology",
+    "project_status",
+    "developer",
+    "town",
+    "county",
+    "region",
+    "size",
+    "latitude",
+    "longitude",
+    "key_development_milestones",
+    "project_image",
+    "interconnection_queue_number",
+    "approved",
+    "state_senate_district",
+    "assembly_district",
+    "zipcode",
+    "proposed_cod",
+    "permit_process",
+    "permit_application_number",
+]
diff --git a/api/webscraper/nyserda_scraper.py b/api/webscraper/nyserda_scraper.py
@@ -10,6 +10,17 @@
 """
 
 
+def solicitation_name_to_date(solicitation_name):
+    if solicitation_name is None:
+        return None
+    if "-" not in solicitation_name:
+        return None
+    else:
+        parts = solicitation_name.split("-")
+        year = parts[0][-2::]
+        return f"%20{year}-%01-%01"
+
+
 def query_nyserda_large():
     nyserda_large_response = requests.get("https://data.ny.gov/resource/dprp-55ye.json")
     if nyserda_large_response.status_code != 200:
@@ -60,8 +71,8 @@ def query_nyserda_large():
                     "project_image": None,
                     "approved": False,
                     # used for updating the kdms
-                    "year_of_delivery_start_date": item.get(
-                        "year_of_delivery_start_date", None
+                    "nyserda_contract_date": solicitation_name_to_date(
+                        item.get("solicitation_name", None)
                     ),
                 }
                 filtered_list.append(project_dict)

diff --git a/api/webscraper/utils/scraper_utils.py b/api/webscraper/utils/scraper_utils.py
@@ -33,22 +33,6 @@ def geocode_lat_long(address):
     return latitude, longitude
 
 
-project_fields = [
-    "project_name",
-    "project_status",
-    "developer",
-    "county",
-    "region",
-    "size",
-    "latitude",
-    "longitude",
-    "key_development_milestones",
-    "project_image",
-    "interconnection_queue_number",
-    "approved",
-]
-
-
 def create_update_object(existing_project, new_project):
     update_object = {}
     for key, value in existing_project.items():