Az state cleaner

notebooks/AZ_EDA.ipynb

@@ -12,8 +12,8 @@
     "import plotly.express as px\n",
     "import plotly.graph_objects as go\n",
     "import warnings\n",
-    "\n",
-    "warnings.filterwarnings(\"ignore\")\n",
+    "warnings.filterwarnings('ignore')\n",
+    "from utils.helper_fns import pre_process_az\n",
     "\n",
     "ind_bla, pac_bla, org_bla, bla_cands = pre_process_az()"
    ]
@@ -203,7 +203,7 @@
     }
    ],
    "source": [
-    "# looking at just proper contributors, ignoring 'multiple contributors' in first place\n",
+    "#looking at just proper contributors, ignoring 'multiple contributors' in first place\n",
     "ind_bla[1:11]"
    ]
   },
@@ -364,12 +364,8 @@
     }
    ],
    "source": [
-    "# looking at all expenses by individual contributors, PACs and organizations\n",
-    "top_expenses = (\n",
-    "    ind_bla.append(org_bla)\n",
-    "    .append(pac_bla)\n",
-    "    .sort_values(by=\"total_spending\", ascending=False)\n",
-    ")\n",
+    "#looking at all expenses by individual contributors, PACs and organizations\n",
+    "top_expenses = ind_bla.append(org_bla).append(pac_bla).sort_values(by=\"total_spending\", ascending=False)\n",
     "\n",
     "top_expenses.head(11)"
    ]
@@ -1318,16 +1314,11 @@
     }
    ],
    "source": [
-    "# means\n",
+    "#means\n",
     "\n",
     "grp_cands = cands_14_22.groupby(by=\"Office Summary\").mean()\n",
     "\n",
-    "fig = px.bar(\n",
-    "    x=list(grp_cands.index),\n",
-    "    y=grp_cands[\"Expense\"].values,\n",
-    "    title=\"Mean Candidate Income by Office, 2014-22\",\n",
-    "    labels={\"x\": \"Year\", \"y\": \"US Dollars\"},\n",
-    ")\n",
+    "fig = px.bar(x=list(grp_cands.index), y=grp_cands[\"Expense\"].values, title=\"Mean Candidate Income by Office, 2014-22\", labels = {\"x\":\"Year\", \"y\":\"US Dollars\"})\n",
     "\n",
     "fig.show()"
    ]
@@ -2278,12 +2269,7 @@
    "source": [
     "grp_cands = cands_14_22.groupby(by=\"Office Summary\").sum()\n",
     "\n",
-    "fig = px.bar(\n",
-    "    x=list(grp_cands.index),\n",
-    "    y=grp_cands[\"Expense\"].values,\n",
-    "    title=\"Gross Candidate Income by Office, 2014-22\",\n",
-    "    labels={\"x\": \"Year\", \"y\": \"US Dollars\"},\n",
-    ")\n",
+    "fig = px.bar(x=list(grp_cands.index), y=grp_cands[\"Expense\"].values, title=\"Gross Candidate Income by Office, 2014-22\", labels = {\"x\":\"Year\", \"y\":\"US Dollars\"})\n",
     "\n",
     "fig.show()"
    ]
@@ -3246,16 +3232,11 @@
     }
    ],
    "source": [
-    "# recipient income by year\n",
+    "#recipient income by year\n",
     "\n",
     "by_year_cands_sum = cands_14_22.groupby(by=\"Year\").sum()\n",
     "\n",
-    "fig = px.bar(\n",
-    "    x=list(by_year_cands_sum.index),\n",
-    "    y=by_year_cands_sum[\"Income\"].values,\n",
-    "    title=\"Gross Candidate Income by Year, 2014-22\",\n",
-    "    labels={\"x\": \"Year\", \"y\": \"US Dollars\"},\n",
-    ")\n",
+    "fig = px.bar(x=list(by_year_cands_sum.index), y=by_year_cands_sum[\"Income\"].values, title=\"Gross Candidate Income by Year, 2014-22\", labels = {\"x\":\"Year\", \"y\":\"US Dollars\"})\n",
     "\n",
     "fig.show()"
    ]
@@ -4173,32 +4154,26 @@
     }
    ],
    "source": [
-    "# individual contributions by year\n",
+    "#individual contributions by year\n",
     "\n",
     "grp = inds_14_22.groupby(by=\"Year\").sum()\n",
     "\n",
     "grp[\"total_spending\"].values\n",
     "\n",
-    "fig = go.Figure(\n",
-    "    data=[\n",
-    "        go.Bar(\n",
-    "            name=\"Individual\",\n",
-    "            x=list(range(2014, 2023)),\n",
-    "            y=grp[\"total_spending\"].values,\n",
-    "        )\n",
-    "    ]\n",
-    ")\n",
+    "fig = go.Figure(data = [\n",
+    "    go.Bar(name = \"Individual\", x = list(range(2014, 2023)), y = grp[\"total_spending\"].values, )\n",
+    "])\n",
     "\n",
     "fig.update_layout(\n",
     "    title=\"Gross Individual Contributions by Year, 2014-22\",\n",
     "    xaxis_title=\"Year\",\n",
     "    yaxis_title=\"US Dollars\",\n",
-    "    #     legend_title=\"Legend Title\",\n",
-    "    #     font=dict(\n",
-    "    #         family=\"Courier New, monospace\",\n",
-    "    #         size=18,\n",
-    "    #         color=\"RebeccaPurple\"\n",
-    "    #     )\n",
+    "#     legend_title=\"Legend Title\",\n",
+    "#     font=dict(\n",
+    "#         family=\"Courier New, monospace\",\n",
+    "#         size=18,\n",
+    "#         color=\"RebeccaPurple\"\n",
+    "#     )\n",
     ")\n",
     "\n",
     "fig.show()"
@@ -5169,40 +5144,36 @@
     }
    ],
    "source": [
-    "# overall expenses by type by year\n",
+    "#overall expenses by type by year\n",
     "\n",
-    "donors_14_22 = (\n",
-    "    org_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]]\n",
-    "    .append(pac_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]])\n",
-    "    .append(inds_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]])\n",
-    ")\n",
+    "donors_14_22 = org_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]].append(pac_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]]).append(inds_14_22[[\"Name\", \"total_spending\", \"type\", \"Year\"]])\n",
     "\n",
-    "donors_14_22.groupby(by=[\"type\", \"Year\"]).sum()\n",
+    "donors_14_22.groupby(by = [\"type\", \"Year\"]).sum()\n",
     "\n",
-    "years = [\"2014\", \"2015\", \"2016\", \"2017\", \"2018\", \"2019\", \"2020\", \"2021\", \"2022\"]\n",
+    "years = ['2014', '2015', '2016', '2017', '2018', '2019', '2020', '2021', '2022']\n",
     "\n",
     "yrs = []\n",
     "\n",
-    "df = donors_14_22.groupby(by=[\"type\", \"Year\"]).sum()\n",
+    "df=donors_14_22.groupby(by = [\"type\", \"Year\"]).sum()\n",
     "\n",
     "for i in range(26):\n",
     "    yrs.append(df.take([i]).values[0][0])\n",
     "\n",
-    "fig = go.Figure(\n",
-    "    data=[\n",
-    "        go.Bar(name=\"Individual\", x=years, y=yrs[0:9]),\n",
-    "        go.Bar(name=\"Organization\", x=years, y=[yrs[9]] + [0] + list(yrs[10:17])),\n",
-    "        go.Bar(name=\"PAC\", x=years, y=yrs[17:26]),\n",
-    "    ]\n",
-    ")\n",
+    "fig = go.Figure(data = [\n",
+    "    go.Bar(name = \"Individual\", x = years, y = yrs[0:9]),\n",
+    "    go.Bar(name = \"Organization\", x = years, y = [yrs[9]]+[0]+list(yrs[10:17])),\n",
+    "    go.Bar(name = \"PAC\", x = years, y = yrs[17:26])\n",
+    "])\n",
     "\n",
     "fig.update_layout(\n",
     "    title=\"Gross Overall Expenses by Donor Type by Year, 2014-22\",\n",
     "    xaxis_title=\"Year\",\n",
     "    yaxis_title=\"US Dollars\",\n",
+    "\n",
     ")\n",
     "\n",
-    "fig.show()"
+    "fig.show()\n",
+    "\n"
    ]
   },
   {
@@ -5232,7 +5203,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.12"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

notebooks/README.md

@@ -7,11 +7,14 @@ This should contain information about what is done in each notebook
 
 * `az_webcrawler_3.ipynb` : This is a notebook of test code, including the final code used in `az_curl_crawler.py`
 
-* `arizona_scraper_proof_of_concept` is a notebook containing proof of cocenpt for a curl-based webcrawler, which was later expanded on in `az_webcrawler_3.ipynb` and finally used to make `az_curl_crawler.py`
+* `arizona_scraper_proof_of_concept` is a notebook containing proof of concept for a curl-based webcrawler, which was later expanded on in `az_webcrawler_3.ipynb` and finally used to make `az_curl_crawler.py`
+
 * `mi_campaign_eda.ipynb`: This notebook contains the exploratory data analysis of the Michigan campaign contribution datasets, with a dropdown that allows the user to select different years to view.
 
 * `mi_campaign_expenditure.ipynb`: This notebook contains the exploratory data analysis of the Michigan campaign expenditure datasets, with a dropdown that allows the user to select different years to view.
 
 * `AZ_EDA` : A notebook containing the EDA and plots for Arizona. 
 
 * `PA_EDA.ipynb` : This notebook contains the EDA for Pennsylvania datasets on contributions, filer information, and expenditure data from 2018-2023.
+
+* `az_cleaner_scratch.ipynb` and `az_cleaner_scratch_thanksgiving.ipynb` are notebooks which were used to test parallel versions of the code which concluded in `clean.py` and `cleaner_utils.py`