M04N02_using_web_API.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# ATMS 523\n",
        "## Module 4 Lecture 2\n",
        "\n",
        "### Using a Web API to obtain data\n",
        "\n",
        "This example shows you how to use a web API to obtain data from NCEI.  It is the same dataset we obtained within Module 3, but here we are making small calls to get only the data we want."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kp6pa5jNqCSX"
      },
      "outputs": [],
      "source": [
        "#needed to make web requests\n",
        "import requests\n",
        "\n",
        "#store the data we get as a dataframe\n",
        "import pandas as pd\n",
        "\n",
        "#convert the response as a strcuctured json\n",
        "import json\n",
        "\n",
        "#mathematical operations on lists\n",
        "import numpy as np\n",
        "\n",
        "#parse the datetimes we get from NOAA\n",
        "from datetime import datetime\n",
        "\n",
        "#import time for sleeping\n",
        "import time\n",
        "\n",
        "#add the access token you got from NOAA # this is snesbitt's so don't mess with it!\n",
        "Token = 'qGhKPoTezWscLZFQAzmnmBzBkrsiwroe'\n",
        "\n",
        "#Enter data type and station ID from inventory - here is GHCND https://www.ncei.noaa.gov/pub/data/ghcn/daily/ghcnd-stations.txt\n",
        "station_id = 'GHCND:USC00118740'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4bSbsdpEqD5J",
        "outputId": "ba7c34a1-c83e-4685-f705-ff46bd3d6683"
      },
      "outputs": [],
      "source": [
        "#initialize lists to store data\n",
        "dates_mintemp = []\n",
        "dates_maxtemp = []\n",
        "dates_precip = []\n",
        "min_temps = []\n",
        "max_temps = []\n",
        "precip = []\n",
        "\n",
        "#for each year from 1905-2022 where we know we have data inventory ...\n",
        "for year in range(1905, 2023):\n",
        "    year = str(year)\n",
        "    print('working on year '+year)\n",
        "    \n",
        "    #make the api call\n",
        "    r = requests.get('https://www.ncdc.noaa.gov/cdo-web/api/v2/data?datasetid=GHCND&datatypeid=TMIN&datatypeid=TMAX&datatypeid=PRCP&limit=1000&stationid='+station_id+'&startdate='+year+'-01-01&enddate='+year+'-12-31', headers={'token':Token})\n",
        "    #load the api response as a json\n",
        "    d = json.loads(r.text)\n",
        "    #TODO check for empty request\n",
        "\n",
        "    #get all items in the response which are max&min temperature readings\n",
        "    maxtemps = [item for item in d['results'] if item['datatype']=='TMAX']\n",
        "    mintemps = [item for item in d['results'] if item['datatype']=='TMIN']\n",
        "    precips = [item for item in d['results'] if item['datatype']=='PRCP']\n",
        "    #get the date field from all average temperature readings\n",
        "    dates_maxtemp += [item['date'] for item in maxtemps]\n",
        "    dates_mintemp += [item['date'] for item in mintemps]\n",
        "    dates_precip += [item['date'] for item in precips]\n",
        "    #get the actual temperature from the returned data\n",
        "    max_temps += [item['value'] for item in maxtemps]\n",
        "    min_temps += [item['value'] for item in mintemps]\n",
        "    precip += [item['value'] for item in precips]\n",
        "    time.sleep(0.2) # API max 5 requests per second\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NZsW99tWrl-e"
      },
      "outputs": [],
      "source": [
        "#initialize dataframe\n",
        "df_temp_min = pd.DataFrame()\n",
        "df_temp_max = pd.DataFrame()\n",
        "df_precip = pd.DataFrame()\n",
        "\n",
        "#populate date and min and max temperature & precip fields (convert string date to datetime)\n",
        "df_temp_min['date'] = [datetime.strptime(d, \"%Y-%m-%dT%H:%M:%S\") for d in dates_mintemp]\n",
        "df_temp_min['minTemp'] = [float(v)/10.0 for v in min_temps]\n",
        "\n",
        "df_temp_max['date'] = [datetime.strptime(d, \"%Y-%m-%dT%H:%M:%S\") for d in dates_maxtemp]\n",
        "df_temp_max['maxTemp'] = [float(v)/10.0 for v in max_temps]\n",
        "\n",
        "df_precip['date'] = [datetime.strptime(d, \"%Y-%m-%dT%H:%M:%S\") for d in dates_precip]\n",
        "df_precip['precip'] = [float(v)/10.0 for v in precip]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SpZKUtGAxzrP"
      },
      "outputs": [],
      "source": [
        "#merge the dataframes\n",
        "newdf_all = pd.merge(df_temp_max,df_temp_min, left_index=True, right_index=True)\n",
        "newdf_all = pd.merge(newdf_all, df_precip, left_index=True, right_index=True)\n",
        "newdf_all.drop(columns = ['date_x', 'date_y'], inplace=True)\n",
        "newdf_all = newdf_all.set_index('date')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "8v8cZRz75I52",
        "outputId": "da5ff2fa-957b-4692-ce2f-ea715f55f016"
      },
      "outputs": [],
      "source": [
        "newdf_all"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.9.7 ('py3')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "vscode": {
      "interpreter": {
        "hash": "7a6b41cc1ffbe7f6292ade58ee9ab0c89bd7fa770a77f101cc95d5710b4e5fa9"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}