1.scrape_TX_weather_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6e01c63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "230\n",
      "Downloading (main): TME\n",
      "we are here?\n",
      "Downloading (main): ATT\n",
      "we are here?\n",
      "Downloading (main): EDC\n",
      "we are here?\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Source: github.com/akrherz/iem/\n",
    "Example script that scrapes data from the IEM ASOS download service\n",
    "\"\"\"\n",
    "from __future__ import print_function\n",
    "import json\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "\n",
    "# Python 2 and 3: alternative 4\n",
    "try:\n",
    "    from urllib.request import urlopen\n",
    "except ImportError:\n",
    "    from urllib2 import urlopen\n",
    "\n",
    "# Number of attempts to download data\n",
    "MAX_ATTEMPTS = 6\n",
    "# HTTPS here can be problematic for installs that don't have Lets Encrypt CA\n",
    "SERVICE = \"http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?\"\n",
    "\n",
    "\n",
    "def download_data(uri):\n",
    "    \"\"\"Fetch the data from the IEM\n",
    "    The IEM download service has some protections in place to keep the number\n",
    "    of inbound requests in check.  This function implements an exponential\n",
    "    backoff to keep individual downloads from erroring.\n",
    "    Args:\n",
    "      uri (string): URL to fetch\n",
    "    Returns:\n",
    "      string data\n",
    "    \"\"\"\n",
    "    attempt = 0\n",
    "    while attempt < MAX_ATTEMPTS:\n",
    "        try:\n",
    "            data = urlopen(uri, timeout=300).read().decode(\"utf-8\")\n",
    "            if data is not None and not data.startswith(\"ERROR\"):\n",
    "                return data\n",
    "        except Exception as exp:\n",
    "            print(\"download_data(%s) failed with %s\" % (uri, exp))\n",
    "            time.sleep(5)\n",
    "        attempt += 1\n",
    "\n",
    "    print(\"Exhausted attempts to download, returning empty data\")\n",
    "    return \"\"\n",
    "\n",
    "\n",
    "def get_stations_from_filelist(filename):\n",
    "    \"\"\"Build a listing of stations from a simple file listing the stations.\n",
    "    The file should simply have one station per line.\n",
    "    \"\"\"\n",
    "    stations = []\n",
    "    for line in open(filename):\n",
    "        stations.append(line.strip())\n",
    "    return stations\n",
    "\n",
    "\n",
    "def get_stations_from_networks():\n",
    "    \"\"\"Build a station list by using a bunch of IEM networks.\"\"\"\n",
    "    stations = []\n",
    "    states = \"\"\"TX\"\"\"\n",
    "    networks = []\n",
    "    for state in states.split():\n",
    "        networks.append(\"%s_ASOS\" % (state,))\n",
    "\n",
    "    for network in networks:\n",
    "        # Get metadata\n",
    "        uri = (\n",
    "            \"https://mesonet.agron.iastate.edu/geojson/network/%s.geojson\"\n",
    "        ) % (network,)\n",
    "        data = urlopen(uri)\n",
    "        jdict = json.load(data)\n",
    "        for site in jdict[\"features\"]:\n",
    "            stations.append(site[\"properties\"][\"sid\"])\n",
    "    return stations\n",
    "\n",
    "\n",
    "def download_alldata():\n",
    "    \"\"\"An alternative method that fetches all available data.\n",
    "    Service supports up to 24 hours worth of data at a time.\"\"\"\n",
    "    # timestamps in UTC to request data for\n",
    "    startts = datetime.datetime(2021, 3, 15)\n",
    "    endts = datetime.datetime(2022, 4, 30)\n",
    "    interval = datetime.timedelta(hours=24)\n",
    "\n",
    "    service = SERVICE + \"data=skyc1&tz=Etc/UTC&format=comma&latlon=yes&\"\n",
    "\n",
    "    now = startts\n",
    "    while now < endts:\n",
    "        thisurl = service\n",
    "        thisurl += now.strftime(\"year1=%Y&month1=%m&day1=%d&\")\n",
    "        thisurl += (now + interval).strftime(\"year2=%Y&month2=%m&day2=%d&\")\n",
    "        print(thisurl)\n",
    "        print(\"Downloading (alldata): %s\" % (now,))\n",
    "        data = download_data(thisurl)\n",
    "        outfn = \"%s.txt\" % (now.strftime(\"%Y%m%d\"),)\n",
    "        with open(outfn, \"w\") as fh:\n",
    "            fh.write(data)\n",
    "        now += interval\n",
    "\n",
    "\n",
    "def main():\n",
    "    \"\"\"Our main method\"\"\"\n",
    "    # timestamps in UTC to request data for\n",
    "    startts = datetime.datetime(2021, 3, 15)\n",
    "    endts = datetime.datetime(2022, 4, 30)\n",
    "\n",
    "    service = SERVICE + \"data=skyc1&tz=Etc/UTC&format=comma&latlon=yes&\"\n",
    "\n",
    "    service += startts.strftime(\"year1=%Y&month1=%m&day1=%d&\")\n",
    "    service += endts.strftime(\"year2=%Y&month2=%m&day2=%d&\")\n",
    "\n",
    "    # Two examples of how to specify a list of stations\n",
    "    stations = get_stations_from_networks()\n",
    "    print(len(stations))\n",
    "    \n",
    "#     num_stations = 10\n",
    "#     # select 10 random stations\n",
    "#     np.random.seed(1)\n",
    "#     stations = np.random.randint(low=0, high=len(stations))\n",
    "#     stations = 1\n",
    "    #stations = [\"TME\", \"FWS\", \"BSM\"]\n",
    "    stations = [\"TME\", \"ATT\", \"EDC\"]\n",
    "    # stations = get_stations_from_filelist(\"mystations.txt\")\n",
    "    for station in stations:\n",
    "        uri = \"%s&station=%s\" % (service, station)\n",
    "        print(\"Downloading (main): %s\" % (station,))\n",
    "        data = download_data(uri)\n",
    "        outfn = \"%s_%s_%s.txt\" % (\n",
    "            station,\n",
    "            startts.strftime(\"%Y%m%d%H%M\"),\n",
    "            endts.strftime(\"%Y%m%d%H%M\"),\n",
    "        )\n",
    "        out = open(outfn, \"w\")\n",
    "        out.write(data)\n",
    "        out.close()\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    #download_alldata()\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83df93b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# Use the data of one station to fill in missing data of the other station:\n",
    "data0 = pd.read_csv(\"./TX_weather/EDC_202103150000_202204300000.txt\")\n",
    "data1 = pd.read_csv(\"./TX_weather/ATT_202103150000_202204300000.txt\")\n",
    "data2 = pd.read_csv(\"./TX_weather/TME_202103150000_202204300000.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bbd177c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24523 10924 25796\n"
     ]
    }
   ],
   "source": [
    "print(len(data0), len(data1), len(data2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1fce0ebe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n"
     ]
    }
   ],
   "source": [
    "timelist = []\n",
    "skyc1 = []\n",
    "\n",
    "for year in [2021, 2022]:\n",
    "    for month in range(12):\n",
    "        for day in range(31):\n",
    "            for hour in range(24):\n",
    "                for minute in range(60):\n",
    "                    time = str(year) + \"-\" + str(month).zfill(2) + \"-\" + str(day).zfill(2) + \" \" + str(hour).zfill(2) + \":\" + str(minute).zfill(2)\n",
    "                    \n",
    "                    if time in data0.valid.values:                       \n",
    "                        i = data0[(data0.valid == time)].index\n",
    "                        if data0.skyc1[i[0]] != \"M\": # Check for missing value\n",
    "                            skyc1.append(data0.skyc1[i[0]])\n",
    "                            timelist.append(time)\n",
    "\n",
    "                    if time in data1.valid.values:                       \n",
    "                        i = data1[(data1.valid == time)].index\n",
    "                        if data1.skyc1[i[0]] != \"M\": # Check for missing value\n",
    "                            skyc1.append(data1.skyc1[i[0]])\n",
    "                            timelist.append(time)\n",
    "                        \n",
    "                    if time in data2.valid.values:                       \n",
    "                        i = data2[(data2.valid == time)].index\n",
    "                        if data2.skyc1[i[0]] != \"M\": # Check for missing value\n",
    "                            skyc1.append(data2.skyc1[i[0]])\n",
    "                            timelist.append(time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3f5dc023",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"weather_times_march15_april30.txt\", \"w\") as f:\n",
    "    for item in timelist:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "402e565c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"weather_skyc1_march15_april30.txt\", \"w\") as f:\n",
    "    for item in skyc1:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}