grab_OOI_with_DO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Housekeeping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/ben/OneDrive - King County/python_data_science/ocean_hack_week\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load libraries\n",
    "import requests\n",
    "import time\n",
    "import json\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import re\n",
    "import requests\n",
    "import xarray as xr\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enter credentials\n",
    "username = 'OOIAPI-FVEH1J9ENOAPBW'\n",
    "token = 'TXTYLJ2Q89HUB2'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the data from the server\n",
    "Specify data identifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference designator for Seafloor oxygen of in-shore surface mooring\n",
    "refdes = 'CE06ISSM-MFD37-03-DOSTAD000'\n",
    "method = 'recovered_inst'\n",
    "stream = 'dosta_abcdjm_ctdbp_instrument_recovered'\n",
    "beginDT = '1990-01-01T01:01:01.500Z' # this time range encompasses all three deployments of this instrument.\n",
    "endDT = '2018-09-01T01:01:01.500Z'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build the GET request URL and send the request to the M2M API endpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://ooinet.oceanobservatories.org/api/m2m/12576/sensor/inv/CE06ISSM/MFD37/03-DOSTAD000/recovered_inst/dosta_abcdjm_ctdbp_instrument_recovered\n"
     ]
    }
   ],
   "source": [
    "base_url = 'https://ooinet.oceanobservatories.org/api/m2m/12576/sensor/inv'\n",
    "data_request_url ='/'.join((base_url,refdes[:8],refdes[9:14],refdes[15:],method,stream))\n",
    "print(data_request_url)\n",
    "params = {\n",
    "    'beginDT':beginDT,\n",
    "    'endDT':endDT,   \n",
    "}\n",
    "r = requests.get(data_request_url, params=params, auth=(username, token))\n",
    "data = r.json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Examine the response\n",
    "Notice that the timeCalculation key is attempting to give you a rough estimate of how long it will take to fulfill your data request."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "dict_keys(['requestUUID', 'outputURL', 'allURLs', 'sizeCalculation', 'timeCalculation', 'numberOfSubJobs'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'https://opendap.oceanobservatories.org/thredds/catalog/ooi/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered/catalog.html'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"requestUUID\": \"ca66c692-afbc-484b-a7ea-5ab8d5cb3d47\",\n",
      "  \"outputURL\": \"https://opendap.oceanobservatories.org/thredds/catalog/ooi/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered/catalog.html\",\n",
      "  \"allURLs\": [\n",
      "    \"https://opendap.oceanobservatories.org/thredds/catalog/ooi/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered/catalog.html\",\n",
      "    \"https://opendap.oceanobservatories.org/async_results/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered\"\n",
      "  ],\n",
      "  \"sizeCalculation\": 5707523,\n",
      "  \"timeCalculation\": 60,\n",
      "  \"numberOfSubJobs\": 529\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "type(data)\n",
    "data.keys()\n",
    "data['allURLs'][0]\n",
    "print(json.dumps(data, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Navigating your THREDDS directory.\n",
    "Check if the request has completed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "request completed\n",
      "CPU times: user 1.95 s, sys: 133 ms, total: 2.08 s\n",
      "Wall time: 2min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Build the URL that contains the status, and ping it until it confirms the request is complete\n",
    "check_complete = data['allURLs'][1] + '/status.txt'\n",
    "# Loop through status for 1800 seconds\n",
    "for i in range(1800): \n",
    "    r = requests.get(check_complete)\n",
    "    if r.status_code == requests.codes.ok:\n",
    "        print('request completed')\n",
    "        break\n",
    "    else:\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check out the THREDDS directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://opendap.oceanobservatories.org/thredds/catalog/ooi/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered/catalog.html'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['allURLs'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check out the Apache directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://opendap.oceanobservatories.org/async_results/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['allURLs'][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parse the thredds server to get a list of all NetCDF files. Each deployment is seperated into a seperate netcdf file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set the threds url recovered from the \n",
    "url = data['allURLs'][0]\n",
    "tds_url = 'https://opendap.oceanobservatories.org/thredds/dodsC'\n",
    "datasets = requests.get(url).text\n",
    "# Regular expression to get all the urls\n",
    "urls = re.findall(r'href=[\\'\"]?([^\\'\" >]+)', datasets)\n",
    "# Find the urls with nc extensions\n",
    "x = re.findall(r'(ooi/.*?.nc)', datasets)\n",
    "for i in x:\n",
    "    if i.endswith('.nc') == False:\n",
    "        x.remove(i)\n",
    "for i in x:\n",
    "    try:\n",
    "        float(i[-4])\n",
    "    except:\n",
    "        x.remove(i)\n",
    "datasets = [os.path.join(tds_url, i) for i in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'https://opendap.oceanobservatories.org/thredds/dodsC/ooi/ben.larson@noaa.gov/20180824T163446-CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered/deployment0005_CE06ISSM-MFD37-03-DOSTAD000-recovered_inst-dosta_abcdjm_ctdbp_instrument_recovered_20160927T164503-20170328T093510.nc'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect the result\n",
    "len(datasets)\n",
    "datasets[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Further prune the data to remove co-located sensor data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_sub = []\n",
    "for i in datasets:\n",
    "    if '03-CTDBPC000' in i:\n",
    "        pass\n",
    "    else:\n",
    "        datasets_sub.append(i) \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore the data in Python using xarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ds = xr.open_mfdataset(datasets_sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:                       (obs: 67633)\n",
       "Coordinates:\n",
       "  * obs                           (obs) int64 0 1 2 3 4 5 6 7 8 9 10 11 12 ...\n",
       "    time                          (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    int_ctd_pressure              (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    lat                           (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    lon                           (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "Data variables:\n",
       "    deployment                    (obs) int32 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    id                            (obs) |S64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ctd_time                      (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    driver_timestamp              (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ingestion_timestamp           (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    internal_timestamp            (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    oxygen                        (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    port_timestamp                (obs) datetime64[ns] dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    preferred_timestamp           (obs) object dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    provenance                    (obs) |S64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ctd_tc_oxygen                 (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    practical_salinity            (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ctdbp_seawater_temperature    (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    dissolved_oxygen              (obs) float64 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ctd_tc_oxygen_qc_executed     (obs) float32 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    ctd_tc_oxygen_qc_results      (obs) float32 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    dissolved_oxygen_qc_executed  (obs) float32 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "    dissolved_oxygen_qc_results   (obs) float32 dask.array<shape=(67633,), chunksize=(11787,)>\n",
       "Attributes:\n",
       "    _NCProperties:                      version=1|netcdflibversion=4.4.1.1|hd...\n",
       "    node:                               MFD37\n",
       "    comment:                            \n",
       "    publisher_email:                    \n",
       "    sourceUrl:                          http://oceanobservatories.org/\n",
       "    collection_method:                  recovered_inst\n",
       "    stream:                             dosta_abcdjm_ctdbp_instrument_recovered\n",
       "    featureType:                        point\n",
       "    creator_email:                      \n",
       "    publisher_name:                     Ocean Observatories Initiative\n",
       "    date_modified:                      2018-08-24T16:38:41.433659\n",
       "    keywords:                           \n",
       "    cdm_data_type:                      Point\n",
       "    references:                         More information can be found at http...\n",
       "    Metadata_Conventions:               Unidata Dataset Discovery v1.0\n",
       "    date_created:                       2018-08-24T16:38:41.433657\n",
       "    id:                                 CE06ISSM-MFD37-03-DOSTAD000-recovered...\n",
       "    requestUUID:                        103017aa-5c2d-41a5-819d-5f2c2c74ec6a\n",
       "    contributor_role:                   \n",
       "    summary:                            Dataset Generated by Stream Engine fr...\n",
       "    keywords_vocabulary:                \n",
       "    institution:                        Ocean Observatories Initiative\n",
       "    naming_authority:                   org.oceanobservatories\n",
       "    feature_Type:                       point\n",
       "    infoUrl:                            http://oceanobservatories.org/\n",
       "    license:                            \n",
       "    contributor_name:                   \n",
       "    uuid:                               103017aa-5c2d-41a5-819d-5f2c2c74ec6a\n",
       "    creator_name:                       Ocean Observatories Initiative\n",
       "    title:                              Data produced by Stream Engine versio...\n",
       "    sensor:                             03-DOSTAD000\n",
       "    standard_name_vocabulary:           NetCDF Climate and Forecast (CF) Meta...\n",
       "    acknowledgement:                    \n",
       "    Conventions:                        CF-1.6\n",
       "    project:                            Ocean Observatories Initiative\n",
       "    source:                             CE06ISSM-MFD37-03-DOSTAD000-recovered...\n",
       "    publisher_url:                      http://oceanobservatories.org/\n",
       "    creator_url:                        http://oceanobservatories.org/\n",
       "    nodc_template_version:              NODC_NetCDF_TimeSeries_Orthogonal_Tem...\n",
       "    subsite:                            CE06ISSM\n",
       "    processing_level:                   L2\n",
       "    history:                            2018-08-24T16:38:41.433625 generated ...\n",
       "    Manufacturer:                       Aanderaa\n",
       "    ModelNumber:                        Optode 4831\n",
       "    SerialNumber:                       313\n",
       "    Description:                        Dissolved Oxygen Stable Response: DOS...\n",
       "    FirmwareVersion:                    Not specified.\n",
       "    SoftwareVersion:                    Not specified.\n",
       "    AssetUniqueID:                      CGINS-DOSTAD-00313\n",
       "    Notes:                              Not specified.\n",
       "    Owner:                              Not specified.\n",
       "    RemoteResources:                    []\n",
       "    ShelfLifeExpirationDate:            Not specified.\n",
       "    Mobile:                             False\n",
       "    AssetManagementRecordLastModified:  2018-08-23T12:36:31.728000\n",
       "    time_coverage_start:                2017-04-14T00:00:01\n",
       "    time_coverage_end:                  2017-08-28T10:15:01\n",
       "    time_coverage_resolution:           P1000.03S\n",
       "    geospatial_lat_min:                 47.13408\n",
       "    geospatial_lat_max:                 47.13408\n",
       "    geospatial_lat_units:               degrees_north\n",
       "    geospatial_lat_resolution:          0.1\n",
       "    geospatial_lon_min:                 -124.27187\n",
       "    geospatial_lon_max:                 -124.27187\n",
       "    geospatial_lon_units:               degrees_east\n",
       "    geospatial_lon_resolution:          0.1\n",
       "    geospatial_vertical_units:          meters\n",
       "    geospatial_vertical_resolution:     0.1\n",
       "    geospatial_vertical_positive:       down\n",
       "    DODS.strlen:                        36\n",
       "    DODS.dimName:                       string36\n",
       "    DODS_EXTRA.Unlimited_Dimension:     obs"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = xr.open_mfdataset(datasets)\n",
    "ds = ds.swap_dims({'obs': 'time'})\n",
    "ds = ds.chunk({'time': 100})\n",
    "ds = ds.sortby('time') # data from different deployments can overlap so we want to sort all data by time stamp.\n",
    "ds"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}