From 1ad410f6c779fbf80176946e1721d4ad1b0e83e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 6 Aug 2024 11:33:10 +0200 Subject: [PATCH 1/5] MNT: do not use wetterdienst, revert to plain html download --- notebooks/basics/wradlib_workflow.ipynb | 6 +- .../fileio/radolan/radolan_network.ipynb | 75 +++++-- notebooks/workflow/recipe4.ipynb | 208 +++++++++--------- 3 files changed, 161 insertions(+), 128 deletions(-) diff --git a/notebooks/basics/wradlib_workflow.ipynb b/notebooks/basics/wradlib_workflow.ipynb index d382fc76..7505861e 100644 --- a/notebooks/basics/wradlib_workflow.ipynb +++ b/notebooks/basics/wradlib_workflow.ipynb @@ -593,8 +593,8 @@ "metadata": {}, "outputs": [], "source": [ - "xgrid = np.linspace(R.x.min(), R.x.mean(), 100)\n", - "ygrid = np.linspace(R.y.min(), R.y.mean(), 100)\n", + "xgrid = np.linspace(R.x.min().values, R.x.mean().values, 100)\n", + "ygrid = np.linspace(R.y.min().values, R.y.mean().values, 100)\n", "cart = xr.Dataset(coords={\"x\": ([\"x\"], xgrid), \"y\": ([\"y\"], ygrid)})\n", "cart" ] @@ -964,7 +964,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.12.4" }, "toc": { "colors": { diff --git a/notebooks/fileio/radolan/radolan_network.ipynb b/notebooks/fileio/radolan/radolan_network.ipynb index 5be00cfd..ddb60994 100644 --- a/notebooks/fileio/radolan/radolan_network.ipynb +++ b/notebooks/fileio/radolan/radolan_network.ipynb @@ -38,8 +38,6 @@ "import matplotlib as mpl\n", "import warnings\n", "from IPython import get_ipython\n", - "from wetterdienst.provider.dwd.radar import DwdRadarParameter, DwdRadarValues\n", - "from wetterdienst.provider.dwd.radar.api import DwdRadarSites\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "try:\n", @@ -56,16 +54,39 @@ "metadata": {}, "outputs": [], "source": [ - "# load radolan data\n", - "start_date = datetime.datetime.utcnow()\n", + "import urllib3\n", + "import os\n", + "import io\n", + "import glob\n", + "import shutil\n", + "import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = \"raa01-rw_10000-latest-dwd---bin.bz2\"\n", + "opendata_url1 = f\"https://opendata.dwd.de/weather/radar/radolan/rw/\"\n", "\n", - "radar_data = DwdRadarValues(\n", - " parameter=DwdRadarParameter.RADOLAN_CDC.RW_REFLECTIVITY,\n", - " start_date=start_date - datetime.timedelta(hours=2),\n", - " end_date=start_date,\n", - ")\n", + "import certifi\n", "\n", - "results = radar_data.query()" + "http = urllib3.PoolManager(cert_reqs=\"CERT_REQUIRED\", ca_certs=certifi.where())\n", + "with http.request(\n", + " \"GET\", os.path.join(opendata_url1, f), preload_content=False\n", + ") as r, open(f, \"wb\") as out:\n", + " shutil.copyfileobj(r, out)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!bzip2 -df $f" ] }, { @@ -74,7 +95,7 @@ "metadata": {}, "outputs": [], "source": [ - "rwdata, rwattrs = wrl.io.read_radolan_composite(next(results).data)" + "rwdata, rwattrs = wrl.io.read_radolan_composite(f[:-4])" ] }, { @@ -137,8 +158,26 @@ "metadata": {}, "outputs": [], "source": [ - "# get radar dict\n", - "radars = DwdRadarSites()" + "radars = dict(\n", + " asb=dict(name=\"ASR Borkum\", lat=53.564011, lon=6.748292, alt=36.0),\n", + " boo=dict(name=\"Boostedt\", lat=54.00438, lon=10.04687, alt=124.56),\n", + " drs=dict(name=\"Dresden\", lat=51.12465, lon=13.76865, alt=263.36),\n", + " eis=dict(name=\"Eisberg\", lat=49.54066, lon=12.40278, alt=798.79),\n", + " emd=dict(name=\"Emden\", lat=53.33872, lon=7.02377, alt=58.0),\n", + " ess=dict(name=\"Essen\", lat=51.40563, lon=6.96712, alt=185.1),\n", + " fbg=dict(name=\"Feldberg\", lat=47.87361, lon=8.00361, alt=1516.1),\n", + " fld=dict(name=\"Flechtdorf\", lat=51.31120, lon=8.802, alt=627.88),\n", + " hnr=dict(name=\"Hannover\", lat=52.46008, lon=9.69452, alt=97.66),\n", + " neu=dict(name=\"Neuhaus\", lat=50.50012, lon=11.13504, alt=878.04),\n", + " nhb=dict(name=\"Neuheilenbach\", lat=50.10965, lon=6.54853, alt=585.84),\n", + " oft=dict(name=\"Offenthal\", lat=49.9847, lon=8.71293, alt=245.8),\n", + " pro=dict(name=\"Prötzel\", lat=52.64867, lon=13.85821, alt=193.92),\n", + " mem=dict(name=\"Memmingen\", lat=48.04214, lon=10.21924, alt=724.4),\n", + " ros=dict(name=\"Rostock\", lat=54.17566, lon=12.05808, alt=37.0),\n", + " isn=dict(name=\"Isen\", lat=48.17470, lon=12.10177, alt=677.77),\n", + " tur=dict(name=\"Türkheim\", lat=48.58528, lon=9.78278, alt=767.62),\n", + " umd=dict(name=\"Ummendorf\", lat=52.16009, lon=11.17609, alt=183.0),\n", + ")" ] }, { @@ -148,7 +187,7 @@ "outputs": [], "source": [ "def plot_radar(radar, ax, proj):\n", - " site = (radar[\"longitude\"], radar[\"latitude\"], radar[\"heightantenna\"])\n", + " site = (radar[\"lon\"], radar[\"lat\"], radar[\"alt\"])\n", "\n", " # build polygons for maxrange rangering\n", " polygons = wrl.georef.spherical_to_polyvert(r, az, 0, site, crs=proj)\n", @@ -168,7 +207,7 @@ "\n", " # plot radar location and information text\n", " ax.plot(x_loc, y_loc, \"r+\")\n", - " ax.text(x_loc, y_loc, radar[\"location\"], color=\"r\")" + " ax.text(x_loc, y_loc, radar[\"name\"], color=\"r\")" ] }, { @@ -193,7 +232,7 @@ " # get radar coords etc from dict\n", " if radar_id == \"mhp\":\n", " continue\n", - " radar = radars.by_odimcode(radar_id)\n", + " radar = radars[radar_id]\n", " plot_radar(radar, ax1, proj_wgs)" ] }, @@ -222,7 +261,7 @@ " # get radar coords etc from dict\n", " if radar_id == \"mhp\":\n", " continue\n", - " radar = radars.by_odimcode(radar_id)\n", + " radar = radars[radar_id]\n", " plot_radar(radar, ax2, proj_stereo)" ] } @@ -239,7 +278,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.12.4" }, "toc": { "colors": { diff --git a/notebooks/workflow/recipe4.ipynb b/notebooks/workflow/recipe4.ipynb index 27041f00..08796246 100644 --- a/notebooks/workflow/recipe4.ipynb +++ b/notebooks/workflow/recipe4.ipynb @@ -46,6 +46,8 @@ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import xarray as xr\n", + "import xradar\n", + "from datatree import DataTree, open_datatree\n", "\n", "try:\n", " get_ipython().run_line_magic(\"matplotlib inline\")\n", @@ -68,28 +70,29 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Download radar volumes of latest 30 minutes from server using `wetterdienst`\n", + "from html.parser import HTMLParser\n", + "\n", "\n", + "class DWDHTMLParser(HTMLParser):\n", + " def handle_starttag(self, tag, attrs):\n", + " if tag != \"a\":\n", + " return\n", + " self.links.append(attrs[0][1])\n", "\n", - "`wetterdienst` is a neat package for easy retrieval of data primarily from DWD. For further information have a look at their [documentation](https://wetterdienst.readthedocs.io/)." + "\n", + "parser = DWDHTMLParser()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from wetterdienst.provider.dwd.radar import (\n", - " DwdRadarDataFormat,\n", - " DwdRadarDataSubset,\n", - " DwdRadarParameter,\n", - " DwdRadarValues,\n", - ")\n", - "from wetterdienst.provider.dwd.radar.sites import DwdRadarSite" + "# Download data from opendata.dwd.de" ] }, { @@ -98,40 +101,42 @@ "metadata": {}, "outputs": [], "source": [ - "elevations = range(10)\n", + "radar = \"ESS\"\n", + "DBZH = \"sweep_vol_z\"\n", + "VRADH = \"sweep_vol_v\"\n", "\n", - "end_date = datetime.datetime.utcnow()\n", - "start_date = end_date - datetime.timedelta(minutes=30)\n", + "import certifi\n", "\n", - "results_velocity = []\n", - "results_reflectivity = []\n", + "opendata_url1 = f\"https://opendata.dwd.de/weather/radar/sites/{DBZH}/{radar.lower()}/hdf5/filter_polarimetric/\"\n", "\n", - "for el in elevations:\n", - " # Horizontal Doppler Velocity\n", - " request_velocity = DwdRadarValues(\n", - " parameter=DwdRadarParameter.SWEEP_VOL_VELOCITY_H,\n", - " start_date=start_date,\n", - " end_date=end_date,\n", - " site=DwdRadarSite.ESS,\n", - " elevation=el,\n", - " fmt=DwdRadarDataFormat.HDF5,\n", - " subset=DwdRadarDataSubset.POLARIMETRIC,\n", - " )\n", + "http = urllib3.PoolManager(cert_reqs=\"CERT_REQUIRED\", ca_certs=certifi.where())\n", + "response = http.request(\"GET\", opendata_url1).data.decode(\"utf-8\")\n", "\n", - " # Horizontal Reflectivity\n", - " request_reflectivity = DwdRadarValues(\n", - " parameter=DwdRadarParameter.SWEEP_VOL_REFLECTIVITY_H,\n", - " start_date=start_date,\n", - " end_date=end_date,\n", - " elevation=el,\n", - " site=DwdRadarSite.ESS,\n", - " fmt=DwdRadarDataFormat.HDF5,\n", - " subset=DwdRadarDataSubset.POLARIMETRIC,\n", - " )\n", + "parser.links = []\n", + "parser.feed(response)\n", + "filelist1 = parser.links[1:]\n", + "\n", + "filelist1.sort(key=lambda x: x.split(\"-\")[2])\n", + "filelist1.reverse()\n", + "\n", + "opendata_url2 = f\"https://opendata.dwd.de/weather/radar/sites/{VRADH}/{radar.lower()}/hdf5/filter_polarimetric/\"\n", + "\n", + "http = urllib3.PoolManager(cert_reqs=\"CERT_REQUIRED\", ca_certs=certifi.where())\n", + "response = http.request(\"GET\", opendata_url2).data.decode(\"utf-8\")\n", "\n", - " # Submit requests.\n", - " results_velocity.append(request_velocity.query())\n", - " results_reflectivity.append(request_reflectivity.query())" + "parser.links = []\n", + "parser.feed(response)\n", + "filelist2 = parser.links[1:]\n", + "\n", + "filelist2.sort(key=lambda x: x.split(\"-\")[2])\n", + "filelist2.reverse()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up local folder" ] }, { @@ -140,16 +145,16 @@ "metadata": {}, "outputs": [], "source": [ - "import wetterdienst\n", - "\n", - "wetterdienst.__version__" + "flist = glob.glob(\"ras07*\")\n", + "for f in flist:\n", + " os.remove(f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Acquire data as memory buffer" + "## Download latest 24 volumes to current directory" ] }, { @@ -158,13 +163,17 @@ "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "volume_velocity = []\n", - "for item1 in results_velocity:\n", - " files = []\n", - " for item2 in item1:\n", - " files.append(item2.data)\n", - " volume_velocity.append(files)" + "for f in filelist1[: 10 * 24]:\n", + " with http.request(\n", + " \"GET\", os.path.join(opendata_url1, f), preload_content=False\n", + " ) as r, open(f, \"wb\") as out:\n", + " shutil.copyfileobj(r, out)\n", + "\n", + "for f in filelist2[: 10 * 24]:\n", + " with http.request(\n", + " \"GET\", os.path.join(opendata_url2, f), preload_content=False\n", + " ) as r, open(f, \"wb\") as out:\n", + " shutil.copyfileobj(r, out)" ] }, { @@ -173,8 +182,8 @@ "metadata": {}, "outputs": [], "source": [ - "volume_velocity = [v[-6:] for v in volume_velocity]\n", - "volume_velocity = np.array(volume_velocity).T.tolist()" + "volume_reflectivity = glob.glob(\"ras07*_dbzh_*\")\n", + "volume_velocity = glob.glob(\"ras07*_vradh_*\")" ] }, { @@ -183,13 +192,11 @@ "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "volume_reflectivity = []\n", - "for item1 in results_reflectivity:\n", - " files = []\n", - " for item2 in item1:\n", - " files.append(item2.data)\n", - " volume_reflectivity.append(files)" + "volume_reflectivity = np.array(\n", + " sorted(volume_reflectivity, key=lambda x: x.split(\"-\")[2])\n", + ")\n", + "volume_velocity = np.array(sorted(volume_velocity, key=lambda x: x.split(\"-\")[2]))\n", + "volume_reflectivity[:15]" ] }, { @@ -198,36 +205,32 @@ "metadata": {}, "outputs": [], "source": [ - "volume_reflectivity = [v[-6:] for v in volume_reflectivity]\n", - "volume_reflectivity = np.array(volume_reflectivity).T.tolist()" + "volume_reflectivity = volume_reflectivity.reshape(-1, 10).T\n", + "volume_velocity = volume_velocity.reshape(-1, 10).T" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Read the data into xarray powered structure" + "volume_reflectivity = sorted(\n", + " volume_reflectivity, key=lambda x: x[0].split(\"-\")[1].split(\"_\")[-1]\n", + ")\n", + "volume_reflectivity = volume_reflectivity[5:] + volume_reflectivity[:5]\n", + "volume_velocity = sorted(\n", + " volume_velocity, key=lambda x: x[0].split(\"-\")[1].split(\"_\")[-1]\n", + ")\n", + "volume_velocity = volume_velocity[5:] + volume_velocity[:5]\n", + "len(volume_reflectivity)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from datatree import DataTree, open_datatree\n", - "import xradar\n", - "\n", - "\n", - "def concat_radar_datatree(objs, dim=\"volume_time\"):\n", - " root_ds = [obj[\"/\"].ds for obj in objs]\n", - " root = xr.concat(root_ds, dim=dim)\n", - " dtree = DataTree(data=root, name=\"root\")\n", - " for grp in objs[0].groups[1:]:\n", - " ngrps = [obj[grp[1:]].ds for obj in objs]\n", - " ngrp = xr.concat(ngrps, dim=dim)\n", - " DataTree(ngrp, name=grp[1:], parent=dtree)\n", - " return dtree" + "## Read the data into xarray powered structure" ] }, { @@ -249,30 +252,16 @@ " xr.open_dataset(v0, engine=\"odim\", group=\"sweep_0\", reindex_angle=reindex_angle)\n", " for v0 in v\n", " ]\n", - " ds = [xr.merge([r0, v0], compat=\"override\") for r0, v0 in zip(ds0, ds1)]\n", - " ds.insert(0, xr.open_dataset(r[0], group=\"/\"))\n", + " ds = xr.concat(\n", + " [\n", + " xr.merge([r0, v0], compat=\"override\").assign(volume_time=r0.time.min())\n", + " for r0, v0 in zip(ds0, ds1)\n", + " ],\n", + " \"volume_time\",\n", + " )\n", " dsl.append(ds)\n", - "# this takes some private functions from xradar, take care here\n", - "trees = [\n", - " DataTree(data=xradar.io.backends.common._assign_root(ds), name=\"root\") for ds in dsl\n", - "]\n", - "trees = [\n", - " xradar.io.backends.common._attach_sweep_groups(tree, ds[1:])\n", - " for tree, ds in zip(trees, dsl)\n", - "]\n", - "vol = concat_radar_datatree(trees, dim=\"volume_time\")\n", - "# align sweep_numbers to cover for single sweep single moment layout of DWD\n", - "for i, swp in enumerate(vol.groups[1:]):\n", - " vol[swp][\"sweep_number\"] = i" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vol" + "\n", + "dsl.insert(0, xr.open_dataset(volume_reflectivity[0][0], group=\"/\"))" ] }, { @@ -281,7 +270,12 @@ "metadata": {}, "outputs": [], "source": [ - "vol[\"sweep_9\"]" + "tree = DataTree(data=xradar.io.backends.common._assign_root(dsl), name=\"root\")\n", + "vol = xradar.io.backends.common._attach_sweep_groups(tree, dsl[1:])\n", + "for i, swp in enumerate(vol.groups[1:]):\n", + " vol[swp][\"sweep_number\"] = i\n", + "# sets arbitrary volume_time in root group to enable isel over groups\n", + "vol = vol.assign_coords(volume_time=vol[\"sweep_0\"].volume_time)" ] }, { @@ -596,7 +590,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.12.4" } }, "nbformat": 4, From 0ea8207729ae1448920e03f686e8e917a2bbca34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 6 Aug 2024 12:13:48 +0200 Subject: [PATCH 2/5] remove wetterdienst from notebooks.yml --- ci/requirements/notebooks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/requirements/notebooks.yml b/ci/requirements/notebooks.yml index 6e294e8a..3fe964a5 100644 --- a/ci/requirements/notebooks.yml +++ b/ci/requirements/notebooks.yml @@ -37,7 +37,6 @@ dependencies: - semver - setuptools - tqdm - - wetterdienst - xarray - xmltodict - xradar From d44db216ba4359ff15ed164f2ae9a90b7f51de86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 6 Aug 2024 12:16:23 +0200 Subject: [PATCH 3/5] use Python 3.12 --- .github/workflows/main.yml | 6 +++--- .github/workflows/render.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6de8f86b..22170c98 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.12"] steps: - uses: actions/checkout@v3 with: @@ -90,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.12"] steps: - uses: actions/checkout@v3 with: @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.12"] steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/render.yml b/.github/workflows/render.yml index 34c5f86f..b9773234 100644 --- a/.github/workflows/render.yml +++ b/.github/workflows/render.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + python-version: ["3.12"] steps: - uses: actions/checkout@v3 with: From b262eee1218c8f503f310facb7d868ba085793aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 6 Aug 2024 13:12:22 +0200 Subject: [PATCH 4/5] add bearer token back into CI --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22170c98..51c173be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,6 +57,8 @@ jobs: python -c "import wradlib; print(wradlib.version.version)" python -c "import wradlib; print(wradlib.show_versions())" - name: Render with pytest + env: + WRADLIB_EARTHDATA_BEARER_TOKEN: ${{ secrets.WRADLIB_EARTHDATA_BEARER_TOKEN }} run: | export WRADLIB_DATA=`realpath $WRADLIB_DATA` pytest -n auto --verbose --durations=15 --pyargs notebooks From eb41bab3673ab9f1155ac341fd13325349ce3e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Tue, 6 Aug 2024 13:15:40 +0200 Subject: [PATCH 5/5] add bearer token back into CI --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51c173be..22c56be6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,9 +121,12 @@ jobs: python -c "import wradlib; print(wradlib.version.version)" python -c "import wradlib; print(wradlib.show_versions())" - name: Render with pytest + env: + WRADLIB_EARTHDATA_BEARER_TOKEN: ${{ secrets.WRADLIB_EARTHDATA_BEARER_TOKEN }} run: | export WRADLIB_DATA=`python -c "import os, sys; print(os.path.realpath(sys.argv[1]))" $WRADLIB_DATA` pytest -n auto --verbose --durations=15 --pyargs notebooks + build_2: name: wradlib notebooks - windows runs-on: windows-latest @@ -164,6 +167,8 @@ jobs: python -c "import wradlib; print(wradlib.version.version)" python -c "import wradlib; print(wradlib.show_versions())" - name: Test with pytest + env: + WRADLIB_EARTHDATA_BEARER_TOKEN: ${{ secrets.WRADLIB_EARTHDATA_BEARER_TOKEN }} run: | export WRADLIB_DATA=`python -c "import os, sys; print(os.path.realpath(sys.argv[1]))" $WRADLIB_DATA` pytest -n auto --verbose --durations=15 --pyargs notebooks