From 8793053b0378f0e14b507447a194f65cf0bd5cf6 Mon Sep 17 00:00:00 2001 From: Craig Arthur Date: Tue, 13 Feb 2024 08:47:02 +1100 Subject: [PATCH] Workflow fix 1 (#144) * Remove deprecated Basemap references * Update conda environment file * Update actions/checkout to v3, setup-miniconda to v2.2 * Leave shapely to dependencies * Remove cartopy version, bump python version * Stringify python version --- .github/workflows/tcrm-pylint.yml | 6 +- .github/workflows/tcrm-tests.yml | 9 +-- Evaluate/evaluate.py | 65 ++++++++--------- tcrmenv.yml | 4 +- tests/kde/kde2.py | 70 ------------------- tests/kde/kde3.py | 112 ------------------------------ 6 files changed, 38 insertions(+), 228 deletions(-) delete mode 100644 tests/kde/kde2.py delete mode 100644 tests/kde/kde3.py diff --git a/.github/workflows/tcrm-pylint.yml b/.github/workflows/tcrm-pylint.yml index d6e3c8aa..df7a2a9b 100644 --- a/.github/workflows/tcrm-pylint.yml +++ b/.github/workflows/tcrm-pylint.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up environment - uses: conda-incubator/setup-miniconda@v2.0.0 + uses: conda-incubator/setup-miniconda@v2 with: python-version: 3.7 - mamba-version: "*" + miniforge-variant: Mambaforge channels: conda-forge,defaults channel-priority: true activate-environment: tcrm @@ -32,7 +32,7 @@ jobs: pylint --rcfile pylintrc --fail-under=7 `find -regextype egrep -regex '(.*.py)$'` | tee pylint.txt - name: Upload pylint.txt as artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pylint report path: pylint.txt diff --git a/.github/workflows/tcrm-tests.yml b/.github/workflows/tcrm-tests.yml index 59f5b5d6..7de5953f 100644 --- a/.github/workflows/tcrm-tests.yml +++ b/.github/workflows/tcrm-tests.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ['3.9', '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up environment - uses: conda-incubator/setup-miniconda@v2.0.0 + uses: conda-incubator/setup-miniconda@v2 with: + miniforge-variant: MambaForge python-version: ${{ matrix.python-version }} - mamba-version: "*" channels: conda-forge,defaults channel-priority: true activate-environment: tcrm @@ -30,6 +30,7 @@ jobs: auto-activate-base: false use-only-tar-bz2: true + - name: Test with pytest env: PYTHONPATH: ~/tcrm;~/tcrm/Utilities diff --git a/Evaluate/evaluate.py b/Evaluate/evaluate.py index c306c8c1..dfa8f3e6 100644 --- a/Evaluate/evaluate.py +++ b/Evaluate/evaluate.py @@ -32,7 +32,8 @@ from matplotlib import pyplot, cm from matplotlib.dates import date2num -from mpl_toolkits.basemap import Basemap +from cartopy import crs as ccrs +from cartopy import feature as cfeature from scipy.stats import scoreatpercentile as percentile from datetime import datetime @@ -76,7 +77,7 @@ [ProcessMultipliers] MaxWorkingThreads = 4 ProcessMultiVersion = 2 -ProcessingSegmentSize = 256 +ProcessingSegmentSize = 256 WarpMemoryLimit = 500 [Logging] @@ -169,17 +170,6 @@ def plotDensity(x, y, data, llLon=None, llLat=None, urLon=None, urLat=None, else: urcrnrlat = y.max() - meridians = np.arange(dl * np.floor(llcrnrlon / dl), - dl * np.ceil(urcrnrlon / dl), dl) - parallels = np.arange(dl * np.floor(llcrnrlat / dl), - dl * np.ceil(urcrnrlat / dl), dl) - - m = Basemap(projection='cyl', - resolution=res, - llcrnrlon=llcrnrlon, - urcrnrlon=urcrnrlon, - llcrnrlat=llcrnrlat, - urcrnrlat=urcrnrlat) # Set the colour map: if hasattr(cm, cmap): @@ -187,29 +177,35 @@ def plotDensity(x, y, data, llLon=None, llLat=None, urLon=None, urLat=None, else: cmap = colours.colourMap(cmap, 'stretched') - if maskocean: - try: - from mpl_toolkits.basemap import maskoceans - except ImportError: - log.debug("Maskoceans module unavailable, skipping this command") - else: - datam = maskoceans(xx, yy, data, inlands=False) - m.pcolormesh(xx, yy, datam, edgecolors='None', - vmin=datarange[0], vmax=datarange[1], - cmap=cmap) - else: - m.pcolormesh(xx, yy, data, edgecolors='None', - vmin=datarange[0], vmax=datarange[1], - cmap=cmap) + ax = pyplot.axes(projection=ccrs.PlateCarree()) + pyplot.pcolormesh(xx, yy, data, edgecolors='None', + vmin=datarange[0], vmax=datarange[1], + cmap=cmap, transfom=ccrs.PlateCarree()) - m.drawcoastlines(linewidth=0.5) if maskland: - m.fillcontinents(color='white') + ax.add_feature(cfeature.LAND, zorder=100, edgecolor='k') + + if maskocean: + ax.add_feature(cfeature.OCEAN, zorder=100, edgecolor='k') + + ax.coastlines(linewidth=0.5) + gl = ax.gridlines(draw_labels=True, crs=ccrs.PlateCarree(), linewidth=0.2) + gl.top_labels = False + gl.right_labels = False + + ax.set_extent([llcrnrlon, urcrnrlon, + llcrnrlat, urcrnrlat]) + + cb = pyplot.colorbar(shrink=0.5, aspect=30, + orientation='horizontal', + extend='max', pad=0.1) + + if cb.orientation == 'horizontal': + for t in cb.ax.get_xticklabels(): + t.set_fontsize(8) - m.drawparallels(parallels, labels=[1, 0, 0, 0], - fontsize=7.5, linewidth=0.2) - m.drawmeridians(meridians, labels=[0, 0, 0, 1], - fontsize=7.5, linewidth=0.2) + if clabel: + cb.set_label(clabel) if ylab: pyplot.ylabel(ylab, fontsize=7.5) if xlab: @@ -217,9 +213,6 @@ def plotDensity(x, y, data, llLon=None, llLat=None, urLon=None, urLat=None, if title: pyplot.title(title) - pyplot.grid(True) - pyplot.tick_params(direction='out', right='off', top='off') - cb = pyplot.colorbar(shrink=0.5, aspect=30, orientation='horizontal', extend='max', pad=0.1) diff --git a/tcrmenv.yml b/tcrmenv.yml index f42ac7c0..142647d7 100644 --- a/tcrmenv.yml +++ b/tcrmenv.yml @@ -9,8 +9,6 @@ dependencies: - numpy - scipy - matplotlib - - basemap - - shapely - nose - netcdf4 - cftime @@ -23,7 +21,7 @@ dependencies: - sqlite - statsmodels - configparser - - cartopy>=0.18.0 + - cartopy - affine - tqdm - xarray diff --git a/tests/kde/kde2.py b/tests/kde/kde2.py deleted file mode 100644 index 58f55102..00000000 --- a/tests/kde/kde2.py +++ /dev/null @@ -1,70 +0,0 @@ -import pickle -import os -import numpy as np -import matplotlib.pyplot as plt -from scipy import stats -from mpl_toolkits.basemap import Basemap -import Utilities.KPDF as KPDF - -ff = os.path.join('..', 'test_data', 'kde_origin_lonLat.pkl') -lonlat = pickle.load(open(ff, 'rb')) - -xmin = 70.0 -xmax = 180.0 -ymin = -36. -ymax = 0. - -fig, axes = plt.subplots(nrows=2, ncols=1) - -for ax in axes.flat: - m = Basemap(ax=ax, projection='cyl', resolution='i', llcrnrlon=xmin, - urcrnrlon=xmax, llcrnrlat=ymin, urcrnrlat=ymax) - m.drawcoastlines() - m.drawcountries() - m.drawmapboundary(fill_color='#ffffff') - m.fillcontinents(color='#dedcd2', lake_color='#ffffff') - meridians = np.arange(xmin, xmax, 10.) - parallels = np.arange(ymin, ymax, 10.) - m.drawparallels(parallels, labels=[1, 0, 0, 0], fontsize=9, linewidth=0.2) - m.drawmeridians(meridians, labels=[0, 0, 0, 1], fontsize=9, linewidth=0.2) - ax.grid(True) - -xy = lonlat -n, d = xy.shape -bw_kpdf = KPDF.MPDFOptimumBandwidth(xy) -bw_scott = n ** (-1.0 / (d + 4)) -bw_silverman = (n * (d + 2) / 4.) ** (-1. / (d + 4)) - -X, Y = np.mgrid[xmin:xmax:30j, ymin:ymax:30j] -positions = np.vstack([X.ravel(), Y.ravel()]).T - -# KDE - -z = KPDF.MPDFGaussian(xy, positions, bw_kpdf) -z = np.reshape(z, X.shape) -print(('z max = %f' % z.max())) - -ax = axes[0] -ax.imshow(np.rot90(z) / z.max(), cmap=plt.cm.PuRd, - extent=[xmin, xmax, ymin, ymax]) -ax.scatter(lonlat[:, 0], lonlat[:, 1], s=2, marker='o', - facecolor='darkred', edgecolor='darkred', alpha=0.8) -ax.contour(X, Y, np.rot90(z) / z.max(), colour='magenta') -ax.set_title('KPDF') - -# Scipy - -kde = stats.gaussian_kde(lonlat.T) -Z = np.reshape(kde(positions.T), X.shape) -print(('Z max = %f' % Z.max())) - -ax = axes[1] -ax.imshow(np.rot90(Z) / Z.max(), cmap=plt.cm.PuRd, - extent=[xmin, xmax, ymin, ymax]) -ax.scatter(lonlat[:, 0], lonlat[:, 1], s=2, marker='o', - facecolor='darkred', edgecolor='darkred', alpha=0.8) -ax.contour(X, Y, np.rot90(Z) / Z.max(), colour='magenta') -ax.set_title('Scipy') - -plt.savefig('fig2.pdf') -plt.show() diff --git a/tests/kde/kde3.py b/tests/kde/kde3.py deleted file mode 100644 index a8dbae73..00000000 --- a/tests/kde/kde3.py +++ /dev/null @@ -1,112 +0,0 @@ -import numpy as np -import scipy.stats as stats -import matplotlib.pyplot as plt - -import Utilities.KPDF as KPDF -import statsmodels.nonparametric.kernel_density as smkde -import statsmodels.nonparametric.bandwidths as smbw - -import seaborn as sns -sns.set_context("paper") - -xrvs = [stats.norm(loc=1.0, scale=0.2), stats.norm(loc=3.0, scale=0.1)] -yrvs = [stats.norm(loc=0.0, scale=0.2), stats.norm(loc=1.0, scale=0.3)] - -N = 3000 -M = N / (len(xrvs) + len(yrvs)) - -rvs = [] -for xrv in xrvs: - for yrv in yrvs: - rvs.append(np.hstack([ - xrv.rvs(size=(M, 1)), - yrv.rvs(size=(M, 1))])) -rvs = np.vstack(rvs) - - -def pdf(x, y): - return (sum([rv.pdf(x) for rv in xrvs]) * - sum([rv.pdf(y) for rv in yrvs])) - - -def plot(ax, Z, title): - ax.scatter(rvs[:, 0], rvs[:, 1], alpha=0.3, s=1, color='black') - ax.imshow(Z, aspect=1, origin='lower', - cmap=sns.light_palette((210, 90, 60), - input='husl', - as_cmap=True), - extent=(rvs[:, 0].min(), - rvs[:, 0].max(), - rvs[:, 1].min(), - rvs[:, 1].max())) - ax.contour(x, y, Z, linewidth=1) - ax.set_title(title) - -x_flat = np.r_[rvs[:, 0].min():rvs[:, 0].max():128j] #pylint: disable=E1127 -y_flat = np.r_[rvs[:, 1].min():rvs[:, 1].max():128j] #pylint: disable=E1127 -x, y = np.meshgrid(x_flat, y_flat) -grid = np.hstack([x.reshape(-1, 1), y.reshape(-1, 1)]) - -n, d = rvs.shape -bw_kpdf = KPDF.MPDFOptimumBandwidth(rvs) -bw_scott = n ** (-1.0 / (d + 4)) -bw_silverman = (n * (d + 2) / 4.) ** (-1. / (d + 4)) - -scale = np.array([[x_flat.ptp(), y_flat.ptp()]]) -print(('scale: %s' % scale)) - -print(('bandwidth kpdf=%f scott=%f silverman=%f' % - (bw_kpdf, bw_scott, bw_silverman))) - -fig, axes = plt.subplots(nrows=4, ncols=2, figsize=(8, 11)) -axes = axes.flat - -p = pdf(x, y) -p = p / p.max() -plot(axes[0], p, 'True density') - -kde = stats.kde.gaussian_kde(rvs.T) -print((kde.covariance)) -z = kde(grid.T) -z = z.reshape(x.shape) / z.max() -plot(axes[1], z, 'Scipy ($\ell_2$ norm: %.3f)' % np.linalg.norm((p - z).flat)) - -bw = bw_kpdf -# bw = bw_scott - -w = KPDF.MPDFGaussian(rvs, grid, bw_kpdf) -w = w.reshape(x.shape) / w.max() -plot(axes[2], w, 'KPDF bw:kpdf ($\ell_2$ norm: %.3f)' % np.linalg.norm((p - - w).flat)) - -w = KPDF.MPDFGaussian(rvs, grid, bw_kpdf / 2) -w = w.reshape(x.shape) / w.max() -plot(axes[3], w, 'KPDF bw:kpdf/2 ($\ell_2$ norm: %.3f)' % np.linalg.norm(( - p - w).flat)) - -w = KPDF.MPDFGaussian(rvs, grid, bw_scott) -w = w.reshape(x.shape) / w.max() -plot(axes[4], w, 'KPDF bw:scott ($\ell_2$ norm: %.3f)' % np.linalg.norm((p - - w).flat)) - -w = KPDF.MPDFGaussian(rvs, grid, bw_scott / 2) -w = w.reshape(x.shape) / w.max() -plot(axes[5], w, 'KPDF bw:scott/2 ($\ell_2$ norm: %.3f)' % np.linalg.norm(( - p - w).flat)) - -dens = smkde.KDEMultivariate(rvs, 'cc', bw='cv_ml') -print("SM bandwidth (cv_ml): " + repr(dens.bw)) -w = dens.pdf(grid) -w = w.reshape(x.shape) / w.max() -plot(axes[6], w, 'SM bw:CVML ($\ell_2$ norm: %.3f)' % np.linalg.norm((p - - w).flat)) - -dens = smkde.KDEMultivariate(rvs, 'cc', bw='cv_ls') -print("SM bandwidth (cv_ls): " + repr(dens.bw)) -w = dens.pdf(grid) -w = w.reshape(x.shape) / w.max() -plot(axes[7], w, 'SM bw:CVLS ($\ell_2$ norm: %.3f)' % np.linalg.norm((p - - w).flat)) - -plt.savefig('fig3.png') -plt.show()