From 3dc69c19a3ea5629b5dd52bb18023ac2e51fe8ad Mon Sep 17 00:00:00 2001 From: "K.-Michael Aye" Date: Fri, 29 Apr 2016 17:56:17 -0600 Subject: [PATCH] add summed_spectrum saver and moviemaker to FUV --- .gitignore | 1 + notebooks/March occultation.py | 439 +++++++++++++++++++++++++++++++++ notebooks/netcdf reader.py | 46 +++- pyuvis-local.sublime-project | 17 ++ pyuvis/io.py | 55 ++++- 5 files changed, 538 insertions(+), 20 deletions(-) create mode 100644 notebooks/March occultation.py create mode 100644 pyuvis-local.sublime-project diff --git a/.gitignore b/.gitignore index 51cbe85..a48128c 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ coverage.xml # Sphinx documentation docs/_build/ +*.sublime-workspace diff --git a/notebooks/March occultation.py b/notebooks/March occultation.py new file mode 100644 index 0000000..eb688c5 --- /dev/null +++ b/notebooks/March occultation.py @@ -0,0 +1,439 @@ + +# coding: utf-8 + +# In[ ]: + +get_ipython().magic('matplotlib nbagg') + + +# In[ ]: + +fname = '/Users/klay6683/Dropbox/SternchenAndMe/UVIS_Enc_Occ_2016_03_11/HSP2016_03_11_11_48_26_000_UVIS_233EN_ICYEXO001_PIE' + + +# In[ ]: + +from pyuvis.io import HSP +from pandas import datetools + + +# In[ ]: + +hsp = HSP(fname, freq='2ms') + + +# In[ ]: + +hsp + + +# In[ ]: + +resampled = hsp.cleaned_data_copy + + +# In[ ]: + +resampled = resampled.resample('1s').mean() + + +# # FUV + +# In[ ]: + +from pathlib import Path +from pyuvis.io import FUV, HSP + + +# In[ ]: + +folder = Path('/Users/klay6683/Dropbox/SternchenAndMe/UVIS_Enc_Occ_2016_03_11') + + +# In[ ]: + +fuvfiles = list(folder.glob('*FUV*')) + + +# In[ ]: + +fuvfiles[0] + + +# In[ ]: + +fuv = FUV(fuvfiles[0]) + + +# In[ ]: + +pix1 = fuv.data[:, 1].sum('spectral_dim_0') + + +# In[ ]: + +hspdata = hsp.cleaned_data_copy + + +# In[ ]: + +fuvdata = pix1.to_series() +low = np.percentile(fuvdata, 0.5) +fuvdata[fuvdata< low] = np.nan + + +# In[ ]: + +fuvdata = fuvdata/fuvdata[10:75].mean() + +fuvt0 = fuvdata.index[10] +fuvt1 = fuvdata.index[74] + +t0_I0 = '2016-03-11 11:51:25' +t1_I0 = '2016-03-11 11:52:00' +I0 = hspdata[t0_I0:t1_I0].mean() + +hspdata = hspdata / I0 + + +# In[ ]: + +df = pd.DataFrame(hspdata, columns=['hsp']) +df['fuv'] = fuvdata +subdf = df[df.index[-1] - datetools.Second(50):] + + +# In[ ]: + +corrs = [] +shifts = range(900, 1100, 2) +for i in shifts: + resampled = subdf.hsp.shift(i, freq='ms').resample('1s').mean() + corrs.append(resampled.corr(subdf.fuv)) + +plt.figure() +plt.plot(shifts, corrs) + +print(shifts[np.argmax(corrs)]) + + +# In[ ]: + +shifts = [990, 996, 1000] +fig, axes = plt.subplots(nrows=3) +for i,shift in enumerate(shifts): + df = pd.DataFrame(hspdata.shift(shift,freq='ms'), columns=['hsp']) + df['fuv'] = fuvdata + subdf = df[df.index[-1] - datetools.Second(50):] + subdf.resample('1s').mean().plot(ax=axes[i]) + + +# In[ ]: + +df = pd.DataFrame(hspdata.shift(990,freq='ms'), columns=['hsp']) + +df['fuv'] = fuvdata + +subdf = df[df.index[-1] - datetools.Second(50):] + +subdf.resample('1s').mean().plot() + + +# In[ ]: + +t0 = '2016-03-11 11:51:50' + + +# In[ ]: + +fig, ax = plt.subplots() +resampled=df.resample('1s').mean() +resampled.hsp.plot(ax=ax, label='HSP', legend=True, lw=1.5) +resampled.fuv.plot(ax=ax, label='FUV', legend=True, lw=1.5) +plt.axvspan(t0_I0, t1_I0, facecolor='b', alpha=0.2) +plt.text(t0_I0, 0.92, r'$I_0(HSP)$', fontsize=16) +plt.axvspan(fuvt0, fuvt1, facecolor='g', alpha=0.2) +plt.text(fuvt0, 0.91, r'$I_0(FUV)$', fontsize=16) +ax.set_ylabel(r"$I/I_0$", fontsize=16) +ax.set_xlabel('Time', fontsize=16) +ax.set_title(r"Normalized, HSP vs FUV, showing $I_0$ regions") +plt.savefig("HSP_FUV_all_data_HSP990ms_shifted.pdf") + + +# In[ ]: + +from matplotlib import dates + + +# In[ ]: + +fig, ax = plt.subplots() +toplot = resampled.hsp[t0:] +toplot2 = resampled.fuv[t0:] +ax.plot_date(toplot.index, toplot, 'o-', label='HSP', markersize=3, lw=1.5) +ax.plot_date(toplot2.index, toplot2, 'v-', label='FUV', markersize=3,lw=1.5) +ax.xaxis.set_major_locator(dates.SecondLocator(interval=5)) +ax.xaxis.set_major_formatter(dates.DateFormatter('%S')) +ax.grid() +ax.set_ylabel(r'$I / I_0$', fontsize=16) +ax.set_xlabel("Time [s]", fontsize=16) +ax.set_title("Last 50 s, normalized") +ax.legend() +fig.savefig("HSP_FUV_comparison_last_50s_HSP990ms_shifted.pdf") + + +# In[ ]: + +fuv.data + + +# In[ ]: + +fuv.data.coords + + +# In[ ]: + +fuv.data.attrs + + +# In[ ]: + +fuv.data.dims + + +# In[ ]: + +data = fuv.data + + +# In[ ]: + +fuv.times + + +# In[ ]: + +hsp.times + + +# In[ ]: + +hsp.n_integrations + + +# In[ ]: + +hsp + + +# In[ ]: + +fuv.data + + +# In[ ]: + +fuv.data.sel(integrations=0).mean() + + +# In[ ]: + +plt.figure() +fuv.data.mean(['spatial_dim_0', 'spectral_dim_0']).plot() + + +# In[ ]: + +spec = fuv.data[100] + + +# In[ ]: + +get_ipython().magic('matplotlib inline') + + +# In[ ]: + +p = Path('./plots') + + +# In[ ]: + +waves = np.linspace(111.5, 190, 512) + + +# In[ ]: + +import xarray as xr + + +# In[ ]: + +xrwaves = xr.DataArray(waves, dims=['wavelength']) + + +# In[ ]: + +fuv.data.coords + + +# In[ ]: + +fuv.data.dims + + +# In[ ]: + +xr.DataArray(fuv.data, coords=[fuv.data.integrations, + fuv.data.spatial_dim_0, + fuv.data.spectral_dim_0]) + + +# In[ ]: + +fuv.data.coords + + +# In[ ]: + +fuv.data.dims + + +# In[ ]: + +data = np.random.rand(4, 3) + +locs = ['IA', 'IL', 'IN'] + +times = pd.date_range('2000-01-01', periods=4) + +foo = xr.DataArray(data, coords=[times, locs], dims=['time', 'space']) + +foo + + +# In[ ]: + +foo.dims + + +# In[ ]: + +foo.coords + + +# In[ ]: + +xr.DataArray(fuv.data, coords=[fuv.data.integrations, + fuv.data.spatial_dim_0]) + + +# In[ ]: + + + + +# In[ ]: + + + + +# In[ ]: + +fuv.ds.update({'wind':xrwaves}) + + +# In[ ]: + +newz = xr.DataArray(np.random.randn(3), [('y', [10,20,30])]) +newz + + +# In[ ]: + +xr.concat([arr, newz], dim='x') + + +# In[ ]: + +arr + + +# In[ ]: + +newz + + +# In[ ]: + + + + +# In[ ]: + +for i,spec in enumerate(fuv.data): + print(i) + fig, ax = plt.subplots() + ax.plot(xrwaves, spec[1]) + ax.set_ylim((0,72)) + fig.savefig("plots/spec_{}.png".format(str(i).zfill(3)), dpi=150) + plt.close(fig) + + +# In[ ]: + +spec[1] + + +# In[ ]: + +import xarray as xr + + +# In[ ]: + +waves = xr.DataArray(waves, dims=['wavelength']) + + +# In[ ]: + +fuv.data.spectral_dim_0 = waves + + +# In[ ]: + +plt.figure() +fuv.data.sum(['spectral_dim_0'])[:, 1].plot() + + +# In[ ]: + +spec.spectral_dim_0 = waves + + +# In[ ]: + +waves + + +# In[ ]: + +np.percentile(fuv.data, 1.5) + + +# In[ ]: + +fuv.data[0].plot + + +# In[ ]: + +plt.figure() +hsp.resampled.mean().plot() + + +# In[ ]: + + + diff --git a/notebooks/netcdf reader.py b/notebooks/netcdf reader.py index fbbca63..bae95d1 100644 --- a/notebooks/netcdf reader.py +++ b/notebooks/netcdf reader.py @@ -103,47 +103,77 @@ # # FUV -# In[259]: +# In[1]: from pathlib import Path from pyuvis.io import FUV, HSP -# In[260]: +# In[2]: folder = Path('/Users/klay6683/Dropbox/SternchenAndMe/UVIS_Enc_Occ_2016_03_11') -# In[261]: +# In[3]: fuvfiles = list(folder.glob('*FUV*')) -# In[262]: +# In[4]: fuvfiles[0] -# In[263]: +# In[5]: fuv = FUV(fuvfiles[0]) -# In[264]: +# In[6]: fuv -# In[266]: +# In[7]: + +get_ipython().magic('matplotlib inline') + + +# In[20]: + +fuv.data.sum(['wavelengths','pixels']).plot() + + +# In[8]: + +fuv.save_spectrums() + + +# In[9]: + +fuv.create_spec_time_sequence_movie() + + +# In[10]: fuv.save_spectograms() -# In[267]: +# In[11]: fuv.create_spectogram_movie() +# In[19]: + +fuv.data[23].sum('pixels').plot() + + +# In[20]: + +fuv.data[23].sum('pixels').max() + + # In[205]: obj = fuv.data[:, 1].sum('wavelengths') diff --git a/pyuvis-local.sublime-project b/pyuvis-local.sublime-project new file mode 100644 index 0000000..9452cc4 --- /dev/null +++ b/pyuvis-local.sublime-project @@ -0,0 +1,17 @@ +{ + "build_systems": + [ + { + "file_regex": "^[ ]*File \"(...*?)\", line ([0-9]*)", + "name": "Anaconda Python Builder", + "selector": "source.python", + "shell_cmd": "\"python\" -u \"$file\"" + } + ], + "folders": + [ + { + "path": "." + } + ] +} diff --git a/pyuvis/io.py b/pyuvis/io.py index e8e0464..1ea79d2 100644 --- a/pyuvis/io.py +++ b/pyuvis/io.py @@ -4,9 +4,11 @@ import numpy as np import pandas as pd import pvl +import seaborn as sns import xarray as xr from pandas import datetools from pathlib import Path + from .hsp_sensitivity import sens_df try: @@ -196,32 +198,61 @@ def __init__(self, fname, freq='1s'): def data(self): return self.ds.counts + @property + def plotfolder(self): + f = self.path.parent / 'plots' + f.mkdir(exist_ok=True) + return f + def save_spectograms(self): - savefolder = self.path.parent / 'plots' - savefolder.mkdir(exist_ok=True) - vmax = self.data.max() + sns.set_context('talk') + vmax = self.data.max()*1.05 for i, spec in enumerate(self.data): fig, ax = plt.subplots() spec.plot(ax=ax, vmax=vmax) fig.tight_layout() fname = "spectogram_{}.png".format(str(i).zfill(4)) - savepath = str(savefolder / fname) + savepath = str(self.plotfolder / fname) fig.savefig(savepath, dpi=150) plt.close(fig) - self.savefolder = savefolder - print("Saved spectrograms in ", savefolder) + print("Saved spectrograms in", self.plotfolder) - def create_spectogram_movie(self): - if not _FFMPY_INSTALLED: - print("ffmpy is not installed: 'pip install ffmpy'.") - return + def _run_ffmpy(self, inputs, output_name): opts = '-framerate 3 -y' - inputs = {str(self.savefolder / 'spectogram_%04d.png'):None} output_options = '-c:v libx264 -pix_fmt yuv420p' - outputs = {str(self.savefolder / 'spectograms.mp4'): output_options} + outputs = {str(self.plotfolder / output_name): output_options} ff = ffmpy.FF(global_options=opts, inputs=inputs, outputs=outputs) print("Running", ff.cmd_str) ff.run() + def create_spectogram_movie(self): + if not _FFMPY_INSTALLED: + print("ffmpy is not installed: 'pip install ffmpy'.") + return + inputs = {str(self.plotfolder / 'spectogram_%04d.png'): None} + self._run_ffmpy(inputs, 'spectograms.mp4') + + def save_spectrums(self): + "plotting spectrums over time summing all pixels" + sns.set_context('talk') + vmax = self.data.sum('pixels').max()*1.05 + for i, spec in enumerate(self.data.sum('pixels')): + fig, ax = plt.subplots() + spec.plot() + ax.set_ylim(0, vmax) + fig.tight_layout() + fname = "summed_spectrum_{}.png".format(str(i).zfill(4)) + savepath = str(self.plotfolder / fname) + fig.savefig(savepath, dpi=150) + plt.close(fig) + print("Saved spectrums in", self.plotfolder) + + def create_spec_time_sequence_movie(self): + if not _FFMPY_INSTALLED: + print("ffmpy is not installed: 'pip install ffmpy'.") + return + inputs = {str(self.plotfolder / 'summed_spectrum_%04d.png'): None} + self._run_ffmpy(inputs, 'summed_spectrums.mp4') + def __repr__(self): return self.ds.__repr__()