Merge pull request #17 from eteq/cache

cache the downloaded datafiles and use symlinks
spacetelescope · Apr 28, 2021 · fb62e0b · fb62e0b
2 parents d47f1e8 + 5cd3ce0
commit fb62e0b
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 49 deletions.
diff --git a/pipeline_products_session/jwst-data-products-part1-live.ipynb b/pipeline_products_session/jwst-data-products-part1-live.ipynb
@@ -224,19 +224,17 @@
     "# Data for the notebook\n",
     "uncal_obs_link = \"https://stsci.box.com/shared/static/mpbrc3lszdjif6kpcw1acol00e0mm2zh.fits\"\n",
     "uncal_obs = \"example_nircam_imaging_uncal.fits\"\n",
-    "demo_file = download_file(uncal_obs_link+uncal_obs)\n",
+    "demo_file = download_file(uncal_obs_link+uncal_obs, cache=True)\n",
     "\n",
     "# Data for the exercise \n",
     "exercise_obs_link = \"https://stsci.box.com/shared/static/l1aih8rmwbtzyupv8hsl0adfa36why30.fits\"\n",
     "exercise_obs = \"example_exercise_uncal.fits\"\n",
-    "demo_ex_file = download_file(exercise_obs_link+exercise_obs)\n",
+    "demo_ex_file = download_file(exercise_obs_link+exercise_obs, cache=True)\n",
     "\n",
-    "# Save the files so that we can use them later\n",
-    "with fits.open(demo_file, ignore_missing_end=True) as f:\n",
-    "    f.writeto(uncal_obs, overwrite=True)\n",
-    "    \n",
-    "with fits.open(demo_ex_file, ignore_missing_end=True) as f:\n",
-    "    f.writeto(exercise_obs, overwrite=True)        "
+    "# Create local links to the cached copy - this is not necessary - you can use the `demo_file`/`demo_ex_file` \n",
+    "# names directly.  But this is a convenient to see what you've downloaded and remind yourself laiter\n",
+    "os.symlink(demo_file, uncal_obs)\n",
+    "os.symlink(demo_ex_file, exercise_obs)  "
    ]
   },
   {
@@ -1120,7 +1118,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,

diff --git a/pipeline_products_session/jwst-data-products-part1-solutions.ipynb b/pipeline_products_session/jwst-data-products-part1-solutions.ipynb
@@ -224,19 +224,17 @@
     "# Data for the notebook\n",
     "uncal_obs_link = \"https://stsci.box.com/shared/static/mpbrc3lszdjif6kpcw1acol00e0mm2zh.fits\"\n",
     "uncal_obs = \"example_nircam_imaging_uncal.fits\"\n",
-    "demo_file = download_file(uncal_obs_link+uncal_obs)\n",
+    "demo_file = download_file(uncal_obs_link+uncal_obs, cache=True)\n",
     "\n",
     "# Data for the exercise \n",
     "exercise_obs_link = \"https://stsci.box.com/shared/static/l1aih8rmwbtzyupv8hsl0adfa36why30.fits\"\n",
     "exercise_obs = \"example_exercise_uncal.fits\"\n",
-    "demo_ex_file = download_file(exercise_obs_link+exercise_obs)\n",
+    "demo_ex_file = download_file(exercise_obs_link+exercise_obs, cache=True)\n",
     "\n",
-    "# Save the files so that we can use them later\n",
-    "with fits.open(demo_file, ignore_missing_end=True) as f:\n",
-    "    f.writeto(uncal_obs, overwrite=True)\n",
-    "    \n",
-    "with fits.open(demo_ex_file, ignore_missing_end=True) as f:\n",
-    "    f.writeto(exercise_obs, overwrite=True)        "
+    "# Create local links to the cached copy - this is not necessary - you can use the `demo_file`/`demo_ex_file` \n",
+    "# names directly.  But this is a convenient to see what you've downloaded and remind yourself laiter\n",
+    "os.symlink(demo_file, uncal_obs)\n",
+    "os.symlink(demo_ex_file, exercise_obs)     "
    ]
   },
   {
@@ -1164,7 +1162,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,

diff --git a/pipeline_products_session/jwst-data-products-part2-live.ipynb b/pipeline_products_session/jwst-data-products-part2-live.ipynb
@@ -254,11 +254,10 @@
     "             demo_ex_file]\n",
     "\n",
     "for file in all_files:\n",
-    "    demo_file = download_file(file[0])\n",
+    "    demo_file = download_file(file[0], cache=True)\n",
     "    \n",
-    "    # Save the file so that we can use it later\n",
-    "    with fits.open(demo_file) as f:\n",
-    "        f.writeto(file[1], overwrite=True)"
+    "    # Make a symbolic link using a local name for convenience\n",
+    "    os.symlink(demo_file, file[1])"
    ]
   },
   {
@@ -1283,7 +1282,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,

diff --git a/pipeline_products_session/jwst-data-products-part2-solutions.ipynb b/pipeline_products_session/jwst-data-products-part2-solutions.ipynb
@@ -254,11 +254,10 @@
     "             demo_ex_file]\n",
     "\n",
     "for file in all_files:\n",
-    "    demo_file = download_file(file[0])\n",
+    "    demo_file = download_file(file[0], cache=True)\n",
     "    \n",
-    "    # Save the file so that we can use it later\n",
-    "    with fits.open(demo_file) as f:\n",
-    "        f.writeto(file[1], overwrite=True)"
+    "    # Make a symbolic link using a local name for convenience\n",
+    "    os.symlink(demo_file, file[1])"
    ]
   },
   {
@@ -1335,7 +1334,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,

diff --git a/pipeline_products_session/jwst-data-products-part3-live.ipynb b/pipeline_products_session/jwst-data-products-part3-live.ipynb
@@ -272,24 +272,22 @@
    "source": [
     "# For the catalog file:\n",
     "catalog_file_link = 'https://stsci.box.com/shared/static/272qsdpoax1cchy0gox96mobjpol780n.ecsv'\n",
-    "output_catalog = download_file(catalog_file_link)\n",
+    "output_catalog = download_file(catalog_file_link, cache=True)\n",
     "\n",
     "# For the NIRCam combined 2D image:\n",
     "combined_i2d_file_link = 'https://stsci.box.com/shared/static/1xjdi28u5o1lmkmau0wuojdyv8fnre5n.fits'\n",
-    "combined_i2d = download_file(combined_i2d_file_link)\n",
+    "combined_i2d = download_file(combined_i2d_file_link, cache=True)\n",
     "combined_i2d_file = \"example_nircam_imaging_i2d.fits\"\n",
     "\n",
     "# For the NIRCam WFSS 1D file:\n",
     "final_c1d_file_link = 'https://stsci.box.com/shared/static/ixfnu50ju78vs40dcec8i7w0u6kwtoli.fits'\n",
-    "final_c1d = download_file(final_c1d_file_link)\n",
+    "final_c1d = download_file(final_c1d_file_link, cache=True)\n",
     "final_c1d_file = \"example_nircam_wfss_c1d.fits\"\n",
     "\n",
-    "# Save the files so that we can use them later\n",
-    "with fits.open(combined_i2d, ignore_missing_end=True) as f:\n",
-    "    f.writeto(combined_i2d_file, overwrite=True)\n",
-    "    \n",
-    "with fits.open(final_c1d, ignore_missing_end=True) as f:\n",
-    "    f.writeto(final_c1d_file, overwrite=True)    "
+    "# Create local links to the cached copies of the fits file.  This is not necessary - you can use \n",
+    "#`combined_i2d`/`final_c1d`  directly.  But this is convenient to remind yourself later what you downloaded\n",
+    "os.symlink(combined_i2d, combined_i2d_file)\n",
+    "os.symlink(final_c1d, final_c1d_file)  "
    ]
   },
   {
@@ -1058,7 +1056,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,

diff --git a/pipeline_products_session/jwst-data-products-part3-solutions.ipynb b/pipeline_products_session/jwst-data-products-part3-solutions.ipynb
@@ -272,24 +272,22 @@
    "source": [
     "# For the catalog file:\n",
     "catalog_file_link = 'https://stsci.box.com/shared/static/272qsdpoax1cchy0gox96mobjpol780n.ecsv'\n",
-    "output_catalog = download_file(catalog_file_link)\n",
+    "output_catalog = download_file(catalog_file_link, cache=True)\n",
     "\n",
     "# For the NIRCam combined 2D image:\n",
     "combined_i2d_file_link = 'https://stsci.box.com/shared/static/1xjdi28u5o1lmkmau0wuojdyv8fnre5n.fits'\n",
-    "combined_i2d = download_file(combined_i2d_file_link)\n",
+    "combined_i2d = download_file(combined_i2d_file_link, cache=True)\n",
     "combined_i2d_file = \"example_nircam_imaging_i2d.fits\"\n",
     "\n",
     "# For the NIRCam WFSS 1D file:\n",
     "final_c1d_file_link = 'https://stsci.box.com/shared/static/ixfnu50ju78vs40dcec8i7w0u6kwtoli.fits'\n",
-    "final_c1d = download_file(final_c1d_file_link)\n",
-    "final_c1d_file = \"example_nircam_wfss_c1d.fits\"\n",
+    "final_c1d = download_file(final_c1d_file_link, cache=True)\n",
+    "final_c1d_file = \"example_nircam_wfss_c1d.fits\" \n",
     "\n",
-    "# Save the files so that we can use them later\n",
-    "with fits.open(combined_i2d, ignore_missing_end=True) as f:\n",
-    "    f.writeto(combined_i2d_file, overwrite=True)\n",
-    "    \n",
-    "with fits.open(final_c1d, ignore_missing_end=True) as f:\n",
-    "    f.writeto(final_c1d_file, overwrite=True)    "
+    "# Create local links to the cached copies of the fits file.  This is not necessary - you can use \n",
+    "#`combined_i2d`/`final_c1d`  directly.  But this is convenient to remind yourself later what you downloaded\n",
+    "os.symlink(combined_i2d, combined_i2d_file)\n",
+    "os.symlink(final_c1d, final_c1d_file)  "
    ]
   },
   {
@@ -1076,7 +1074,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.9.4"
   }
  },
  "nbformat": 4,