From 01cfb8d6b1e6af124d8eafcd9cc3abd3bec81cd5 Mon Sep 17 00:00:00 2001 From: Aji John Date: Thu, 3 Sep 2020 05:06:17 +0000 Subject: [PATCH] Dask to Scale-up --- Scale-up.ipynb | 293 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 Scale-up.ipynb diff --git a/Scale-up.ipynb b/Scale-up.ipynb new file mode 100644 index 0000000..5e2aae3 --- /dev/null +++ b/Scale-up.ipynb @@ -0,0 +1,293 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "## Goal of this notebook is to scale through a cluster" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(https://landsat.usgs.gov/landsat_acq)\n", + "Use the tools on this page to determine when the Landsat 7 and Landsat 8 satellites acquire data over your area of interest, and to view the paths that were acquired on any given day. \n", + "\n", + "Landsat satellites image the entire Earth every 16 days in an 8-day offset. Landsat 7 acquires data in descending (daytime) node, while Landsat 8 acquires data in both descending and occasional ascending (nighttime) node. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Landsat 8 satellite orbits the the Earth in a sun-synchronous, near-polar orbit, at an altitude of 705 km (438 mi), inclined at 98.2 degrees, and circles the Earth every 99 minutes. The satellite has a 16-day repeat cycle with an equatorial crossing time: 10:00 a.m. +/- 15 minutes.\n", + "\n", + "Landsat satellites image the entire Earth every 16 days in an 8-day offset. Landsat 7 acquires data in descending (daytime) node, while Landsat 8 acquires data in both descending and occasional ascending (nighttime) node. \n", + "\n", + "Row refers to the latitudinal center line of a frame of imagery. As the satellite moves along its path, the observatory instruments are continuously scanning the terrain below. These will be squares centered on the orbital path, but tilted clockwise when views on the UTM projection used for the distributed data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "url = 'http://landsat-pds.s3.amazonaws.com/c1/L8/227/065/LC08_L1TP_227065_20200608_20200626_01_T1/'\n", + "redband = url+'LC08_L1TP_227065_20200608_20200626_01_T1_B{}.TIF'.format(4)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2020-09-03 02:52:24-- http://landsat-pds.s3.amazonaws.com/c1/L8/227/065/LC08_L1TP_227065_20200608_20200626_01_T1/LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF\n", + "Resolving landsat-pds.s3.amazonaws.com (landsat-pds.s3.amazonaws.com)... 52.218.217.179\n", + "Connecting to landsat-pds.s3.amazonaws.com (landsat-pds.s3.amazonaws.com)|52.218.217.179|:80... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 59222647 (56M) [image/tiff]\n", + "Saving to: ‘LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF.2’\n", + "\n", + "LC08_L1TP_227065_20 100%[===================>] 56.48M 98.6MB/s in 0.6s \n", + "\n", + "2020-09-03 02:52:25 (98.6 MB/s) - ‘LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF.2’ saved [59222647/59222647]\n", + "\n" + ] + } + ], + "source": [ + "!wget {redband}" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Driver: GTiff/GeoTIFF\n", + "Files: LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF\n", + "Size is 7621, 7761\n", + "Coordinate System is:\n", + "PROJCRS[\"WGS 84 / UTM zone 21N\",\n", + " BASEGEOGCRS[\"WGS 84\",\n", + " DATUM[\"World Geodetic System 1984\",\n", + " ELLIPSOID[\"WGS 84\",6378137,298.257223563,\n", + " LENGTHUNIT[\"metre\",1]]],\n", + " PRIMEM[\"Greenwich\",0,\n", + " ANGLEUNIT[\"degree\",0.0174532925199433]],\n", + " ID[\"EPSG\",4326]],\n", + " CONVERSION[\"UTM zone 21N\",\n", + " METHOD[\"Transverse Mercator\",\n", + " ID[\"EPSG\",9807]],\n", + " PARAMETER[\"Latitude of natural origin\",0,\n", + " ANGLEUNIT[\"degree\",0.0174532925199433],\n", + " ID[\"EPSG\",8801]],\n", + " PARAMETER[\"Longitude of natural origin\",-57,\n", + " ANGLEUNIT[\"degree\",0.0174532925199433],\n", + " ID[\"EPSG\",8802]],\n", + " PARAMETER[\"Scale factor at natural origin\",0.9996,\n", + " SCALEUNIT[\"unity\",1],\n", + " ID[\"EPSG\",8805]],\n", + " PARAMETER[\"False easting\",500000,\n", + " LENGTHUNIT[\"metre\",1],\n", + " ID[\"EPSG\",8806]],\n", + " PARAMETER[\"False northing\",0,\n", + " LENGTHUNIT[\"metre\",1],\n", + " ID[\"EPSG\",8807]]],\n", + " CS[Cartesian,2],\n", + " AXIS[\"(E)\",east,\n", + " ORDER[1],\n", + " LENGTHUNIT[\"metre\",1]],\n", + " AXIS[\"(N)\",north,\n", + " ORDER[2],\n", + " LENGTHUNIT[\"metre\",1]],\n", + " USAGE[\n", + " SCOPE[\"unknown\"],\n", + " AREA[\"World - N hemisphere - 60°W to 54°W - by country\"],\n", + " BBOX[0,-60,84,-54]],\n", + " ID[\"EPSG\",32621]]\n", + "Data axis to CRS axis mapping: 1,2\n", + "Origin = (573285.000000000000000,-683685.000000000000000)\n", + "Pixel Size = (30.000000000000000,-30.000000000000000)\n", + "Metadata:\n", + " AREA_OR_POINT=Point\n", + "Image Structure Metadata:\n", + " COMPRESSION=DEFLATE\n", + " INTERLEAVE=BAND\n", + "Corner Coordinates:\n", + "Upper Left ( 573285.000, -683685.000) ( 56d20'15.33\"W, 6d11' 5.49\"S)\n", + "Lower Left ( 573285.000, -916515.000) ( 56d20' 4.24\"W, 8d17'27.11\"S)\n", + "Upper Right ( 801915.000, -683685.000) ( 54d16'19.33\"W, 6d10'41.77\"S)\n", + "Lower Right ( 801915.000, -916515.000) ( 54d15'33.74\"W, 8d16'55.22\"S)\n", + "Center ( 687600.000, -800100.000) ( 55d18' 3.19\"W, 7d14' 6.72\"S)\n", + "Band 1 Block=512x512 Type=UInt16, ColorInterp=Gray\n" + ] + } + ], + "source": [ + "!gdalinfo LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[573285.0, -916515.0, 801915.0, -683685.0]\n" + ] + } + ], + "source": [ + "import gdal\n", + "from gdalconst import GA_ReadOnly\n", + "\n", + "data = gdal.Open('LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF', GA_ReadOnly)\n", + "geoTransform = data.GetGeoTransform()\n", + "minx = geoTransform[0]\n", + "maxy = geoTransform[3]\n", + "maxx = minx + geoTransform[1] * data.RasterXSize\n", + "miny = maxy + geoTransform[5] * data.RasterYSize\n", + "print ([minx, miny, maxx, maxy])\n", + "data = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "EPSG:4326\n", + "WGS 84 -- WGS84 - World Geodetic System 1984, used in GPS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Reproject \"EPSG\",32621 to EPSG:4326" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lets find the bounding box for the region" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + " >" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from osgeo import gdal\n", + "\n", + "input_raster=gdal.Open('LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF')\n", + "output_raster = \"Reproj\" + 'LC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF'\n", + "gdal.Warp(output_raster,input_raster,dstSRS='EPSG:4326')" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[-56.33759064732078, -8.290860873734873, -54.2594726602711, -6.178270448812246]\n" + ] + } + ], + "source": [ + "data = gdal.Open('ReprojLC08_L1TP_227065_20200608_20200626_01_T1_B4.TIF', GA_ReadOnly)\n", + "geoTransform = data.GetGeoTransform()\n", + "minx = geoTransform[0]\n", + "maxy = geoTransform[3]\n", + "maxx = minx + geoTransform[1] * data.RasterXSize\n", + "miny = maxy + geoTransform[5] * data.RasterYSize\n", + "print ([minx, miny, maxx, maxy])\n", + "data = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you have trouble getting the right scene to cover the area you want:\n", + "\n", + "Go to https://landsat.usgs.gov/wrs-2-pathrow-latitudelongitude-converter and enter the lat/long. Then note the path row, and when you get the Landsat data, insure you have the correct path and row, which are listed in the download table.\n", + "Use https://landsat.usgs.gov/landsat_acq tool, which will show the coverage for each data\n", + "Path/Row shapefiles and KML: https://www.usgs.gov/land-resources/nli/landsat/landsat-shapefiles-and-kml-files\n", + "KML file direct download:https://prd-wret.s3-us-west-2.amazonaws.com/assets/palladium/production/atoms/files/WRS-2_bound_world_0.kml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}