Untitled Notebook 2024-10-17 21_46_13.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "6da1e853-2adc-4182-8fdb-96a431a7525c",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# we have 4 types of utilities:\n",
    "### File system utilities\n",
    "### Secrets utilities\n",
    "### widget utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "8cfa6efd-a6d4-43f0-9cbc-f0f00be27350",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "## File System utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "fde4a63f-7ea5-4456-8870-6ca6b9721fcc",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "### On this notebook we review some scripts about file system utilities.\n",
    "- we should remember we can use file system utilities on python , Scala and R, but not in SQL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "54c105e4-498c-4e99-84a8-7b59b41097ba",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>path</th><th>name</th><th>size</th><th>modificationTime</th></tr></thead><tbody><tr><td>dbfs:/databricks-datasets/</td><td>databricks-datasets/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-results/</td><td>databricks-results/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/user/</td><td>user/</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         "dbfs:/databricks-datasets/",
         "databricks-datasets/",
         0,
         0
        ],
        [
         "dbfs:/databricks-results/",
         "databricks-results/",
         0,
         0
        ],
        [
         "dbfs:/user/",
         "user/",
         0,
         0
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {
        "isDbfsCommandResult": false
       },
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "path",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "size",
         "type": "\"long\""
        },
        {
         "metadata": "{}",
         "name": "modificationTime",
         "type": "\"long\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fs refers to file system utility\n",
    "%fs\n",
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "312e365f-1a85-463e-8140-e687dba9afb1",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>path</th><th>name</th><th>size</th><th>modificationTime</th></tr></thead><tbody><tr><td>dbfs:/databricks-datasets/COVID/</td><td>COVID/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/README.md</td><td>README.md</td><td>976</td><td>1532468253000</td></tr><tr><td>dbfs:/databricks-datasets/Rdatasets/</td><td>Rdatasets/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/SPARK_README.md</td><td>SPARK_README.md</td><td>3359</td><td>1455043490000</td></tr><tr><td>dbfs:/databricks-datasets/adult/</td><td>adult/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/airlines/</td><td>airlines/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/amazon/</td><td>amazon/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/asa/</td><td>asa/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/atlas_higgs/</td><td>atlas_higgs/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/bikeSharing/</td><td>bikeSharing/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/cctvVideos/</td><td>cctvVideos/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/credit-card-fraud/</td><td>credit-card-fraud/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/cs100/</td><td>cs100/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/cs110x/</td><td>cs110x/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/cs190/</td><td>cs190/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/data.gov/</td><td>data.gov/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/definitive-guide/</td><td>definitive-guide/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/delta-sharing/</td><td>delta-sharing/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/flights/</td><td>flights/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/flower_photos/</td><td>flower_photos/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/flowers/</td><td>flowers/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/genomics/</td><td>genomics/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/hail/</td><td>hail/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/identifying-campaign-effectiveness/</td><td>identifying-campaign-effectiveness/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/iot/</td><td>iot/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/iot-stream/</td><td>iot-stream/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/learning-spark/</td><td>learning-spark/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/learning-spark-v2/</td><td>learning-spark-v2/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/lending-club-loan-stats/</td><td>lending-club-loan-stats/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/med-images/</td><td>med-images/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/media/</td><td>media/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/mnist-digits/</td><td>mnist-digits/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/news20.binary/</td><td>news20.binary/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/nyctaxi/</td><td>nyctaxi/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/nyctaxi-with-zipcodes/</td><td>nyctaxi-with-zipcodes/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/online_retail/</td><td>online_retail/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/overlap-join/</td><td>overlap-join/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/power-plant/</td><td>power-plant/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/retail-org/</td><td>retail-org/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/rwe/</td><td>rwe/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/sai-summit-2019-sf/</td><td>sai-summit-2019-sf/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/sample_logs/</td><td>sample_logs/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/samples/</td><td>samples/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/sfo_customer_survey/</td><td>sfo_customer_survey/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/sms_spam_collection/</td><td>sms_spam_collection/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/songs/</td><td>songs/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/structured-streaming/</td><td>structured-streaming/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/timeseries/</td><td>timeseries/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/tpch/</td><td>tpch/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/travel_recommendations_realtime/</td><td>travel_recommendations_realtime/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/warmup/</td><td>warmup/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/weather/</td><td>weather/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/wiki/</td><td>wiki/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/wikipedia-datasets/</td><td>wikipedia-datasets/</td><td>0</td><td>0</td></tr><tr><td>dbfs:/databricks-datasets/wine-quality/</td><td>wine-quality/</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         "dbfs:/databricks-datasets/COVID/",
         "COVID/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/README.md",
         "README.md",
         976,
         1532468253000
        ],
        [
         "dbfs:/databricks-datasets/Rdatasets/",
         "Rdatasets/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/SPARK_README.md",
         "SPARK_README.md",
         3359,
         1455043490000
        ],
        [
         "dbfs:/databricks-datasets/adult/",
         "adult/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/airlines/",
         "airlines/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/amazon/",
         "amazon/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/asa/",
         "asa/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/atlas_higgs/",
         "atlas_higgs/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/bikeSharing/",
         "bikeSharing/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/cctvVideos/",
         "cctvVideos/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/credit-card-fraud/",
         "credit-card-fraud/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/cs100/",
         "cs100/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/cs110x/",
         "cs110x/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/cs190/",
         "cs190/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/data.gov/",
         "data.gov/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/definitive-guide/",
         "definitive-guide/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/delta-sharing/",
         "delta-sharing/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/flights/",
         "flights/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/flower_photos/",
         "flower_photos/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/flowers/",
         "flowers/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/genomics/",
         "genomics/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/hail/",
         "hail/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/identifying-campaign-effectiveness/",
         "identifying-campaign-effectiveness/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/iot/",
         "iot/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/iot-stream/",
         "iot-stream/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/learning-spark/",
         "learning-spark/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/learning-spark-v2/",
         "learning-spark-v2/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/lending-club-loan-stats/",
         "lending-club-loan-stats/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/med-images/",
         "med-images/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/media/",
         "media/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/mnist-digits/",
         "mnist-digits/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/news20.binary/",
         "news20.binary/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/nyctaxi/",
         "nyctaxi/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/nyctaxi-with-zipcodes/",
         "nyctaxi-with-zipcodes/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/online_retail/",
         "online_retail/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/overlap-join/",
         "overlap-join/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/power-plant/",
         "power-plant/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/retail-org/",
         "retail-org/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/rwe/",
         "rwe/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/sai-summit-2019-sf/",
         "sai-summit-2019-sf/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/sample_logs/",
         "sample_logs/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/samples/",
         "samples/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/sfo_customer_survey/",
         "sfo_customer_survey/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/sms_spam_collection/",
         "sms_spam_collection/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/songs/",
         "songs/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/structured-streaming/",
         "structured-streaming/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/timeseries/",
         "timeseries/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/tpch/",
         "tpch/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/travel_recommendations_realtime/",
         "travel_recommendations_realtime/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/warmup/",
         "warmup/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/weather/",
         "weather/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/wiki/",
         "wiki/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/wikipedia-datasets/",
         "wikipedia-datasets/",
         0,
         0
        ],
        [
         "dbfs:/databricks-datasets/wine-quality/",
         "wine-quality/",
         0,
         0
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {
        "isDbfsCommandResult": false
       },
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "path",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "size",
         "type": "\"long\""
        },
        {
         "metadata": "{}",
         "name": "modificationTime",
         "type": "\"long\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%fs\n",
    "ls /databricks-datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "db121955-4ef6-44e9-905a-c319412ab051",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[2]: [FileInfo(path='dbfs:/databricks-datasets/', name='databricks-datasets/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-results/', name='databricks-results/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/user/', name='user/', size=0, modificationTime=0)]"
     ]
    }
   ],
   "source": [
    "#if we want to see all utility files  we can use dbutil.fs.ls command.\n",
    "# here we checked utility files of root folder of databricks\n",
    "dbutils.fs.ls('/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "2db06394-e60c-4454-90c2-0b497bd235ab",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[4]: [FileInfo(path='dbfs:/databricks-datasets/COVID/.DS_Store', name='.DS_Store', size=6148, modificationTime=1615608421000),\n FileInfo(path='dbfs:/databricks-datasets/COVID/CORD-19/', name='CORD-19/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/CSSEGISandData/', name='CSSEGISandData/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/ESRI_hospital_beds/', name='ESRI_hospital_beds/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/IHME/', name='IHME/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/USAFacts/', name='USAFacts/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/coronavirusdataset/', name='coronavirusdataset/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/covid-19-data/', name='covid-19-data/', size=0, modificationTime=0),\n FileInfo(path='dbfs:/databricks-datasets/COVID/download_daily_covid-19_datasets.sh', name='download_daily_covid-19_datasets.sh', size=1083, modificationTime=1609181325000),\n FileInfo(path='dbfs:/databricks-datasets/COVID/estimated_patient_impact_and_hospital_capacity_by_state/', name='estimated_patient_impact_and_hospital_capacity_by_state/', size=0, modificationTime=0)]"
     ]
    }
   ],
   "source": [
    "# the directories above contains few datasets that we can use. for example we enter into Covid folder\n",
    "dbutils.fs.ls('/databricks-datasets/COVID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "e3c5a933-c1af-4332-a061-88dacee6fe20",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FileInfo(path='dbfs:/databricks-datasets/COVID/.DS_Store', name='.DS_Store', size=6148, modificationTime=1615608421000)\nFileInfo(path='dbfs:/databricks-datasets/COVID/CORD-19/', name='CORD-19/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/CSSEGISandData/', name='CSSEGISandData/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/ESRI_hospital_beds/', name='ESRI_hospital_beds/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/IHME/', name='IHME/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/USAFacts/', name='USAFacts/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/coronavirusdataset/', name='coronavirusdataset/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/covid-19-data/', name='covid-19-data/', size=0, modificationTime=0)\nFileInfo(path='dbfs:/databricks-datasets/COVID/download_daily_covid-19_datasets.sh', name='download_daily_covid-19_datasets.sh', size=1083, modificationTime=1609181325000)\nFileInfo(path='dbfs:/databricks-datasets/COVID/estimated_patient_impact_and_hospital_capacity_by_state/', name='estimated_patient_impact_and_hospital_capacity_by_state/', size=0, modificationTime=0)\n"
     ]
    }
   ],
   "source": [
    "for files in dbutils.fs.ls('/databricks-datasets/COVID'):\n",
    "    print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "aeea73c8-47e5-4f25-9f69-e4f8943cf7e8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CORD-19/\nCSSEGISandData/\nESRI_hospital_beds/\nIHME/\nUSAFacts/\ncoronavirusdataset/\ncovid-19-data/\nestimated_patient_impact_and_hospital_capacity_by_state/\n"
     ]
    }
   ],
   "source": [
    "for files in dbutils.fs.ls('/databricks-datasets/COVID'):\n",
    "    if files.name.endswith('/'):\n",
    "        print(files.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "2f298b2e-97e5-49c7-9fae-f791ffc9b31a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<div class = \"ansiout\"><b>dbutils.fs</b> provides utilities for working with FileSystems. Most methods in\n",
       "this package can take either a DBFS path (e.g., \"/foo\" or \"dbfs:/foo\"), or\n",
       "another FileSystem URI.\n",
       "\n",
       "For more info about a method, use <b>dbutils.fs.help(\"methodName\")</b>.\n",
       "\n",
       "In notebooks, you can also use the %fs shorthand to access DBFS. The %fs shorthand maps\n",
       "straightforwardly onto dbutils calls. For example, \"%fs head --maxBytes=10000 /file/path\"\n",
       "translates into \"dbutils.fs.head(\"/file/path\", maxBytes = 10000)\".\n",
       "    <h3>mount</h3><b>mount(source: String, mountPoint: String, encryptionType: String = \"\", owner: String = null, extraConfigs: Map = Map.empty[String, String]): boolean</b> -> Mounts the given source directory into DBFS at the given mount point<br /><b>mounts: Seq</b> -> Displays information about what is mounted within DBFS<br /><b>refreshMounts: boolean</b> -> Forces all machines in this cluster to refresh their mount cache, ensuring they receive the most recent information<br /><b>unmount(mountPoint: String): boolean</b> -> Deletes a DBFS mount point<br /><b>updateMount(source: String, mountPoint: String, encryptionType: String = \"\", owner: String = null, extraConfigs: Map = Map.empty[String, String]): boolean</b> -> Similar to mount(), but updates an existing mount point (if present) instead of creating a new one<br /><br /><h3>fsutils</h3><b>cp(from: String, to: String, recurse: boolean = false): boolean</b> -> Copies a file or directory, possibly across FileSystems<br /><b>head(file: String, maxBytes: int = 65536): String</b> -> Returns up to the first 'maxBytes' bytes of the given file as a String encoded in UTF-8<br /><b>ls(dir: String): Seq</b> -> Lists the contents of a directory<br /><b>mkdirs(dir: String): boolean</b> -> Creates the given directory if it does not exist, also creating any necessary parent directories<br /><b>mv(from: String, to: String, recurse: boolean = false): boolean</b> -> Moves a file or directory, possibly across FileSystems<br /><b>put(file: String, contents: String, overwrite: boolean = false): boolean</b> -> Writes the given String out to a file, encoded in UTF-8<br /><b>rm(dir: String, recurse: boolean = false): boolean</b> -> Removes a file or directory<br /><br /></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class = \"ansiout\"><b>dbutils.fs</b> provides utilities for working with FileSystems. Most methods in\nthis package can take either a DBFS path (e.g., \"/foo\" or \"dbfs:/foo\"), or\nanother FileSystem URI.\n\nFor more info about a method, use <b>dbutils.fs.help(\"methodName\")</b>.\n\nIn notebooks, you can also use the %fs shorthand to access DBFS. The %fs shorthand maps\nstraightforwardly onto dbutils calls. For example, \"%fs head --maxBytes=10000 /file/path\"\ntranslates into \"dbutils.fs.head(\"/file/path\", maxBytes = 10000)\".\n    <h3>mount</h3><b>mount(source: String, mountPoint: String, encryptionType: String = \"\", owner: String = null, extraConfigs: Map = Map.empty[String, String]): boolean</b> -> Mounts the given source directory into DBFS at the given mount point<br /><b>mounts: Seq</b> -> Displays information about what is mounted within DBFS<br /><b>refreshMounts: boolean</b> -> Forces all machines in this cluster to refresh their mount cache, ensuring they receive the most recent information<br /><b>unmount(mountPoint: String): boolean</b> -> Deletes a DBFS mount point<br /><b>updateMount(source: String, mountPoint: String, encryptionType: String = \"\", owner: String = null, extraConfigs: Map = Map.empty[String, String]): boolean</b> -> Similar to mount(), but updates an existing mount point (if present) instead of creating a new one<br /><br /><h3>fsutils</h3><b>cp(from: String, to: String, recurse: boolean = false): boolean</b> -> Copies a file or directory, possibly across FileSystems<br /><b>head(file: String, maxBytes: int = 65536): String</b> -> Returns up to the first 'maxBytes' bytes of the given file as a String encoded in UTF-8<br /><b>ls(dir: String): Seq</b> -> Lists the contents of a directory<br /><b>mkdirs(dir: String): boolean</b> -> Creates the given directory if it does not exist, also creating any necessary parent directories<br /><b>mv(from: String, to: String, recurse: boolean = false): boolean</b> -> Moves a file or directory, possibly across FileSystems<br /><b>put(file: String, contents: String, overwrite: boolean = false): boolean</b> -> Writes the given String out to a file, encoded in UTF-8<br /><b>rm(dir: String, recurse: boolean = false): boolean</b> -> Removes a file or directory<br /><br /></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "textData": null,
       "type": "htmlSandbox"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dbutils.fs.help()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "e3305018-06a2-4f3c-aca0-2a413455779a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<div class = \"ansiout\">/**<br /> * Lists the contents of a directory.<br /> * <br /> * Example: display(ls(\"/mnt/my-folder/\"))<br /> * <br /> * The FileInfo object that is returned has the following helper methods:<br /> * val files = ls(\"/mnt/my-folder/\")<br /> * files.map(_.name)    // [myFile, myDir/]<br /> * files.map(_.length)  // [1286, 0]<br /> * files.map(_.path)    // [/mnt/my-folder/myFile, /mnt/my-folder/myDir/]<br /> * files.map(_.isDir)   // [false, true]<br /> * files.map(_.isFile)  // [true, false]<br /> * files.map(_.modificationTime)  // [1615937446000, 1615937224000]<br /> * <br /> * @param dir FileSystem URI<br /> * @return Ordered sequence of FileInfos containing the name, size and modification time<br /> *  of each file.<br /> */<br /><b>ls(dir: java.lang.String): scala.collection.Seq</b></div><br />"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class = \"ansiout\">/**<br /> * Lists the contents of a directory.<br /> * <br /> * Example: display(ls(\"/mnt/my-folder/\"))<br /> * <br /> * The FileInfo object that is returned has the following helper methods:<br /> * val files = ls(\"/mnt/my-folder/\")<br /> * files.map(_.name)    // [myFile, myDir/]<br /> * files.map(_.length)  // [1286, 0]<br /> * files.map(_.path)    // [/mnt/my-folder/myFile, /mnt/my-folder/myDir/]<br /> * files.map(_.isDir)   // [false, true]<br /> * files.map(_.isFile)  // [true, false]<br /> * files.map(_.modificationTime)  // [1615937446000, 1615937224000]<br /> * <br /> * @param dir FileSystem URI<br /> * @return Ordered sequence of FileInfos containing the name, size and modification time<br /> *  of each file.<br /> */<br /><b>ls(dir: java.lang.String): scala.collection.Seq</b></div><br />",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "textData": null,
       "type": "htmlSandbox"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dbutils.fs.help('ls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e5058871-ad70-4518-905a-249db60c5ae8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "mostRecentlyExecutedCommandWithImplicitDF": {
     "commandId": 1234352451264073,
     "dataframes": [
      "_sqldf"
     ]
    },
    "pythonIndentUnit": 4
   },
   "notebookName": "Untitled Notebook 2024-10-17 21:46:13",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}