Skip to content

Commit

Permalink
created tables for processed data in BigQuery
Browse files Browse the repository at this point in the history
  • Loading branch information
@ridhi96
ridhi96 committed Nov 16, 2023
1 parent c3c1ba2 commit 1be0b58
Show file tree
Hide file tree
Showing 2 changed files with 231 additions and 0 deletions.
37 changes: 37 additions & 0 deletions createTables.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

PROJECT_ID="msca-bdp-student-ap"
DATASET_NAME="chicago_rideshare"
BUCKET_NAME="msca-bdp-student-gcs"
FOLDER_PATH="bdp-rideshare-project/rideshare/processed_data"

# Specify the folder names containing multiple CSV files
FOLDERS=("program_area_2020.csv"
"program_area_time_rides_2018.csv"
"program_area_time_rides_2019.csv"
"program_area_time_rides_2021.csv"
"program_area_time_rides_2022.csv"
"rides_2018.csv"
"rides_2019.csv"
"rides_2020.csv"
"rides_2021.csv"
"rides_2022.csv")

# Load data into BigQuery for the specified folder
load_into_bigquery() {
local TABLE_NAME="$1"
local FOLDER_NAME="$2"

# Construct the GCS file path for CSV files within the folder
GCS_FOLDER_PATH="gs://$BUCKET_NAME/$FOLDER_PATH/$FOLDER_NAME/*.csv"
echo "GCS Folder Path: $GCS_FOLDER_PATH"

# Append all CSV files from the folder into the BigQuery table
bq load --autodetect --source_format=CSV --noreplace "$PROJECT_ID:$DATASET_NAME.$TABLE_NAME" "$GCS_FOLDER_PATH"
}

# Load data into BigQuery
for FOLDER in "${FOLDERS[@]}"; do
TABLE=$(basename "$FOLDER" .csv)
load_into_bigquery "$TABLE" "$FOLDER"
done
194 changes: 194 additions & 0 deletions geospatial.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "5e5fe419-6a99-4aa5-a4b4-c14f0c8038ec",
"metadata": {},
"source": [
"**Execute createTables.sh to create BigQuery tables of processed data**"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "24c29de1-76dd-4574-ac41-f702016a48ed",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/program_area_2020.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r437832d57d68de03_0000018bda760bc8_1 ... (4s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/program_area_time_rides_2018.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r5c5c7815a20d04bc_0000018bda762730_1 ... (2s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/program_area_time_rides_2019.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r7f7904794e836ed3_0000018bda763a3f_1 ... (4s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/program_area_time_rides_2021.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r70332df88fd21e9c_0000018bda76550c_1 ... (3s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/program_area_time_rides_2022.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r6b92ee689f011b8f_0000018bda766b9e_1 ... (5s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/rides_2018.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r396e8a956fe2afb_0000018bda768ad1_1 ... (14s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/rides_2019.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r4d4c4ae64b77786d_0000018bda76cebf_1 ... (14s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/rides_2020.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r580e35676ff43a38_0000018bda77125d_1 ... (22s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/rides_2021.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r730d35da7bd6d21d_0000018bda7774f4_1 ... (22s) Current status: DONE \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GCS Folder Path: gs://msca-bdp-student-gcs/bdp-rideshare-project/rideshare/processed_data/rides_2022.csv/*.csv\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Waiting on bqjob_r26148a34bbf177c9_0000018bda77d7a8_1 ... (22s) Current status: DONE \n"
]
}
],
"source": [
"%%bash\n",
"\n",
"bash \"bdp-rideshare/createTables.sh\"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8d4c7418-8481-427c-8097-6e7dae4058d4",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

0 comments on commit 1be0b58

Please sign in to comment.