From 18dc0e472ffe9bf63657c34c3fce7575fc364908 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sat, 3 Aug 2024 14:50:11 -0400 Subject: [PATCH 01/14] Cleaned up returns on failure, updated docs --- docs/google.rst | 2 + parsons/google/google_bigquery.py | 87 +++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/docs/google.rst b/docs/google.rst index 9590d51f20..d3f1243fc7 100644 --- a/docs/google.rst +++ b/docs/google.rst @@ -142,6 +142,8 @@ API :inherited-members: + + ************* Cloud Storage ************* diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index ec1b8c5b08..a4ec0bef36 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1451,6 +1451,93 @@ def extract( logger.info(f"Finished exporting query result to {gs_destination}.") + def copy_between_projects(self, source_project, source_dataset, source_table, + destination_project, destination_dataset, destination_table, + if_dataset_not_exists='fail', if_table_exists='fail'): + """ + Copy a table from one project to another. Set the flags if_dataset_not_exists and + if_table_exists to control behavior if target dataset for the table does not exist + and if the target table exists (default is to fail in both cases). + Fails if the source or target project does not exist. + + `Args`: + source_project: str + Name of source project + source_dataset: str + Name of source dataset + source_table: str + Name of source table + destination_project: str + Name of destination project + destination_dataset: str + Name of destination dataset + destination_table: str + Name of destination table + if_dataset_not_exists: str + Action if dataset doesn't exist {'fail','create'} + if_table_exists: str + Action if table exists {'fail', 'overwrite'} + + `Returns:` + None + """ + + from google.cloud import bigquery + from google.cloud.exceptions import NotFound + import sys + + destination_table_id = (destination_project + "." + + destination_dataset + "." + + destination_table) + source_table_id = (source_project + "." + + source_dataset + "." + + source_table) + + # check if destination dataset exists + try: + dataset_id = destination_project + "." + destination_dataset + self.client.get_dataset(dataset_id) # Make an API request. + # if it exists: continue + except NotFound: + # if it doesn't exist: check the flag for creating it + if if_dataset_not_exists == 'create': # create a new dataset in the destination + dataset = bigquery.Dataset(dataset_id) + dataset = self.client.create_dataset(dataset, timeout=30) + else: # fail + print("BigQuery copy failed") + sys.exit("Dataset {0} does not exist and if_dataset_not_exists set to {1}" + .format(destination_dataset, if_dataset_not_exists)) + + job_config = bigquery.CopyJobConfig() + + # check if destination table exists + try: + # check if table exists + self.client.get_table(destination_table_id) + if if_table_exists == 'overwrite': # if it exists + job_config = bigquery.CopyJobConfig() + job_config.write_disposition = "WRITE_TRUNCATE" + try: + job = self.client.copy_table(source_table_id, destination_table_id, + location='US', job_config=job_config) + result = job.result() + except Exception as e: + print("Exception during copy between projects", e) + else: + print("BigQuery copy failed") + sys.exit("Table {0} exists and if_table_exists set to {1}" + .format(destination_table, if_table_exists)) + + except NotFound: + # table doesn't exist + try: + job = self.client.copy_table(source_table_id, destination_table_id, + location='US', job_config=job_config) + result = job.result() + print(result) + except Exception as e: + print("Exception during copy between projects", e) + class BigQueryTable(BaseTable): """BigQuery table object.""" From 65e1a63ed6c5419104dbf436c2f0ba274f31788c Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sat, 3 Aug 2024 15:29:08 -0400 Subject: [PATCH 02/14] Cleaned up returns on failure, updated docs --- parsons/google/google_bigquery.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index a4ec0bef36..e8109a01ef 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1486,12 +1486,12 @@ def copy_between_projects(self, source_project, source_dataset, source_table, from google.cloud.exceptions import NotFound import sys - destination_table_id = (destination_project + "." + - destination_dataset + "." + - destination_table) - source_table_id = (source_project + "." + - source_dataset + "." + - source_table) + destination_table_id = ( destination_project + "." + + destination_dataset + "." + + destination_table) + source_table_id = ( source_project + "." + + source_dataset + "." + + source_table) # check if destination dataset exists try: From c7ce13f81fff2d631aee96d125e002b16fa4a4a2 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sat, 3 Aug 2024 15:33:23 -0400 Subject: [PATCH 03/14] Cleaned up returns on failure, updated docs --- parsons/google/google_bigquery.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index e8109a01ef..b3ef619c4b 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1486,10 +1486,10 @@ def copy_between_projects(self, source_project, source_dataset, source_table, from google.cloud.exceptions import NotFound import sys - destination_table_id = ( destination_project + "." - + destination_dataset + "." - + destination_table) - source_table_id = ( source_project + "." + destination_table_id = (destination_project + "." + + destination_dataset + "." + + destination_table) + source_table_id = (source_project + "." + source_dataset + "." + source_table) From 14984c69649d59a5607d513bc584f723a892e0b5 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sat, 3 Aug 2024 15:33:58 -0400 Subject: [PATCH 04/14] Cleaned up returns on failure, updated docs --- parsons/google/google_bigquery.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index b3ef619c4b..6a5eb64f23 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1487,11 +1487,11 @@ def copy_between_projects(self, source_project, source_dataset, source_table, import sys destination_table_id = (destination_project + "." - + destination_dataset + "." - + destination_table) + + destination_dataset + "." + + destination_table) source_table_id = (source_project + "." - + source_dataset + "." - + source_table) + + source_dataset + "." + + source_table) # check if destination dataset exists try: From 79e7057b4ed3e8128782bb35fe63728e2b3f1617 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sat, 3 Aug 2024 16:02:11 -0400 Subject: [PATCH 05/14] more de-linting --- parsons/google/google_bigquery.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 6a5eb64f23..72979137ec 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1486,12 +1486,12 @@ def copy_between_projects(self, source_project, source_dataset, source_table, from google.cloud.exceptions import NotFound import sys - destination_table_id = (destination_project + "." - + destination_dataset + "." - + destination_table) - source_table_id = (source_project + "." - + source_dataset + "." - + source_table) + destination_table_id = (destination_project + "." + + destination_dataset + "." + + destination_table) + source_table_id = (source_project + "." + + source_dataset + "." + + source_table) # check if destination dataset exists try: From 441c206f8f5f67ea89686b0dffd01febd382ffce Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Fri, 30 Aug 2024 11:15:11 -0400 Subject: [PATCH 06/14] fixed issues with PR --- parsons/google/google_bigquery.py | 46 ++++++++++++------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 72979137ec..8d8ad245d8 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1455,10 +1455,11 @@ def copy_between_projects(self, source_project, source_dataset, source_table, destination_project, destination_dataset, destination_table, if_dataset_not_exists='fail', if_table_exists='fail'): """ - Copy a table from one project to another. Set the flags if_dataset_not_exists and - if_table_exists to control behavior if target dataset for the table does not exist - and if the target table exists (default is to fail in both cases). - Fails if the source or target project does not exist. + Copy a table from one project to another. Fails if the source or target project does not exist. + If the target dataset does not exist, fhe flag if_dataset_not_exists controls behavior. It defaults to 'fail'; + set it to 'create' if it's ok to create it. + If the target table exists, the flag if_table_exists controls behavior. It defaults to 'fail'; + set it to 'overwrite' if it's ok to overwrite an existing table. `Args`: source_project: str @@ -1484,7 +1485,6 @@ def copy_between_projects(self, source_project, source_dataset, source_table, from google.cloud import bigquery from google.cloud.exceptions import NotFound - import sys destination_table_id = (destination_project + "." + destination_dataset + "." @@ -1497,47 +1497,37 @@ def copy_between_projects(self, source_project, source_dataset, source_table, try: dataset_id = destination_project + "." + destination_dataset self.client.get_dataset(dataset_id) # Make an API request. - # if it exists: continue + # if it exists: continue; if not, check to see if it's ok to create it except NotFound: - # if it doesn't exist: check the flag for creating it + # if it doesn't exist: check if it's ok to create it if if_dataset_not_exists == 'create': # create a new dataset in the destination dataset = bigquery.Dataset(dataset_id) dataset = self.client.create_dataset(dataset, timeout=30) - else: # fail - print("BigQuery copy failed") - sys.exit("Dataset {0} does not exist and if_dataset_not_exists set to {1}" + else: # if it doesn't exist and it's not ok to create it, fail + logger.error("BigQuery copy failed, Dataset {0} does not exist and if_dataset_not_exists set to {1}" .format(destination_dataset, if_dataset_not_exists)) job_config = bigquery.CopyJobConfig() # check if destination table exists try: - # check if table exists self.client.get_table(destination_table_id) if if_table_exists == 'overwrite': # if it exists job_config = bigquery.CopyJobConfig() job_config.write_disposition = "WRITE_TRUNCATE" - try: - job = self.client.copy_table(source_table_id, destination_table_id, - location='US', job_config=job_config) - result = job.result() - except Exception as e: - print("Exception during copy between projects", e) - else: - print("BigQuery copy failed") - sys.exit("Table {0} exists and if_table_exists set to {1}" - .format(destination_table, if_table_exists)) - - except NotFound: - # table doesn't exist - try: job = self.client.copy_table(source_table_id, destination_table_id, location='US', job_config=job_config) result = job.result() - print(result) - except Exception as e: - print("Exception during copy between projects", e) + else: + logger.error("BigQuery copy failed, Table {0} exists and if_table_exists set to {1}" + .format(destination_table, if_table_exists)) + except NotFound: + # destination table doesn't exist, so we can create one + job = self.client.copy_table(source_table_id, destination_table_id, + location='US', job_config=job_config) + result = job.result() + logger.info(result) class BigQueryTable(BaseTable): """BigQuery table object.""" From b10d640a85c1d125028612d970d18f98db42d2ba Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Fri, 30 Aug 2024 11:26:24 -0400 Subject: [PATCH 07/14] fixed issues with PR --- parsons/google/google_bigquery.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 8d8ad245d8..1b3bd61c96 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1455,11 +1455,12 @@ def copy_between_projects(self, source_project, source_dataset, source_table, destination_project, destination_dataset, destination_table, if_dataset_not_exists='fail', if_table_exists='fail'): """ - Copy a table from one project to another. Fails if the source or target project does not exist. - If the target dataset does not exist, fhe flag if_dataset_not_exists controls behavior. It defaults to 'fail'; - set it to 'create' if it's ok to create it. - If the target table exists, the flag if_table_exists controls behavior. It defaults to 'fail'; - set it to 'overwrite' if it's ok to overwrite an existing table. + Copy a table from one project to another. Fails if the source or target project + does not exist. + If the target dataset does not exist, fhe flag if_dataset_not_exists controls behavior. + It defaults to 'fail'; set it to 'create' if it's ok to create it. + If the target table exists, the flag if_table_exists controls behavior. + It defaults to 'fail'; set it to 'overwrite' if it's ok to overwrite an existing table. `Args`: source_project: str @@ -1492,10 +1493,10 @@ def copy_between_projects(self, source_project, source_dataset, source_table, source_table_id = (source_project + "." + source_dataset + "." + source_table) + dataset_id = destination_project + "." + destination_dataset # check if destination dataset exists try: - dataset_id = destination_project + "." + destination_dataset self.client.get_dataset(dataset_id) # Make an API request. # if it exists: continue; if not, check to see if it's ok to create it except NotFound: @@ -1504,8 +1505,9 @@ def copy_between_projects(self, source_project, source_dataset, source_table, dataset = bigquery.Dataset(dataset_id) dataset = self.client.create_dataset(dataset, timeout=30) else: # if it doesn't exist and it's not ok to create it, fail - logger.error("BigQuery copy failed, Dataset {0} does not exist and if_dataset_not_exists set to {1}" - .format(destination_dataset, if_dataset_not_exists)) + logger.error("BigQuery copy failed") + logger.error("Dataset {0} does not exist and if_dataset_not_exists set to {1}" + .format(destination_dataset, if_dataset_not_exists)) job_config = bigquery.CopyJobConfig() @@ -1520,7 +1522,7 @@ def copy_between_projects(self, source_project, source_dataset, source_table, result = job.result() else: logger.error("BigQuery copy failed, Table {0} exists and if_table_exists set to {1}" - .format(destination_table, if_table_exists)) + .format(destination_table, if_table_exists)) except NotFound: # destination table doesn't exist, so we can create one @@ -1529,6 +1531,7 @@ def copy_between_projects(self, source_project, source_dataset, source_table, result = job.result() logger.info(result) + class BigQueryTable(BaseTable): """BigQuery table object.""" From 6f0b47199e73599a3d858fc95d1aa02ce2584ae4 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Mon, 9 Dec 2024 14:31:38 -0500 Subject: [PATCH 08/14] fixed issues with PR --- parsons/google/google_bigquery.py | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 1b3bd61c96..1ed59cc9ac 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1451,9 +1451,17 @@ def extract( logger.info(f"Finished exporting query result to {gs_destination}.") - def copy_between_projects(self, source_project, source_dataset, source_table, - destination_project, destination_dataset, destination_table, - if_dataset_not_exists='fail', if_table_exists='fail'): + def copy_between_projects( + self, + source_project, + source_dataset, + source_table, + destination_project, + destination_dataset, + destination_table, + if_dataset_not_exists="fail", + if_table_exists="fail", + ): """ Copy a table from one project to another. Fails if the source or target project does not exist. @@ -1487,12 +1495,10 @@ def copy_between_projects(self, source_project, source_dataset, source_table, from google.cloud import bigquery from google.cloud.exceptions import NotFound - destination_table_id = (destination_project + "." - + destination_dataset + "." - + destination_table) - source_table_id = (source_project + "." - + source_dataset + "." - + source_table) + destination_table_id = ( + destination_project + "." + destination_dataset + "." + destination_table + ) + source_table_id = source_project + "." + source_dataset + "." + source_table dataset_id = destination_project + "." + destination_dataset # check if destination dataset exists @@ -1501,33 +1507,41 @@ def copy_between_projects(self, source_project, source_dataset, source_table, # if it exists: continue; if not, check to see if it's ok to create it except NotFound: # if it doesn't exist: check if it's ok to create it - if if_dataset_not_exists == 'create': # create a new dataset in the destination + if if_dataset_not_exists == "create": # create a new dataset in the destination dataset = bigquery.Dataset(dataset_id) dataset = self.client.create_dataset(dataset, timeout=30) else: # if it doesn't exist and it's not ok to create it, fail logger.error("BigQuery copy failed") - logger.error("Dataset {0} does not exist and if_dataset_not_exists set to {1}" - .format(destination_dataset, if_dataset_not_exists)) + logger.error( + "Dataset {0} does not exist and if_dataset_not_exists set to {1}".format( + destination_dataset, if_dataset_not_exists + ) + ) job_config = bigquery.CopyJobConfig() # check if destination table exists try: self.client.get_table(destination_table_id) - if if_table_exists == 'overwrite': # if it exists + if if_table_exists == "overwrite": # if it exists job_config = bigquery.CopyJobConfig() job_config.write_disposition = "WRITE_TRUNCATE" - job = self.client.copy_table(source_table_id, destination_table_id, - location='US', job_config=job_config) + job = self.client.copy_table( + source_table_id, destination_table_id, location='US', job_config=job_config + ) result = job.result() else: - logger.error("BigQuery copy failed, Table {0} exists and if_table_exists set to {1}" - .format(destination_table, if_table_exists)) + logger.error( + "BigQuery copy failed, Table {0} exists and if_table_exists set to {1}".format( + destination_table, if_table_exists + ) + ) except NotFound: # destination table doesn't exist, so we can create one - job = self.client.copy_table(source_table_id, destination_table_id, - location='US', job_config=job_config) + job = self.client.copy_table( + source_table_id, destination_table_id, location='US', job_config=job_config + ) result = job.result() logger.info(result) From 09880eae0f6795f27b9228acca5145ed4931ef7a Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Mon, 9 Dec 2024 14:32:08 -0500 Subject: [PATCH 09/14] Fixed formatting errors flagged by ruff --- bin/Activate.ps1 | 241 ++++++++++++++++++++++++++++++++++++++++++++++ bin/activate | 66 +++++++++++++ bin/activate.csh | 25 +++++ bin/activate.fish | 64 ++++++++++++ bin/black | 8 ++ bin/blackd | 8 ++ bin/delme.py | 71 ++++++++++++++ bin/flake8 | 8 ++ bin/flake8p | 8 ++ bin/normalizer | 8 ++ bin/pip | 8 ++ bin/pip3 | 8 ++ bin/pip3.9 | 8 ++ bin/py.test | 8 ++ bin/pycodestyle | 8 ++ bin/pyflakes | 8 ++ bin/pytest | 8 ++ bin/python | 1 + bin/python3 | 1 + bin/python3.9 | 1 + pyvenv.cfg | 3 + 21 files changed, 569 insertions(+) create mode 100644 bin/Activate.ps1 create mode 100644 bin/activate create mode 100644 bin/activate.csh create mode 100644 bin/activate.fish create mode 100755 bin/black create mode 100755 bin/blackd create mode 100644 bin/delme.py create mode 100755 bin/flake8 create mode 100755 bin/flake8p create mode 100755 bin/normalizer create mode 100755 bin/pip create mode 100755 bin/pip3 create mode 100755 bin/pip3.9 create mode 100755 bin/py.test create mode 100755 bin/pycodestyle create mode 100755 bin/pyflakes create mode 100755 bin/pytest create mode 120000 bin/python create mode 120000 bin/python3 create mode 120000 bin/python3.9 create mode 100644 pyvenv.cfg diff --git a/bin/Activate.ps1 b/bin/Activate.ps1 new file mode 100644 index 0000000000..7bc3e7c698 --- /dev/null +++ b/bin/Activate.ps1 @@ -0,0 +1,241 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/bin/activate b/bin/activate new file mode 100644 index 0000000000..0b8e1c605a --- /dev/null +++ b/bin/activate @@ -0,0 +1,66 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/charleskramer/Documents/parsons" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(parsons) ${PS1:-}" + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/bin/activate.csh b/bin/activate.csh new file mode 100644 index 0000000000..1461282270 --- /dev/null +++ b/bin/activate.csh @@ -0,0 +1,25 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/charleskramer/Documents/parsons" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(parsons) $prompt" +endif + +alias pydoc python -m pydoc + +rehash diff --git a/bin/activate.fish b/bin/activate.fish new file mode 100644 index 0000000000..3dfde83891 --- /dev/null +++ b/bin/activate.fish @@ -0,0 +1,64 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/charleskramer/Documents/parsons" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "(parsons) " (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/bin/black b/bin/black new file mode 100755 index 0000000000..4b9f7a05c7 --- /dev/null +++ b/bin/black @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from black import patched_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(patched_main()) diff --git a/bin/blackd b/bin/blackd new file mode 100755 index 0000000000..a42a841889 --- /dev/null +++ b/bin/blackd @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from blackd import patched_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(patched_main()) diff --git a/bin/delme.py b/bin/delme.py new file mode 100644 index 0000000000..ceca41fab5 --- /dev/null +++ b/bin/delme.py @@ -0,0 +1,71 @@ +import sys +#sys.path.append("~/Documents/parsons") +sys.path.append('/Users/charleskramer/Documents/parsons') +import parsons +from google.auth import default +app_creds,_ = default() + +# app_creds = { +# "type": "service_account", +# "project_id": "sql-practice-425619", +# "private_key_id": "31d4ba8aa75b0fa6a892072e4cb73d907940f24e", +# "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMbfuUhBQX92vv\nYdXaw6gNNLQSY+Mu+Dg2fGN0tqZrJn3HaHeBS1GWoF3rJE9M9/vmivuNSuaPgBts\nqY9x/yQ/6zYtQ62vPO0LMtLWB+NdP2Ha87eFbPEMJA2RuTubeEwfEfnv82ZBVXrI\n9+1DwbPqEqg8CrVEIBC9JJq4qyYEXj6c3ol8DcgB9Vwz8d/XrIQ7Xam2wBF/6Eba\nB8qiyzF0N40YlkjQLZ3MlQvZFBxgXAGz9qjFjTNvhrvBqz+l2+IXPzLAecvWgEck\nekbp/QsfPrCkCd614pXFdklPUzoFaznlu2EpjeQMPH3Jc9vFIzI6EjppP1wKThNk\nQPsVaedDAgMBAAECggEALd++VVkAM1Kb33d6aGGAjBoYEIpmuCsOXjeyj8XO+XuF\nPSK1sodm0yDAgpw+yVxapCTrBw1YSLpsLQmtvVyOU2OiYCzwREMtRFaO1mWlwU6Q\nVHRdae0E+H78podFF4G6vzwm9JAPkbivWXq9PPeyOmQeaX7Tp2p0pyUYLTmGyfK/\nbXT7GiIT74U6mK2mhjUws36gp/bZj2ciwOPLdKZz6qFT7c2ncp3xHZGtZOGQi0WZ\neYu0dxu8MUSNWA6yf0tdhGLGY6s5D9lgUd6S7fREd5146gc95fRmWwNXkl1ywLTp\nLHx6IvSe0Xuo6gg0kKCJmeRY26gTWl0QtsdWkqjdWQKBgQDrtOu6vBgvhm27b4VA\nFkVxHPlNNFgnegt2EhCmK2WCqCk7GbQrkQRlHr+UT5SL+gDbcTXuc34ui+/xm0XG\nUqjs2+1ntX2nkM1h7hdhPwSiiizlX2ggx7Mvz2gWBkOaXX3sBYy3ixfkzIY78Y8Q\nwIfnG+3oBsOKwP/2ET+K5UAmewKBgQDeB7NveBuTvMK7evhUzWAGvSquBHAn9BzI\n18vlkywODFhuQ0QqPses/ErB/KpercaMSnqBtyFnlpMzqYiH/aLSAPTghkLhfO7i\n9m8RfXtSPftj7glgm0uVfkOPkzupeJFRTEvTiW/qyyUBCGEszCujpMVGFh5/OsNF\n16Xp1IYL2QKBgC1YYQFLrlt2QSYODlSF9FYOfOedalgt4oaUcx+EucKpF0WHbGH1\nRCMwBytBJBLJxeBxpy54iY3q3f5dIM9Gl7j5lnKdN89EzD5Kz7Slqv6aFokKEb48\nrPxFqoCSM+8+jTsa1jd5St95eVmO3zcZ0wtjFCHPK09GUffE2bSe5fiBAoGBAIXg\nxNx44sABeLYNXwHbWISXuc12FZ6xvk2IeYSzakQKQ6Qw4UBL3KC8++LelGhEhkz/\nd8ERiI3qqFXlatL9KBlSxFVB/7/xUiX+jFhSfnLHtva5iGP9H+VDXj+LetUkQxNv\nWuVxoM9FsMlfoSCe981TxDQPkFINP8O0VXGaWT15AoGAfGtNOQ25T41F2DLo6JSe\noayqyJe2b99jL92+BHD987U9w//j/FzwnYBUcXMKuYma75LYkpampAxtuMuamlNn\nq6FLJsfTJ7a9pP3ifN4tcdrK2v2FT4yK1d03kTG6HNNaqZBat5oe4y+8DeupWovW\nU17W2i4QavuYn0FNFMXOusc=\n-----END PRIVATE KEY-----\n", +# "client_email": "parsons-bq@sql-practice-425619.iam.gserviceaccount.com", +# "client_id": "104516682065371288685", +# "auth_uri": "https://accounts.google.com/o/oauth2/auth", +# "token_uri": "https://oauth2.googleapis.com/token", +# "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", +# "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/parsons-bq%40sql-practice-425619.iam.gserviceaccount.com", +# "universe_domain": "googleapis.com" +# } +# + + +gbq = parsons.GoogleBigQuery(app_creds=app_creds) + + + +# input parameters +source_project = 'sql-practice-425619' +source_dataset = 'bqtest' +source_table = 'va_2022_general' +# source_table_id = "sql-practice-425619.bqtest.va_2022_general" +# destination_table_id = "bq-for.bq2test.va_2022_general" +destination_project = 'bq-for' +destination_dataset = 'bqnew' +destination_table = 'va_2022_general_copy' +if_dataset_not_exists = 'create' # create +if_table_exists = 'overwrite' # overwrite + +#Access Denied: Dataset bq-for:bqnew: Permission bigquery.datasets.get denied on dataset bq-for:bqnew (or it may not exist). + +destination_project = 'sql-practice-425619' +destination_dataset = 'bqtest' +destination_table = 'va_2022_general' +# source_table_id = "sql-practice-425619.bqtest.va_2022_general" +# destination_table_id = "bq-for.bq2test.va_2022_general" +source_project = 'bq-for' +source_dataset = 'bqnew' +source_table = 'va_2022_general_copy' +if_dataset_not_exists = 'create' # create +if_table_exists = 'overwrite' # overwrite + +#Access Denied: Dataset sql-practice-425619:bqtest: Permission bigquery.datasets.get denied on dataset sql-practice-425619:bqtest (or it may not exist). + +# removed the client call in the method +source_project = 'sql-practice-425619' +source_dataset = 'bqtest' +source_table = 'va_2022_general' +# source_table_id = "sql-practice-425619.bqtest.va_2022_general" +# destination_table_id = "bq-for.bq2test.va_2022_general" +destination_project = 'bq-for' +destination_dataset = 'bqnew' +destination_table = 'va_2022_general_copy' + +if_dataset_not_exists = 'create' # create +if_table_exists = 'overwrite' # overwrite + + +gbq.copy_between_projects(source_project,source_dataset,source_table, + destination_project,destination_dataset,destination_table, + if_dataset_not_exists,if_table_exists) diff --git a/bin/flake8 b/bin/flake8 new file mode 100755 index 0000000000..6a8c39a9be --- /dev/null +++ b/bin/flake8 @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from flake8.main.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/flake8p b/bin/flake8p new file mode 100755 index 0000000000..bd2a3b6061 --- /dev/null +++ b/bin/flake8p @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from flake8p import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/normalizer b/bin/normalizer new file mode 100755 index 0000000000..3398ea5a26 --- /dev/null +++ b/bin/normalizer @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from charset_normalizer.cli import cli_detect +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli_detect()) diff --git a/bin/pip b/bin/pip new file mode 100755 index 0000000000..0a809cafaa --- /dev/null +++ b/bin/pip @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/pip3 b/bin/pip3 new file mode 100755 index 0000000000..0a809cafaa --- /dev/null +++ b/bin/pip3 @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/pip3.9 b/bin/pip3.9 new file mode 100755 index 0000000000..0a809cafaa --- /dev/null +++ b/bin/pip3.9 @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/py.test b/bin/py.test new file mode 100755 index 0000000000..80ad406e9f --- /dev/null +++ b/bin/py.test @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pytest import console_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_main()) diff --git a/bin/pycodestyle b/bin/pycodestyle new file mode 100755 index 0000000000..9d9418a70e --- /dev/null +++ b/bin/pycodestyle @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pycodestyle import _main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(_main()) diff --git a/bin/pyflakes b/bin/pyflakes new file mode 100755 index 0000000000..2bde4c56a3 --- /dev/null +++ b/bin/pyflakes @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pyflakes.api import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bin/pytest b/bin/pytest new file mode 100755 index 0000000000..80ad406e9f --- /dev/null +++ b/bin/pytest @@ -0,0 +1,8 @@ +#!/Users/charleskramer/Documents/parsons/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pytest import console_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(console_main()) diff --git a/bin/python b/bin/python new file mode 120000 index 0000000000..4d6a61c2dd --- /dev/null +++ b/bin/python @@ -0,0 +1 @@ +/Users/charleskramer/opt/anaconda3/bin/python \ No newline at end of file diff --git a/bin/python3 b/bin/python3 new file mode 120000 index 0000000000..d8654aa0e2 --- /dev/null +++ b/bin/python3 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/bin/python3.9 b/bin/python3.9 new file mode 120000 index 0000000000..d8654aa0e2 --- /dev/null +++ b/bin/python3.9 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/pyvenv.cfg b/pyvenv.cfg new file mode 100644 index 0000000000..0f2fbb6fdf --- /dev/null +++ b/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Users/charleskramer/opt/anaconda3/bin +include-system-site-packages = false +version = 3.9.13 From 78c34a6224de55c31e1da36813a3b2a8cacd0a12 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Mon, 9 Dec 2024 14:50:20 -0500 Subject: [PATCH 10/14] fixed more ruff errors --- parsons/google/google_bigquery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 1ed59cc9ac..23168f9b44 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -1527,7 +1527,7 @@ def copy_between_projects( job_config = bigquery.CopyJobConfig() job_config.write_disposition = "WRITE_TRUNCATE" job = self.client.copy_table( - source_table_id, destination_table_id, location='US', job_config=job_config + source_table_id, destination_table_id, location="US", job_config=job_config ) result = job.result() else: @@ -1540,7 +1540,7 @@ def copy_between_projects( except NotFound: # destination table doesn't exist, so we can create one job = self.client.copy_table( - source_table_id, destination_table_id, location='US', job_config=job_config + source_table_id, destination_table_id, location="US", job_config=job_config ) result = job.result() logger.info(result) From 54694a531300d0e112164948b027bd0d1f453285 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Mon, 9 Dec 2024 15:07:59 -0500 Subject: [PATCH 11/14] removing accidentally included file --- bin/delme.py | 71 ---------------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 bin/delme.py diff --git a/bin/delme.py b/bin/delme.py deleted file mode 100644 index ceca41fab5..0000000000 --- a/bin/delme.py +++ /dev/null @@ -1,71 +0,0 @@ -import sys -#sys.path.append("~/Documents/parsons") -sys.path.append('/Users/charleskramer/Documents/parsons') -import parsons -from google.auth import default -app_creds,_ = default() - -# app_creds = { -# "type": "service_account", -# "project_id": "sql-practice-425619", -# "private_key_id": "31d4ba8aa75b0fa6a892072e4cb73d907940f24e", -# "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMbfuUhBQX92vv\nYdXaw6gNNLQSY+Mu+Dg2fGN0tqZrJn3HaHeBS1GWoF3rJE9M9/vmivuNSuaPgBts\nqY9x/yQ/6zYtQ62vPO0LMtLWB+NdP2Ha87eFbPEMJA2RuTubeEwfEfnv82ZBVXrI\n9+1DwbPqEqg8CrVEIBC9JJq4qyYEXj6c3ol8DcgB9Vwz8d/XrIQ7Xam2wBF/6Eba\nB8qiyzF0N40YlkjQLZ3MlQvZFBxgXAGz9qjFjTNvhrvBqz+l2+IXPzLAecvWgEck\nekbp/QsfPrCkCd614pXFdklPUzoFaznlu2EpjeQMPH3Jc9vFIzI6EjppP1wKThNk\nQPsVaedDAgMBAAECggEALd++VVkAM1Kb33d6aGGAjBoYEIpmuCsOXjeyj8XO+XuF\nPSK1sodm0yDAgpw+yVxapCTrBw1YSLpsLQmtvVyOU2OiYCzwREMtRFaO1mWlwU6Q\nVHRdae0E+H78podFF4G6vzwm9JAPkbivWXq9PPeyOmQeaX7Tp2p0pyUYLTmGyfK/\nbXT7GiIT74U6mK2mhjUws36gp/bZj2ciwOPLdKZz6qFT7c2ncp3xHZGtZOGQi0WZ\neYu0dxu8MUSNWA6yf0tdhGLGY6s5D9lgUd6S7fREd5146gc95fRmWwNXkl1ywLTp\nLHx6IvSe0Xuo6gg0kKCJmeRY26gTWl0QtsdWkqjdWQKBgQDrtOu6vBgvhm27b4VA\nFkVxHPlNNFgnegt2EhCmK2WCqCk7GbQrkQRlHr+UT5SL+gDbcTXuc34ui+/xm0XG\nUqjs2+1ntX2nkM1h7hdhPwSiiizlX2ggx7Mvz2gWBkOaXX3sBYy3ixfkzIY78Y8Q\nwIfnG+3oBsOKwP/2ET+K5UAmewKBgQDeB7NveBuTvMK7evhUzWAGvSquBHAn9BzI\n18vlkywODFhuQ0QqPses/ErB/KpercaMSnqBtyFnlpMzqYiH/aLSAPTghkLhfO7i\n9m8RfXtSPftj7glgm0uVfkOPkzupeJFRTEvTiW/qyyUBCGEszCujpMVGFh5/OsNF\n16Xp1IYL2QKBgC1YYQFLrlt2QSYODlSF9FYOfOedalgt4oaUcx+EucKpF0WHbGH1\nRCMwBytBJBLJxeBxpy54iY3q3f5dIM9Gl7j5lnKdN89EzD5Kz7Slqv6aFokKEb48\nrPxFqoCSM+8+jTsa1jd5St95eVmO3zcZ0wtjFCHPK09GUffE2bSe5fiBAoGBAIXg\nxNx44sABeLYNXwHbWISXuc12FZ6xvk2IeYSzakQKQ6Qw4UBL3KC8++LelGhEhkz/\nd8ERiI3qqFXlatL9KBlSxFVB/7/xUiX+jFhSfnLHtva5iGP9H+VDXj+LetUkQxNv\nWuVxoM9FsMlfoSCe981TxDQPkFINP8O0VXGaWT15AoGAfGtNOQ25T41F2DLo6JSe\noayqyJe2b99jL92+BHD987U9w//j/FzwnYBUcXMKuYma75LYkpampAxtuMuamlNn\nq6FLJsfTJ7a9pP3ifN4tcdrK2v2FT4yK1d03kTG6HNNaqZBat5oe4y+8DeupWovW\nU17W2i4QavuYn0FNFMXOusc=\n-----END PRIVATE KEY-----\n", -# "client_email": "parsons-bq@sql-practice-425619.iam.gserviceaccount.com", -# "client_id": "104516682065371288685", -# "auth_uri": "https://accounts.google.com/o/oauth2/auth", -# "token_uri": "https://oauth2.googleapis.com/token", -# "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", -# "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/parsons-bq%40sql-practice-425619.iam.gserviceaccount.com", -# "universe_domain": "googleapis.com" -# } -# - - -gbq = parsons.GoogleBigQuery(app_creds=app_creds) - - - -# input parameters -source_project = 'sql-practice-425619' -source_dataset = 'bqtest' -source_table = 'va_2022_general' -# source_table_id = "sql-practice-425619.bqtest.va_2022_general" -# destination_table_id = "bq-for.bq2test.va_2022_general" -destination_project = 'bq-for' -destination_dataset = 'bqnew' -destination_table = 'va_2022_general_copy' -if_dataset_not_exists = 'create' # create -if_table_exists = 'overwrite' # overwrite - -#Access Denied: Dataset bq-for:bqnew: Permission bigquery.datasets.get denied on dataset bq-for:bqnew (or it may not exist). - -destination_project = 'sql-practice-425619' -destination_dataset = 'bqtest' -destination_table = 'va_2022_general' -# source_table_id = "sql-practice-425619.bqtest.va_2022_general" -# destination_table_id = "bq-for.bq2test.va_2022_general" -source_project = 'bq-for' -source_dataset = 'bqnew' -source_table = 'va_2022_general_copy' -if_dataset_not_exists = 'create' # create -if_table_exists = 'overwrite' # overwrite - -#Access Denied: Dataset sql-practice-425619:bqtest: Permission bigquery.datasets.get denied on dataset sql-practice-425619:bqtest (or it may not exist). - -# removed the client call in the method -source_project = 'sql-practice-425619' -source_dataset = 'bqtest' -source_table = 'va_2022_general' -# source_table_id = "sql-practice-425619.bqtest.va_2022_general" -# destination_table_id = "bq-for.bq2test.va_2022_general" -destination_project = 'bq-for' -destination_dataset = 'bqnew' -destination_table = 'va_2022_general_copy' - -if_dataset_not_exists = 'create' # create -if_table_exists = 'overwrite' # overwrite - - -gbq.copy_between_projects(source_project,source_dataset,source_table, - destination_project,destination_dataset,destination_table, - if_dataset_not_exists,if_table_exists) From 892b8c481b874fec4d07b13b924dff9b8d7e69fe Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sun, 15 Dec 2024 08:12:51 -0500 Subject: [PATCH 12/14] added unit tests for missing/existing files --- .gitignore | 3 + test/test_google/test_bigquery_copy.py | 121 +++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 test/test_google/test_bigquery_copy.py diff --git a/.gitignore b/.gitignore index 8d6520e18e..3f994ee23f 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ bill_com_credentials.* docs/html docs/dirhtml *.sw* + +# bin files +bin/ diff --git a/test/test_google/test_bigquery_copy.py b/test/test_google/test_bigquery_copy.py new file mode 100644 index 0000000000..92ee470271 --- /dev/null +++ b/test/test_google/test_bigquery_copy.py @@ -0,0 +1,121 @@ +# testing bigquery copy method of google class + +import unittest +from unittest import TestCase +from unittest.mock import Mock +import logging +from testfixtures import log_capture +from parsons import GoogleBigQuery + + +class TestGoogleBigQuery(TestCase): + + def setUp(self): + # mock the GoogleBigQuery class + self.bq = Mock(spec=GoogleBigQuery) + + # define inputs to copy method + self.source_project = ("project1",) + self.source_dataset = ("dataset1",) + self.source_table = ("table1",) + self.destination_project = ("project2",) + self.destination_dataset = ("dataset2",) + self.destination_table = ("table2",) + self.if_dataset_not_exists = ("fail",) + self.if_table_exists = "fail" + + def tearDown(self): + pass + + def test_copy_called_once_with(self): + self.bq.copy_between_projects( + source_project=self.source_project, + source_dataset=self.destination_dataset, + source_table=self.source_table, + destination_project=self.destination_project, + destination_dataset=self.destination_dataset, + destination_table=self.destination_table, + if_dataset_not_exists=self.if_dataset_not_exists, + if_table_exists=self.if_table_exists, + ) + self.bq.copy_between_projects.assert_called_once_with( + source_project=self.source_project, + source_dataset=self.destination_dataset, + source_table=self.source_table, + destination_project=self.destination_project, + destination_dataset=self.destination_dataset, + destination_table=self.destination_table, + if_dataset_not_exists=self.if_dataset_not_exists, + if_table_exists=self.if_table_exists, + ) + + @log_capture() + def test_logger_fail_on_dataset_does_not_exist(self, capture): + + # create and set up logger + logger = logging.getLogger() + logger.error( + "Dataset {0} does not exist and if_dataset_not_exists set to {1}".format( + self.destination_dataset, self.if_dataset_not_exists + ) + ) + + # call the method to generate log message + self.bq.copy_between_projects( + source_project=self.source_project, + source_dataset=self.destination_dataset, + source_table=self.source_table, + destination_project=self.destination_project, + destination_dataset=self.destination_dataset, + destination_table=self.destination_table, + if_dataset_not_exists=self.if_dataset_not_exists, + if_table_exists=self.if_table_exists, + ) + + # check that the log message was generated correctly + capture.check( + ( + "root", + "ERROR", + "Dataset {0} does not exist and if_dataset_not_exists set to {1}".format( + self.destination_dataset, self.if_dataset_not_exists + ), + ) + ) + + @log_capture() + def test_logger_fail_on_table_exists(self, capture): + + # create and set up logger + logger = logging.getLogger() + + ## now test with table copy error + logger.error( + "BigQuery copy failed, Table {0} exists and if_table_exists set to {1}".format( + self.destination_table, self.if_table_exists + ) + ) + + # call the method to generate log message + self.bq.copy_between_projects( + source_project=self.source_project, + source_dataset=self.destination_dataset, + source_table=self.source_table, + destination_project=self.destination_project, + destination_dataset=self.destination_dataset, + destination_table=self.destination_table, + if_dataset_not_exists=self.if_dataset_not_exists, + if_table_exists=self.if_table_exists, + ) + + # check that the log message was generated correctly + capture.check( + ( + "root", + "ERROR", + "BigQuery copy failed, Table {0} exists and if_table_exists set to {1}".format( + self.destination_table, self.if_table_exists + ), + ) + ) + From b746ca2411d16b1979e59b0c8cf95d3c41e12c7e Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sun, 15 Dec 2024 08:21:08 -0500 Subject: [PATCH 13/14] fixed ruff formatting --- test/test_google/test_bigquery_copy.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_google/test_bigquery_copy.py b/test/test_google/test_bigquery_copy.py index 92ee470271..73e1abd94c 100644 --- a/test/test_google/test_bigquery_copy.py +++ b/test/test_google/test_bigquery_copy.py @@ -1,6 +1,5 @@ # testing bigquery copy method of google class -import unittest from unittest import TestCase from unittest.mock import Mock import logging @@ -9,7 +8,6 @@ class TestGoogleBigQuery(TestCase): - def setUp(self): # mock the GoogleBigQuery class self.bq = Mock(spec=GoogleBigQuery) @@ -51,7 +49,6 @@ def test_copy_called_once_with(self): @log_capture() def test_logger_fail_on_dataset_does_not_exist(self, capture): - # create and set up logger logger = logging.getLogger() logger.error( @@ -85,7 +82,6 @@ def test_logger_fail_on_dataset_does_not_exist(self, capture): @log_capture() def test_logger_fail_on_table_exists(self, capture): - # create and set up logger logger = logging.getLogger() From b4451092542af154f533a296661605d09f2c1980 Mon Sep 17 00:00:00 2001 From: Charlie Kramer Date: Sun, 15 Dec 2024 08:24:36 -0500 Subject: [PATCH 14/14] fixed ruff formatting --- test/test_google/test_bigquery_copy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_google/test_bigquery_copy.py b/test/test_google/test_bigquery_copy.py index 73e1abd94c..db0d48f68e 100644 --- a/test/test_google/test_bigquery_copy.py +++ b/test/test_google/test_bigquery_copy.py @@ -114,4 +114,3 @@ def test_logger_fail_on_table_exists(self, capture): ), ) ) -