Merge pull request #134 from mongulu-cm/feat/delete-useless-tools

refactor: #113 remove useless code

.envrc.gitpod

@@ -35,20 +35,3 @@ then
     mv csvdiff_1.4.0_linux_64-bit.deb csvdiff.deb &&\
     sudo dpkg -i csvdiff.deb
 fi
-
-if ! command -v aws &> /dev/null
-then
-    pushd /tmp || exit
-    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    unzip awscliv2.zip && sudo ./aws/install && rm awscliv2.zip && rm -rf aws/ &&\
-    mkdir "$WORKPLACE"/.aws  && touch "$WORKPLACE"/.aws/credentials && touch "$WORKPLACE"/.aws/config &&\
-    export AWS_SHARED_CREDENTIALS_FILE=$WORKPLACE/.aws/credentials
-    export AWS_CONFIG_FILE=$WORKPLACE/.aws/config
-    export AWS_SDK_LOAD_CONFIG=1
-    aws configure
-    popd || exit
-fi
-
-echo "export AWS_SHARED_CREDENTIALS_FILE=$WORKPLACE/.aws/credentials" >> ~/.bashrc
-echo "export AWS_CONFIG_FILE=$WORKPLACE/.aws/config" >> ~/.bashrc
-echo "export AWS_SDK_LOAD_CONFIG=1" >> ~/.bashrc

.gitignore

@@ -178,8 +178,7 @@ node_modules/**
 package-lock.json
 package.json
 
-## To avoid commit of aws creds in gitpod env
-.aws/
+
 ## To avoid commit of .envrc in gitpod because this cause gitpod to execute .envrc instead of gitpod.yml
 .envrc
 etl/experiments/.chainlit/**

.gitpod.yml

@@ -8,7 +8,6 @@ tasks:
       pushd etl/
       pre-commit install
       pre-commit install --hook-type commit-msg
-      aws s3 cp s3://mongulu-files/enrich_cache.sqlite enrich_cache.sqlite
       curl "https://media.interieur.gouv.fr/rna/rna_waldec_20220301.zip" -o rna_waldec_20220301.zip
       unzip rna_waldec_20220301.zip -d rna_waldec_20220301
       rm -f rna_waldec_20220301.zip
@@ -22,4 +21,3 @@ vscode:
     - vivaxy.vscode-conventional-commits
     - sourcegraph.cody-ai
     - donjayamanne.githistory
-    - amazonwebservices.aws-toolkit-vscode

README.md

@@ -22,9 +22,7 @@ If you are here, it means that you are interested in an in-house deployment of t
 
 ### Prerequisites
 
-* Have a minimum of competence on the AWS and Terraform cloud
 * Create a Sourcegraph account and get credentials to use [CodyAI](https://www.youtube.com/watch?v=_csyHcEcxDA)
-* Create a AWS Builder ID to use [Code whisperer](https://www.youtube.com/watch?v=rHNMfOK8pWI) *[Optional]*
 *  [Devspace](https://www.devspace.sh/) installed locally
 * Have admin access on a [Gogocarto](https://gogocarto.fr/projects)
 * Go through the [Gogocarto tutorials](https://peertube.openstreetmap.fr/c/gogo_tutos/videos)
@@ -38,7 +36,6 @@ If you are here, it means that you are interested in an in-house deployment of t
 Execute  `filter-cameroon.ipynb` et `enrich-database.ipynb` notebooks :
   ```
     pipenv shell
-    aws s3 cp s3://mongulu-files/enrich_cache.sqlite enrich_cache.sqlite
     secretsfoundry run --script 'python filter-cameroon.py'
   ```
 

etl/filter-cameroon.py

@@ -16,6 +16,9 @@
 from pandarallel import pandarallel
 from rich.console import Console
 
+# enrich_cache finally not used, it was store in aws but with clear params
+# geocode is used now => store in local => 1000 requ free per day
+
 # %%
 start = time.time()
 file_location = os.getcwd() + "/rna_waldec_20220301/"

etl/requirements-dev.txt

@@ -1,11 +1,7 @@
 nbdime==3.1.1
-python-lsp-server[all]==1.4.1
-jupyterlab-lsp== 3.10.1
 nbqa==1.3.1
-jupyterlab-tabular-data-editor==1.0.0
 csvs-to-sqlite==1.3
 ipython>=8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability
 numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
-jupyter-server>=2.11.2 # not directly required, pinned by Snyk to avoid a vulnerability
 tornado>=6.3.3 # not directly required, pinned by Snyk to avoid a vulnerability

etl/requirements.txt

@@ -2,8 +2,6 @@ colorama==0.4.4
 findspark==2.0.1
 geocoder==1.38.1
 geopy==2.2.0
-#jupyter==1.0.0
-#jupyter-utils==1.2.6
 pandarallel==1.6.1
 pipdeptree==2.2.1
 requests-cache==0.9.4
@@ -12,7 +10,6 @@ wheel==0.38.1
 abbreviations==0.2.5
 ftfy==6.1.1
 rich==12.4.4
-#jupyterlab==3.6.7
 cryptography>=3.2 # not directly required, pinned by Snyk to avoid a vulnerability
 ipython>=8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability