[WIP] Captura de recursos do subsídio SPPO

.flake8

@@ -1,3 +1,5 @@
 [flake8]
 ignore = E203,F401,F403,W503
+exclude =
+    *cookiecutter*
 max-line-length = 100

.github/workflows/cd_staging.yaml

@@ -25,10 +25,9 @@ env:
 
 jobs:
   build-container:
-    if: startsWith(github.head_ref, 'staging/') || contains( github.event.pull_request.labels.*.name, 'staging')
+    if: startsWith(github.head_ref, 'staging/')
     name: Register flows (staging)
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/ci.yaml

@@ -48,3 +48,6 @@ jobs:
         with:
           message: "${{ steps.code-tree-analysis.outputs.pr-message }}"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run tests
+        run: |
+          pytest

.github/workflows/lint_python.yaml

@@ -12,13 +12,24 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.9.x"
+          python-version: "3.8.x"
       - name: Set up Poetry and upgrade pip
         run: |
           pip install -U pip poetry
       - name: Install flows
         run: |
-          poetry install
-      - name: Lint with flake8
-        run: |
-          poetry run lint
+          pip install --prefer-binary .
+      - name: Lint flows
+        uses: gabriel-milan/action-pylint@v1
+        with:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          path: "pipelines/"
+          fail: true
+          pr-message: true
+      - name: Lint tests
+        uses: gabriel-milan/action-pylint@v1
+        with:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          path: "tests/"
+          fail: true
+          pr-message: true

.github/workflows/scripts/code_tree_analysis.py

@@ -444,6 +444,7 @@ def identify_code_owners(files: List[str]) -> List[str]:
 
 
 if __name__ == "__main__":
+
     # Assert arguments.
     if len(sys.argv) not in [2, 3]:
         print(f"Usage: python {sys.argv[0]} <changed_files> [--write-to-file]")
@@ -464,7 +465,6 @@ def identify_code_owners(files: List[str]) -> List[str]:
         if file_.endswith(".py")
         and file_.startswith("pipelines")
         and "cookiecutter" not in file_
-        and Path(file_).exists()
     ]
     print("We're interested in these files:")
     for file_ in changed_files:

.github/workflows/scripts/register_flows.py

@@ -104,11 +104,7 @@ def build_and_register(  # pylint: disable=too-many-branches
                 while attempts < max_retries:
                     attempts += 1
                     try:
-                        (
-                            flow_id,
-                            flow_version,
-                            is_new,
-                        ) = register_serialized_flow(
+                        (flow_id, flow_version, is_new,) = register_serialized_flow(
                             client=client,
                             serialized_flow=serialized_flow,
                             project_id=project_id,
@@ -484,8 +480,8 @@ def get_affected_flows(fpath: str = None):
     for flow in declared_flows:
         try:
             flows.append(eval(flow))
-        except Exception as exc:
-            logger.exception(f"Could not evaluate {flow}: {exc}")
+        except Exception:
+            logger.warning(f"Could not evaluate {flow}")
     return flows
 
 

.gitignore

@@ -5,13 +5,7 @@ replit.nix
 test_local.py
 pylint.txt
 test.py
-test/*
-test/*.ipynb
-test/*.csv
 setup.py
-.vscode/*
-*.hdf
-*.DS_Store
 
 
 # Byte-compiled / optimized / DLL files

.pre-commit-config.yaml

@@ -1,19 +1,20 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
     hooks:
-      - id: check-added-large-files # prevents adding large files
-      - id: detect-private-key # detects private keys
-      - id: fix-byte-order-marker # fixes BOM
-      - id: fix-encoding-pragma # fixes encoding pragma
-      - id: no-commit-to-branch # prevents committing to protected branches
-      - id: trailing-whitespace # prevents trailing whitespace
-  - repo: https://github.com/psf/black
-    rev: 23.7.0
+    - id: check-added-large-files # prevents adding large files
+    - id: detect-private-key # detects private keys
+    - id: fix-byte-order-marker # fixes BOM
+    - id: fix-encoding-pragma # fixes encoding pragma
+    - id: no-commit-to-branch # prevents committing to protected branches
+    - id: trailing-whitespace # prevents trailing whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.10.0
     hooks:
-      - id: black
+    -   id: black
         language_version: python3.9
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+        exclude: 'pipelines\/\{\{cookiecutter\.project_name\}\}.*'
+-   repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
     hooks:
-      - id: flake8
+    - id: flake8

.pylintrc

@@ -20,7 +20,7 @@ fail-on=
 fail-under=10.0
 
 # Files or directories to be skipped. They should be base names, not paths.
-ignore=
+ignore={{cookiecutter.project_name}}
 
 # Add files or directories matching the regex patterns to the ignore-list. The
 # regex matches against paths.

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use o IntelliSense para saber mais sobre os atributos possíveis.
+    // Focalizar para exibir as descrições dos atributos existentes.
+    // Para obter mais informações, acesse: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Arquivo Atual",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "python.formatting.provider": "none"
+}

LICENSE

@@ -1,8 +1,3 @@
-**Attention:** This repository contains part of the code under the GPL-3.0 license and part under an EULA license.
-All code under the EULA license will have a header indicating that it is proprietary.
-See the respective topics under LICENSE for the terms and conditions of each license.
-
-
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
@@ -677,89 +672,3 @@ may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
 <https://www.gnu.org/licenses/why-not-lgpl.html>.
-
-
-###########################################################################
-
-
-                    END USER LICENSE AGREEMENT (EULA)
-
-    IMPORTANT - PLEASE READ CAREFULLY BEFORE DOWNLOADING, INSTALLING OR USING
-ANY PART OF THIS CODE:
-
-    This End User License Agreement ("EULA") is entered into between the
-Municipality of Rio de Janeiro ("Licensor") and any individual or entity
-that downloads, installs or uses Licensor code ("Licensee") made available in
-this directory and in its subdirectories.
-
-    BY DOWNLOADING, INSTALLING, COPYING OR USING ANY PART OF THIS CODE, YOU
-AGREE TO BE BOUND BY THE TERMS OF THIS EULA. IF YOU DO NOT AGREE TO THE TERMS
-OF THIS EULA, DO NOT DOWNLOAD, INSTALL OR USE THE PRODUCT.
-
-
-    1. License Grant.
-
-    Licensor grants Licensee a non-exclusive, non-transferable license to
-use this code for personal or commercial purposes, subject to the terms and
-conditions of this Agreement. This license does not grant you the right to
-distribute, modify, rent, lease or sublicense the product.
-
-
-    2. Restrictions
-
-    Licensee may not reproduce, distribute, modify, reverse engineer, create
-derivative works or otherwise use or exploit any part of this code, particularly
-that which is specifically identified by Licensor as proprietary or confidential
-information ("Restricted Code"). Licensee acknowledges that any breach of this
-restriction may cause Licensor irreparable harm and that Licensor may seek
-injunctive relief in addition to any other available legal or equitable remedies.
-
-
-    3. Property right
-
-    The Product is the property of Licensor and is protected by copyright laws and
-international treaty provisions. Licensor retains all right, title and interest in
-and to the Code, including all intellectual property rights therein. You agree to
-protect the Product from unauthorized use, reproduction or distribution. This
-Agreement does not transfer any ownership rights to Licensee.
-
-
-    4. Limitation of Liability
-
-    IN NO EVENT SHALL LICENSOR BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL,
-CONSEQUENTIAL OR PUNITIVE DAMAGES, OR ANY DAMAGES RESULTING FROM LOSS OF USE, DATA
-OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHERWISE, ARISING OUT
-OF OR IN CONNECTION WITH WITH THE USE OR PERFORMANCE OF THE PRODUCT.
-
-
-    5. Disclaimer of Warranties
-
-    THE PRODUCT IS PROVIDED WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED,
-INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. LICENSOR DOES NOT WARRANT THAT THE
-PRODUCT WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION OF THE PRODUCT WILL BE
-UNINTERRUPTED OR ERROR-FREE.
-
-
-    6. Applicable law
-
-    This EULA will be governed by and constructed in accordance with the laws of
-the state of Rio de Janeiro, without giving effect to any choice of law or conflict
-of law provisions.
-
-
-    7. Total agreement
-
-    This Agreement constitutes the entire agreement between the parties with
-respect to the subject matter herein and supersedes all prior or contemporaneous
-communications and proposals, oral or written, between the parties.
-
-
-    By downloading, installing or using the code, Licensee acknowledges that
-Licensee has read and understood this Agreement and agrees to be bound by its
-terms and conditions.
-
-    If you have any questions about this EULA or the Product, please contact us
-at [email protected]
-
-                        END OF TERMS AND CONDITIONS

README.md

@@ -5,6 +5,7 @@ da Prefeitura do Rio de Janeiro. O repositório é gerido pelo Escritório
 Municipal de Dados (EMD) e alimentado de forma colaborativa com as equipes de
 dados e tecnologia das Secretarias.
 
+
 > 💜 Todo o código é desenvolvido em Python utilizando o software livre [Prefect](https://prefect.io/).
 
 ## Configuração de ambiente para desenvolvimento
@@ -16,12 +17,6 @@ dados e tecnologia das Secretarias.
 - `pip`
 - (Opcional, mas recomendado) Um ambiente virtual para desenvolvimento (`miniconda`, `virtualenv` ou similares)
 
-### Licenças
-
-Este repositório contém parte do código sob a licença GPL-3.0 e parte sob uma licença EULA.
-Todo código sob a licença EULA terá um cabeçalho indicando que é proprietário.
-Consulte os respectivos tópicos em LICENÇA para os termos e condições de cada licença.
-
 ### Procedimentos
 
 - Clonar esse repositório
@@ -93,7 +88,7 @@ constants.py                 # valores constantes para todos os órgãos
 
 ```
 
-### Adicionando órgãos e projetos (descontinuado)
+### Adicionando órgãos e projetos
 
 O script `manage.py` é responsável por criar e listar projetos desse repositório. Para usá-lo, no entanto, você deve instalar as dependências em `requirements-cli.txt`:
 

backfill.py

@@ -64,6 +64,7 @@ def create_timestamp_parameters(
 
 
 if __name__ == "__main__":
+
     timestamp_parameters = create_timestamp_parameters(
         start=BACKFILL_START,
         end=BACKFILL_END,

code_owners.yaml

@@ -19,9 +19,6 @@ pipelines:
         - hellcassius
         - fernandascovino
         - eng-rodrigocunha
-        - borismarinho
-        - pixuimpou
-        - lingsv
     rj_escritorio:
       owners:
         - gabriel-milan