feat: implement ndjson artifact

Signed-off-by: Emanuel Calvo <[email protected]>

.github/workflows/main.yml

@@ -24,5 +24,5 @@ jobs:
           pip install -r requirements.txt
   
       - name: execute artifact_builder 
-        run: python artifact_builder -D sql -O index.json
+        run: python artifact_builder -D sql -O index.json -E postgres
 

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
           pip install -r requirements.txt
   
       - name: execute artifact_builder
-        run: python artifact_builder -D sql -O index.json
+        run: python artifact_builder -D sql -O index.json -E postgres
 
       - name: Extract tag name
         id: tag
@@ -66,4 +66,14 @@ jobs:
           upload_url: ${{ steps.latest_release_info.outputs.upload_url }}
           asset_path: index.json
           asset_name: index.json
+          asset_content_type: application/json
+
+      - name: upload ndjson artifact
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.latest_release_info.outputs.upload_url }}
+          asset_path: ndindex.json
+          asset_name: ndindex.json
           asset_content_type: application/json

.gitignore

@@ -19,4 +19,5 @@ pip-selfcheck.json
 # End of https://www.gitignore.io/api/venv
 
 .artifacts/
-index.json
+index.json
+ndindex.json

.gitlab-ci.yml

@@ -3,16 +3,17 @@ image: python:latest
 
 default:
   script:
-    - python3 artifact_builder -D sql -O index.json
+    - python3 artifact_builder -D sql -O index.json -E postgres
   only:
     - master
 
 release:
   script:
-    - python3 artifact_builder -D sql -O index.json
+    - python3 artifact_builder -D sql -O index.json -E postgres
   artifacts:
     paths:
       - index.json
+      - ndindex.json
   # rules:
   #   - if: $CI_COMMIT_BRANCH == "master"
   only:

artifact_builder/__main__.py

@@ -1,6 +1,7 @@
 import indexer
 from filemap import fileMap
 import json 
+import ndjson
 import argparse
 
 
@@ -9,21 +10,30 @@
 _engine: str
 
 def main():
-    _engine = 'postgres'
 
     parser = argparse.ArgumentParser()
     parser.add_argument("-D", "--directory", help="Directory to be indexed", type=str, required=False, default=sqlDirectory)
-    parser.add_argument("-O", "--output", help="Output artifact", required=False, type=str, default="./index.json")
+    parser.add_argument("-O", "--output", help="Output json artifact", required=False, type=str, default="./index.json")
+    parser.add_argument("-o", "--output-ndjson", help="Output ndjson artifact", required=False, type=str, default="./ndindex.json")
     parser.add_argument("-v", "--verbose", help="Kind of debug", default=False, action=argparse.BooleanOptionalAction)
+    parser.add_argument("-E", "--engine", help="Engine name", default="postgres", type=str, required=True)
     args = parser.parse_args()
 
-    fileMap = indexer.indexDir(args.directory, _engine)
+    fileMap = indexer.indexDir(args.directory, args.engine)
 
     if args.verbose:
-        print(json.dumps(fileMap, indent=2))
+        print(json.dumps(fileMap, indent=1))
 
-    with open(args.output, 'w+', encoding="utf-8") as f:
-        json.dump(fileMap, f)
+    with open(args.output, 'w+', encoding='utf-8-sig') as f:
+        json.dump(fileMap, f, indent=1)
+
+    with open(args.output, 'rb') as f:
+        data = json.load(f)
+
+    with open(args.output_ndjson, 'w+', encoding='utf-8-sig') as f:
+        writer = ndjson.writer(f)
+        for key in data:
+            writer.writerow(data[key])
 
 if __name__ == "__main__":
     main()

artifact_builder/indexer.py

@@ -16,15 +16,18 @@ def indexDir(sqlDirectory: str, _engine: str) -> fileMap:
             key = sub('.sql|.md', '', filename)
             fpath = join(root.removeprefix('../'), filename)
 
-            if key not in _fileMap:
+            # For now, we ignore READMEs. But, we might furtherly include some documentation
+            # artifact.
+            if key not in _fileMap and filename.removesuffix(".md").lower() != 'readme':
                 _fileMap[key]={'engine': _engine}
+                _fileMap[key]={'title': sub('[_-]'," ", str(key)).capitalize()}
 
             if filename.endswith(".sql"):
                 with open(fpath, encoding="utf-8") as f:
                     _fileMap[key].update({'fpath': fpath, 
-                                        'category': root.removeprefix(sqlDirectory),
+                                        'category': root.removeprefix(sqlDirectory + '/'),
                                         'query': f.read()})
-            elif filename.endswith(".md"):
+            elif filename.endswith(".md") and filename.removesuffix(".md").lower() != 'readme':
                 with open(fpath, encoding="utf-8") as f:
                     _fileMap[key].update({'docFPath': fpath,
                                           'doc': f.read()})

artifact_builder/requirements.txt

@@ -0,0 +1 @@
+ndjson==0.3.1