diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 89dba97..b8af22a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,5 +24,5 @@ jobs: pip install -r requirements.txt - name: execute artifact_builder - run: python artifact_builder -D sql -O index.json + run: python artifact_builder -D sql -O index.json -E postgres diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40d8ae5..d0dc258 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: pip install -r requirements.txt - name: execute artifact_builder - run: python artifact_builder -D sql -O index.json + run: python artifact_builder -D sql -O index.json -E postgres - name: Extract tag name id: tag @@ -66,4 +66,14 @@ jobs: upload_url: ${{ steps.latest_release_info.outputs.upload_url }} asset_path: index.json asset_name: index.json + asset_content_type: application/json + + - name: upload ndjson artifact + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ steps.latest_release_info.outputs.upload_url }} + asset_path: ndindex.json + asset_name: ndindex.json asset_content_type: application/json \ No newline at end of file diff --git a/.gitignore b/.gitignore index f006bfe..7c11264 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ pip-selfcheck.json # End of https://www.gitignore.io/api/venv .artifacts/ -index.json \ No newline at end of file +index.json +ndindex.json \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cfef9cf..ecd2e4b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,16 +3,17 @@ image: python:latest default: script: - - python3 artifact_builder -D sql -O index.json + - python3 artifact_builder -D sql -O index.json -E postgres only: - master release: script: - - python3 artifact_builder -D sql -O index.json + - python3 artifact_builder -D sql -O index.json -E postgres artifacts: paths: - index.json + - ndindex.json # rules: # - if: $CI_COMMIT_BRANCH == "master" only: diff --git a/artifact_builder/__main__.py b/artifact_builder/__main__.py index 9e5019d..27cceb3 100644 --- a/artifact_builder/__main__.py +++ b/artifact_builder/__main__.py @@ -1,6 +1,7 @@ import indexer from filemap import fileMap import json +import ndjson import argparse @@ -9,21 +10,30 @@ _engine: str def main(): - _engine = 'postgres' parser = argparse.ArgumentParser() parser.add_argument("-D", "--directory", help="Directory to be indexed", type=str, required=False, default=sqlDirectory) - parser.add_argument("-O", "--output", help="Output artifact", required=False, type=str, default="./index.json") + parser.add_argument("-O", "--output", help="Output json artifact", required=False, type=str, default="./index.json") + parser.add_argument("-o", "--output-ndjson", help="Output ndjson artifact", required=False, type=str, default="./ndindex.json") parser.add_argument("-v", "--verbose", help="Kind of debug", default=False, action=argparse.BooleanOptionalAction) + parser.add_argument("-E", "--engine", help="Engine name", default="postgres", type=str, required=True) args = parser.parse_args() - fileMap = indexer.indexDir(args.directory, _engine) + fileMap = indexer.indexDir(args.directory, args.engine) if args.verbose: - print(json.dumps(fileMap, indent=2)) + print(json.dumps(fileMap, indent=1)) - with open(args.output, 'w+', encoding="utf-8") as f: - json.dump(fileMap, f) + with open(args.output, 'w+', encoding='utf-8-sig') as f: + json.dump(fileMap, f, indent=1) + + with open(args.output, 'rb') as f: + data = json.load(f) + + with open(args.output_ndjson, 'w+', encoding='utf-8-sig') as f: + writer = ndjson.writer(f) + for key in data: + writer.writerow(data[key]) if __name__ == "__main__": main() diff --git a/artifact_builder/indexer.py b/artifact_builder/indexer.py index c9afd01..bb2ab88 100644 --- a/artifact_builder/indexer.py +++ b/artifact_builder/indexer.py @@ -16,15 +16,18 @@ def indexDir(sqlDirectory: str, _engine: str) -> fileMap: key = sub('.sql|.md', '', filename) fpath = join(root.removeprefix('../'), filename) - if key not in _fileMap: + # For now, we ignore READMEs. But, we might furtherly include some documentation + # artifact. + if key not in _fileMap and filename.removesuffix(".md").lower() != 'readme': _fileMap[key]={'engine': _engine} + _fileMap[key]={'title': sub('[_-]'," ", str(key)).capitalize()} if filename.endswith(".sql"): with open(fpath, encoding="utf-8") as f: _fileMap[key].update({'fpath': fpath, - 'category': root.removeprefix(sqlDirectory), + 'category': root.removeprefix(sqlDirectory + '/'), 'query': f.read()}) - elif filename.endswith(".md"): + elif filename.endswith(".md") and filename.removesuffix(".md").lower() != 'readme': with open(fpath, encoding="utf-8") as f: _fileMap[key].update({'docFPath': fpath, 'doc': f.read()}) diff --git a/artifact_builder/requirements.txt b/artifact_builder/requirements.txt index e69de29..1040777 100644 --- a/artifact_builder/requirements.txt +++ b/artifact_builder/requirements.txt @@ -0,0 +1 @@ +ndjson==0.3.1 \ No newline at end of file