initial commit

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,18 @@
+## Description of the goal of the PR
+
+Description: 
+
+## Changes this PR introduces (fill it before implementation)
+
+- [ ] : Change 1
+- [ ] : Change 2
+
+## Checklist before requesting a review
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have typed my code
+- [ ] I have created / updated the docstrings
+- [ ] I have updated the README, if relevant
+- [ ] I have updated the requirements files if a new package is used
+- [ ] I have tested my code
+- [ ] The CI pipeline passes
+- [ ] I have performed a self-review of my code

.github/workflows/ci.yaml

@@ -0,0 +1,25 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  CI:
+    name: Launching CI
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-developer.txt
+      - name: Run Pre commit hook (formatting, linting & tests)
+        run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure

.github/workflows/deploy_docs.yaml

@@ -0,0 +1,28 @@
+name: Deploy MkDocs to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy-docs:
+    name: Deploy docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-developer.txt
+      - name: Deploying MkDocs documentation
+        run: |
+          mkdocs build
+          mkdocs gh-deploy --force
+        
+    

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+site/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VSCode project settings
+.vscode/
+
+# Secret files
+secrets/*
+!secrets/.gitkeep
+
+# Mac OS
+.DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+repos:
+  - repo: "https://github.com/pre-commit/pre-commit-hooks"
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-toml
+      - id: check-yaml
+      - id: check-json
+      - id: check-added-large-files
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: Formatting (ruff)
+        entry: ruff format
+        types: [python]
+        language: system
+      - id: ruff-fix
+        name: Linting & sorting (ruff)
+        entry: ruff --fix --fixable I001  # allow only to fix unsorted imports
+        types: [python]
+        language: system
+      - id: nbstripout
+        name: Strip Jupyter notebook output (nbstripout)
+        entry: nbstripout
+        types: [file]
+        files: (.ipynb)$
+        language: system
+      - id: python-bandit-vulnerability-check
+        name: Security check (bandit)
+        entry: bandit
+        types: [python]
+        args: ["--recursive", "lib/"]
+        language: system
+      - id: pytest-check
+        name: Tests (pytest)
+        stages: [push]
+        entry: pytest tests/
+        types: [python]
+        language: system
+        pass_filenames: false
+        always_run: true
+exclude: ^(.svn|CVS|.bzr|.hg|.git|__pycache__|.tox|.ipynb_checkpoints|assets|tests/assets/|venv/|.venv/)

Makefile

@@ -0,0 +1,34 @@
+USE_CONDA ?= 1
+INSTALL_SCRIPT = install_with_conda.sh
+ifeq (false,$(USE_CONDA))
+	INSTALL_SCRIPT = install_with_venv.sh
+endif
+
+.DEFAULT_GOAL = help
+
+# help: help					- Display this makefile's help information
+.PHONY: help
+help:
+	@grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://'
+
+# help: install					- Create a virtual environment and install dependencies
+.PHONY: install
+install:
+	@bash bin/$(INSTALL_SCRIPT)
+
+# help: install_precommit			- Install pre-commit hooks
+.PHONY: install_precommit
+install_precommit:
+	@pre-commit install -t pre-commit
+	@pre-commit install -t pre-push
+
+# help: serve_docs_locally			- Serve docs locally on port 8001
+.PHONY: serve_docs_locally
+serve_docs_locally:
+	@mkdocs serve --livereload -a localhost:8001
+
+# help: deploy_docs				- Deploy documentation to GitHub Pages
+.PHONY: deploy_docs
+deploy_docs:
+	@mkdocs build
+	@mkdocs gh-deploy

README.md

@@ -0,0 +1,76 @@
+<div align="center">
+
+# skaff-rag-accelerator
+
+[![CI status](https://github.com/artefactory/skaff-rag-accelerator/actions/workflows/ci.yaml/badge.svg)](https://github.com/artefactory/skaff-rag-accelerator/actions/workflows/ci.yaml?query=branch%3Amain)
+[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg)]()
+
+[![Linting , formatting, imports sorting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
+[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-informational?logo=pre-commit&logoColor=white)](https://github.com/artefactory/skaff-rag-accelerator/blob/main/.pre-commit-config.yaml)
+</div>
+
+TODO: if not done already, check out the [Skaff documentation](https://artefact.roadie.so/catalog/default/component/repo-builder-ds/docs/) for more information about the generated repository.
+
+Deploy RAGs quickly
+
+## Table of Contents
+
+- [skaff-rag-accelerator](#skaff-rag-accelerator)
+  - [Table of Contents](#table-of-contents)
+  - [Installation](#installation)
+  - [Usage](#usage)
+  - [Documentation](#documentation)
+  - [Repository Structure](#repository-structure)
+
+## Installation
+
+To install the required packages in a virtual environment, run the following command:
+
+```bash
+make install
+```
+
+TODO: Choose between conda and venv if necessary or let the Makefile as is and copy/paste the [MORE INFO installation section](MORE_INFO.md#eased-installation) to explain how to choose between conda and venv.
+
+A complete list of available commands can be found using the following command:
+
+```bash
+make help
+```
+
+## Usage
+
+TODO: Add usage instructions here
+
+## Documentation
+
+TODO: Github pages is not enabled by default, you need to enable it in the repository settings: Settings > Pages > Source: "Deploy from a branch" / Branch: "gh-pages" / Folder: "/(root)"
+
+A detailed documentation of this project is available [here](https://artefactory.github.io/skaff-rag-accelerator/)
+
+To serve the documentation locally, run the following command:
+
+```bash
+mkdocs serve
+```
+
+To build it and deploy it to GitHub pages, run the following command:
+
+```bash
+make deploy_docs
+```
+
+## Repository Structure
+
+```
+.
+├── .github    <- GitHub Actions workflows and PR template
+├── bin        <- Bash files
+├── config     <- Configuration files
+├── docs       <- Documentation files (mkdocs)
+├── lib        <- Python modules
+├── notebooks  <- Jupyter notebooks
+├── secrets    <- Secret files (ignored by git)
+└── tests      <- Unit tests
+```

bin/install_with_conda.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+read -p "Want to install conda env named 'skaff-rag-accelerator'? (y/n)" answer
+if [ "$answer" = "y" ]; then
+  echo "Installing conda env..."
+  conda create -n skaff-rag-accelerator python=3.10 -y
+  source $(conda info --base)/etc/profile.d/conda.sh
+  conda activate skaff-rag-accelerator
+  echo "Installing requirements..."
+  pip install -r requirements-developer.txt
+  python3 -m ipykernel install --user --name=skaff-rag-accelerator
+  conda install -c conda-forge --name skaff-rag-accelerator notebook -y
+  echo "Installing pre-commit..."
+  make install_precommit
+  echo "Installation complete!";
+else
+  echo "Installation of conda env aborted!";
+fi