From b8709074ec3cbb9e0037ce00e478a4c77e257437 Mon Sep 17 00:00:00 2001 From: Scaffolder Date: Tue, 12 Dec 2023 13:07:00 +0000 Subject: [PATCH] initial commit --- .github/PULL_REQUEST_TEMPLATE.md | 18 ++++ .github/workflows/ci.yaml | 25 +++++ .github/workflows/deploy_docs.yaml | 28 ++++++ .gitignore | 138 +++++++++++++++++++++++++++ .pre-commit-config.yaml | 43 +++++++++ Makefile | 34 +++++++ README.md | 76 +++++++++++++++ bin/install_with_conda.sh | 18 ++++ bin/install_with_venv.sh | 20 ++++ config/config.py | 4 + config/config.toml | 2 + docs/code.md | 1 + docs/index.md | 3 + lib/.gitkeep | 0 mkdocs.yaml | 42 ++++++++ notebooks/private/.gitignore | 2 + notebooks/template.ipynb | 79 +++++++++++++++ pyproject.toml | 65 +++++++++++++ requirements-developer.txt | 10 ++ requirements.txt | 3 + secrets/.gitkeep | 0 tests/data/.gitkeep | 0 tests/integration_tests/.gitkeep | 0 tests/unit_tests/.gitkeep | 0 tests/unit_tests/test_placeholder.py | 6 ++ 25 files changed, 617 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/deploy_docs.yaml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 Makefile create mode 100644 README.md create mode 100644 bin/install_with_conda.sh create mode 100644 bin/install_with_venv.sh create mode 100644 config/config.py create mode 100644 config/config.toml create mode 100644 docs/code.md create mode 100644 docs/index.md create mode 100644 lib/.gitkeep create mode 100644 mkdocs.yaml create mode 100644 notebooks/private/.gitignore create mode 100644 notebooks/template.ipynb create mode 100644 pyproject.toml create mode 100644 requirements-developer.txt create mode 100644 requirements.txt create mode 100644 secrets/.gitkeep create mode 100644 tests/data/.gitkeep create mode 100644 tests/integration_tests/.gitkeep create mode 100644 tests/unit_tests/.gitkeep create mode 100644 tests/unit_tests/test_placeholder.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..390c9a4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +## Description of the goal of the PR + +Description: + +## Changes this PR introduces (fill it before implementation) + +- [ ] : Change 1 +- [ ] : Change 2 + +## Checklist before requesting a review +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have typed my code +- [ ] I have created / updated the docstrings +- [ ] I have updated the README, if relevant +- [ ] I have updated the requirements files if a new package is used +- [ ] I have tested my code +- [ ] The CI pipeline passes +- [ ] I have performed a self-review of my code \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..6052675 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,25 @@ +name: CI + +on: [push, pull_request] + +jobs: + CI: + name: Launching CI + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install requirements + run: | + python -m pip install --upgrade pip + pip install -r requirements-developer.txt + - name: Run Pre commit hook (formatting, linting & tests) + run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml new file mode 100644 index 0000000..d8b7124 --- /dev/null +++ b/.github/workflows/deploy_docs.yaml @@ -0,0 +1,28 @@ +name: Deploy MkDocs to GitHub Pages + +on: + push: + branches: + - main + +jobs: + deploy-docs: + name: Deploy docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install requirements + run: | + python -m pip install --upgrade pip + pip install -r requirements-developer.txt + - name: Deploying MkDocs documentation + run: | + mkdocs build + mkdocs gh-deploy --force + + \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0dbaa4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +site/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode project settings +.vscode/ + +# Secret files +secrets/* +!secrets/.gitkeep + +# Mac OS +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c285e93 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +repos: + - repo: "https://github.com/pre-commit/pre-commit-hooks" + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-toml + - id: check-yaml + - id: check-json + - id: check-added-large-files + - repo: local + hooks: + - id: ruff-format + name: Formatting (ruff) + entry: ruff format + types: [python] + language: system + - id: ruff-fix + name: Linting & sorting (ruff) + entry: ruff --fix --fixable I001 # allow only to fix unsorted imports + types: [python] + language: system + - id: nbstripout + name: Strip Jupyter notebook output (nbstripout) + entry: nbstripout + types: [file] + files: (.ipynb)$ + language: system + - id: python-bandit-vulnerability-check + name: Security check (bandit) + entry: bandit + types: [python] + args: ["--recursive", "lib/"] + language: system + - id: pytest-check + name: Tests (pytest) + stages: [push] + entry: pytest tests/ + types: [python] + language: system + pass_filenames: false + always_run: true +exclude: ^(.svn|CVS|.bzr|.hg|.git|__pycache__|.tox|.ipynb_checkpoints|assets|tests/assets/|venv/|.venv/) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d8f173 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +USE_CONDA ?= 1 +INSTALL_SCRIPT = install_with_conda.sh +ifeq (false,$(USE_CONDA)) + INSTALL_SCRIPT = install_with_venv.sh +endif + +.DEFAULT_GOAL = help + +# help: help - Display this makefile's help information +.PHONY: help +help: + @grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://' + +# help: install - Create a virtual environment and install dependencies +.PHONY: install +install: + @bash bin/$(INSTALL_SCRIPT) + +# help: install_precommit - Install pre-commit hooks +.PHONY: install_precommit +install_precommit: + @pre-commit install -t pre-commit + @pre-commit install -t pre-push + +# help: serve_docs_locally - Serve docs locally on port 8001 +.PHONY: serve_docs_locally +serve_docs_locally: + @mkdocs serve --livereload -a localhost:8001 + +# help: deploy_docs - Deploy documentation to GitHub Pages +.PHONY: deploy_docs +deploy_docs: + @mkdocs build + @mkdocs gh-deploy diff --git a/README.md b/README.md new file mode 100644 index 0000000..e68d0c7 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +
+ +# skaff-rag-accelerator + +[![CI status](https://github.com/artefactory/skaff-rag-accelerator/actions/workflows/ci.yaml/badge.svg)](https://github.com/artefactory/skaff-rag-accelerator/actions/workflows/ci.yaml?query=branch%3Amain) +[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg)]() + +[![Linting , formatting, imports sorting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit) +[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-informational?logo=pre-commit&logoColor=white)](https://github.com/artefactory/skaff-rag-accelerator/blob/main/.pre-commit-config.yaml) +
+ +TODO: if not done already, check out the [Skaff documentation](https://artefact.roadie.so/catalog/default/component/repo-builder-ds/docs/) for more information about the generated repository. + +Deploy RAGs quickly + +## Table of Contents + +- [skaff-rag-accelerator](#skaff-rag-accelerator) + - [Table of Contents](#table-of-contents) + - [Installation](#installation) + - [Usage](#usage) + - [Documentation](#documentation) + - [Repository Structure](#repository-structure) + +## Installation + +To install the required packages in a virtual environment, run the following command: + +```bash +make install +``` + +TODO: Choose between conda and venv if necessary or let the Makefile as is and copy/paste the [MORE INFO installation section](MORE_INFO.md#eased-installation) to explain how to choose between conda and venv. + +A complete list of available commands can be found using the following command: + +```bash +make help +``` + +## Usage + +TODO: Add usage instructions here + +## Documentation + +TODO: Github pages is not enabled by default, you need to enable it in the repository settings: Settings > Pages > Source: "Deploy from a branch" / Branch: "gh-pages" / Folder: "/(root)" + +A detailed documentation of this project is available [here](https://artefactory.github.io/skaff-rag-accelerator/) + +To serve the documentation locally, run the following command: + +```bash +mkdocs serve +``` + +To build it and deploy it to GitHub pages, run the following command: + +```bash +make deploy_docs +``` + +## Repository Structure + +``` +. +├── .github <- GitHub Actions workflows and PR template +├── bin <- Bash files +├── config <- Configuration files +├── docs <- Documentation files (mkdocs) +├── lib <- Python modules +├── notebooks <- Jupyter notebooks +├── secrets <- Secret files (ignored by git) +└── tests <- Unit tests +``` diff --git a/bin/install_with_conda.sh b/bin/install_with_conda.sh new file mode 100644 index 0000000..a8e1f74 --- /dev/null +++ b/bin/install_with_conda.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +read -p "Want to install conda env named 'skaff-rag-accelerator'? (y/n)" answer +if [ "$answer" = "y" ]; then + echo "Installing conda env..." + conda create -n skaff-rag-accelerator python=3.10 -y + source $(conda info --base)/etc/profile.d/conda.sh + conda activate skaff-rag-accelerator + echo "Installing requirements..." + pip install -r requirements-developer.txt + python3 -m ipykernel install --user --name=skaff-rag-accelerator + conda install -c conda-forge --name skaff-rag-accelerator notebook -y + echo "Installing pre-commit..." + make install_precommit + echo "Installation complete!"; +else + echo "Installation of conda env aborted!"; +fi diff --git a/bin/install_with_venv.sh b/bin/install_with_venv.sh new file mode 100644 index 0000000..1086890 --- /dev/null +++ b/bin/install_with_venv.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +read -p "Want to install virtual env named 'venv' in this project ? (y/n)" answer +if [ "$answer" = "y" ]; then + echo "Installing virtual env..." + declare VENV_DIR=$(pwd)/venv + if ! [ -d "$VENV_DIR" ]; then + python3 -m venv $VENV_DIR + fi + + source $VENV_DIR/bin/activate + echo "Installing requirements..." + pip install -r requirements-developer.txt + python3 -m ipykernel install --user --name=venv + echo "Installing pre-commit..." + make install_precommit + echo "Installation complete!"; +else + echo "Installation of virtual env aborted!"; +fi diff --git a/config/config.py b/config/config.py new file mode 100644 index 0000000..111cc12 --- /dev/null +++ b/config/config.py @@ -0,0 +1,4 @@ +"""Module for dynamic configuration variables of the project.""" +from pathlib import Path + +ROOT_PATH = Path(__file__).parent.parent diff --git a/config/config.toml b/config/config.toml new file mode 100644 index 0000000..6b45397 --- /dev/null +++ b/config/config.toml @@ -0,0 +1,2 @@ +# Write config variables here +# See https://toml.io/en/ for more information diff --git a/docs/code.md b/docs/code.md new file mode 100644 index 0000000..aa45473 --- /dev/null +++ b/docs/code.md @@ -0,0 +1 @@ +# Code diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..8013429 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Welcome to the documentation! + +For more information, make sure to check the [Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/getting-started/) diff --git a/lib/.gitkeep b/lib/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/mkdocs.yaml b/mkdocs.yaml new file mode 100644 index 0000000..218e74c --- /dev/null +++ b/mkdocs.yaml @@ -0,0 +1,42 @@ +site_name: skaff-rag-accelerator + +theme: + name: "material" + palette: + - media: "(prefers-color-scheme: dark)" + scheme: default + primary: teal + accent: amber + toggle: + icon: material/moon-waning-crescent + name: Switch to dark mode + - media: "(prefers-color-scheme: light)" + scheme: slate + primary: teal + accent: amber + toggle: + icon: material/white-balance-sunny + name: Switch to light mode + features: + - search.suggest + - search.highlight + - content.tabs.link + - content.code.annotation + - content.code.copy + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + +plugins: + - mkdocstrings + - search + +nav: + - Home: index.md + - Source code: code.md diff --git a/notebooks/private/.gitignore b/notebooks/private/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/notebooks/private/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/notebooks/template.ipynb b/notebooks/template.ipynb new file mode 100644 index 0000000..298da4c --- /dev/null +++ b/notebooks/template.ipynb @@ -0,0 +1,79 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import pandas as pd\n", + "pd.set_option('display.max_columns', 500)\n", + "pd.options.plotting.backend = \"plotly\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(\"\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# EDA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a5fed58 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "skaff-rag-accelerator" +authors = [ + { name = "alexisVLRT", email = "alexis.vialaret@artefact.com" } +] +description = "Deploy RAGs quickly" +version = "0.0.1" +readme = "README.md" +requires-python = ">=3.8" + +[project.urls] +"Homepage" = "https://github.com/artefactory/skaff-rag-accelerator" +"Documentation" = "https://artefactory.github.io/skaff-rag-accelerator" + +[tool.setuptools] +packages = ["lib", "config", "tests"] + +[tool.ruff] +select = [ + "E", + "W", + "F", + "I", + "N", + "D", + "ANN", + "Q", + "RET", + "ARG", + "PTH", + "PD", +] # See: https://beta.ruff.rs/docs/rules/ +ignore = ["D203", "D213", "ANN101", "ANN102"] +line-length = 100 +target-version = "py310" +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", +] + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.format] +quote-style = "double" + +[tool.ruff.isort] +known-first-party = ["lib", "config", "tests"] diff --git a/requirements-developer.txt b/requirements-developer.txt new file mode 100644 index 0000000..20cf8a4 --- /dev/null +++ b/requirements-developer.txt @@ -0,0 +1,10 @@ +-r requirements.txt +ruff==0.1.2 +pre-commit==3.3.3 +pytest==7.3.2 +mkdocs==1.4.3 +mkdocs-material==9.1.15 +mkdocstrings-python==1.1.2 +bandit==1.7.5 +nbstripout==0.6.1 +ipykernel==6.24.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d23458e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +-e . +pandas==1.5.3 +numpy==1.24.2 diff --git a/secrets/.gitkeep b/secrets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/.gitkeep b/tests/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/.gitkeep b/tests/integration_tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/.gitkeep b/tests/unit_tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/test_placeholder.py b/tests/unit_tests/test_placeholder.py new file mode 100644 index 0000000..338a8e0 --- /dev/null +++ b/tests/unit_tests/test_placeholder.py @@ -0,0 +1,6 @@ +"""Placeholder test file for unit tests. To be replaced with actual tests.""" + + +def test_placeholder() -> None: + """To be replaced with actual tests.""" + pass