ci: Add workflows

.github/CODEOWNERS

@@ -0,0 +1,11 @@
+# See https://help.github.com/articles/about-codeowners/ for syntax
+
+# Core Engineering will be the default owners for everything
+# in the repo. Unless a later match takes precedence,
+# @deepset-ai/core-engineering will be requested for review
+# when someone opens a pull request.
+*                       @deepset-ai/open-source-engineering
+
+# Documentation
+*.md                    @deepset-ai/documentation @deepset-ai/open-source-engineering
+releasenotes/notes/*    @deepset-ai/documentation @deepset-ai/open-source-engineering

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Errors you encountered
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Error message**
+Error that was thrown (if available)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.
+
+**To Reproduce**
+Steps to reproduce the behavior
+
+**FAQ Check**
+- [ ] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?
+
+**System:**
+ - OS:
+ - GPU/CPU:
+ - Haystack version (commit or version number):
+ - DocumentStore:
+ - Reader:
+ - Retriever:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/actionlint.yml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of string
+  labels: ["cml", "ubuntu-latest-4-cores"]

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'daily'

.github/pull_request_template.md

@@ -0,0 +1,25 @@
+### Related Issues
+
+- fixes #issue-number
+
+### Proposed Changes:
+
+ <!--- In case of a bug: Describe what caused the issue and how you solved it -->
+ <!--- In case of a feature: Describe what did you add and how it works -->
+
+### How did you test it?
+
+<!-- unit tests, integration tests, manual verification, instructions for manual tests -->
+
+### Notes for the reviewer
+
+<!-- E.g. point out section where the reviewer  -->
+
+### Checklist
+
+- I have read the [contributors guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/deepset-ai/haystack/blob/main/code_of_conduct.txt)
+- I have updated the related issue with new insights and changes
+- I added unit tests and updated the docstrings
+- I've used one of the [conventional commit types](https://www.conventionalcommits.org/en/v1.0.0/) for my PR title: `fix:`, `feat:`, `build:`, `chore:`, `ci:`, `docs:`, `style:`, `refactor:`, `perf:`, `test:`.
+- I documented my code
+- I ran [pre-commit hooks](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md#installation) and fixed any issue

.github/utils/pyproject_to_requirements.py

@@ -0,0 +1,47 @@
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import toml
+
+matcher = re.compile(r"farm-haystack\[(.+)\]")
+parser = argparse.ArgumentParser(
+    prog="pyproject_to_requirements.py", description="Convert pyproject.toml to requirements.txt"
+)
+parser.add_argument("pyproject_path")
+parser.add_argument("--extra", default="")
+
+
+def resolve(target: str, extras: dict, results: set):
+    if target not in extras:
+        results.add(target)
+        return
+
+    for t in extras[target]:
+        m = matcher.match(t)
+        if m:
+            for i in m.group(1).split(","):
+                resolve(i, extras, results)
+        else:
+            resolve(t, extras, results)
+
+
+def main(pyproject_path: Path, extra: str = ""):
+    content = toml.load(pyproject_path)
+    # basic set of dependencies
+    deps = set(content["project"]["dependencies"])
+
+    if extra:
+        extras = content["project"]["optional-dependencies"]
+        resolve(extra, extras, deps)
+
+    sys.stdout.write("\n".join(sorted(deps)))
+    sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    pyproject_path = Path(args.pyproject_path).absolute()
+
+    main(pyproject_path, args.extra)

.github/workflows/docstrings_linting.yml

@@ -0,0 +1,20 @@
+name: run docstrings linting
+
+on:
+  push:
+    branches:
+      - docstrings-linting
+
+jobs:
+  docstrings-linting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Hatch
+        run: pip install hatch=="1.9.3"
+
+      - name: ruff docstrings linting
+        run: hatch run ruff check haystack-experimental

.github/workflows/license_compliance.yml

@@ -0,0 +1,92 @@
+name: License Compliance
+
+on:
+  pull_request:
+    paths:
+      - "**/pyproject.toml"
+  # Since we test PRs, there is no need to run the workflow at each
+  # merge on `main`. Let's use a cron job instead.
+  schedule:
+    - cron: "0 0 * * *" # every day at midnight
+
+env:
+  CORE_DATADOG_API_KEY: ${{ secrets.CORE_DATADOG_API_KEY }}
+  PYTHON_VERSION: "3.10"
+
+jobs:
+  license_check_direct:
+    name: Direct dependencies only
+    env:
+      REQUIREMENTS_FILE: requirements_direct.txt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Get direct dependencies
+        run: |
+          pip install toml
+          python .github/utils/pyproject_to_requirements.py pyproject.toml > ${{ env.REQUIREMENTS_FILE }}
+
+      - name: Check Licenses
+        id: license_check_report
+        uses: pilosus/action-pip-license-checker@v2
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          requirements: ${{ env.REQUIREMENTS_FILE }}
+          fail: "Copyleft,Other,Error"
+          # Exclusions in the vanilla distribution must be explicitly motivated
+          #
+          # - tqdm is MLP but there are no better alternatives
+          exclude: "(?i)^(tqdm).*"
+
+      # We keep the license inventory on FOSSA
+      - name: Send license report to Fossa
+        uses: fossas/[email protected]
+        continue-on-error: true # not critical
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+      - name: Print report
+        if: ${{ always() }}
+        run: echo "${{ steps.license_check_report.outputs.report }}"
+
+      - name: Calculate alert data
+        id: calculator
+        shell: bash
+        if: (success() || failure())
+        run: |
+          if [ "${{ job.status }}" = "success" ]; then
+            echo "alert_type=success" >> "$GITHUB_OUTPUT";
+          else
+            echo "alert_type=error" >> "$GITHUB_OUTPUT";
+          fi
+
+      - name: Send event to Datadog
+        # This step would fail when running in PRs opened from forks since
+        # secrets are not accessible.
+        # To prevent showing bogus failures in those PRs we skip the step.
+        # The workflow will fail in any case if the actual check fails in the previous steps.
+        if: (success() || failure()) && env.CORE_DATADOG_API_KEY != ''
+        uses: masci/datadog@v1
+        with:
+          api-key: ${{ env.CORE_DATADOG_API_KEY }}
+          api-url: https://api.datadoghq.eu
+          events: |
+            - title: "${{ github.job }} in ${{ github.workflow }} workflow"
+              text: "License compliance check: direct dependencies only."
+              alert_type: "${{ steps.calculator.outputs.alert_type }}"
+              source_type_name: "Github"
+              host: ${{ github.repository_owner }}
+              tags:
+                - "project:${{ github.repository }}"
+                - "job:${{ github.job }}"
+                - "run_id:${{ github.run_id }}"
+                - "workflow:${{ github.workflow }}"
+                - "branch:${{ github.ref_name }}"
+                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/linting.yml

@@ -0,0 +1,84 @@
+# If you change this name also do it in linting-skipper.yml and ci_metrics.yml
+name: Linting
+
+on:
+  pull_request:
+    paths:
+      - "haystack-experimental/**/*.py"
+      - "test/**/*.py"
+      - "pyproject.toml"
+
+env:
+  PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
+
+jobs:
+  license-header:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check License Header
+        run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check
+
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: files
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+            **/*.py
+          files_ignore: |
+            test/**
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
+
+      - name: Mypy
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          mkdir .mypy_cache
+          hatch run test:types ${{ steps.files.outputs.all_changed_files }}
+
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: files
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+            haystack-experimental/**/*.py
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
+
+      - name: Pylint
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          hatch run test:lint ${{ steps.files.outputs.all_changed_files }}

.github/workflows/linting_skipper.yml

@@ -0,0 +1,29 @@
+# If you change this name also do it in linting.yml and ci_metrics.yml
+name: Linting
+
+on:
+  pull_request:
+    paths-ignore:
+      - "haystack/preview/**/*.py"
+      - "test/preview/**/*.py"
+      - "e2e/preview/**/*.py"
+      - "**/pyproject.toml"
+
+jobs:
+  license-header:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skip mypy
+        run: echo "Skipped mypy"
+
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skip mypy
+        run: echo "Skipped mypy"
+
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skip pylint
+        run: echo "Skipped pylint"

.github/workflows/project.yml

@@ -0,0 +1,16 @@
+name: Track issues with Github project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add new issues to project for triage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/deepset-ai/projects/5
+          github-token: ${{ secrets.GH_PROJECT_PAT }}