Merge branch 'feast-dev:master' into master

.devcontainer/devcontainer.json

@@ -17,13 +17,13 @@
 		"ghcr.io/devcontainers-contrib/features/maven-sdkman:2": {
 			"jdkVersion": "11.0.24-amzn"
 		}
-	},
+	}
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 
 	// Uncomment the next line to run commands after the container is created.
-	"postCreateCommand": "make install-python-ci-dependencies-uv-venv"
+	// "postCreateCommand": "make install-python-ci-dependencies-uv-venv"
 
 	// Configure tool-specific properties.
 	// "customizations": {},

.github/workflows/smoke_tests.yml

@@ -0,0 +1,38 @@
+name: smoke-tests
+
+on: [pull_request]
+jobs:
+  unit-test-python:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11"]
+        os: [ ubuntu-latest ]
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Get uv cache dir
+        id: uv-cache
+        run: |
+          echo "::set-output name=dir::$(uv cache dir)"
+      - name: uv cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.uv-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
+      - name: Install dependencies
+        run: make install-python-dependencies-uv
+      - name: Test Imports
+        run: python -c "from feast import cli"

.gitignore

@@ -185,7 +185,6 @@ dmypy.json
 
 # Protos
 sdk/python/docs/html
-sdk/python/feast/protos/
 sdk/go/protos/
 go/protos/
 

Makefile

@@ -37,32 +37,31 @@ build: protos build-java build-docker
 
 # Python SDK
 
+install-python-dependencies-uv:
+	uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt
+	uv pip install --system --no-deps .
+
+install-python-dependencies-uv-venv:
+	uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt
+	uv pip install --no-deps .
+
 install-python-ci-dependencies:
 	python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	pip install --no-deps -e .
-	python setup.py build_python_protos --inplace
 
 install-python-ci-dependencies-uv:
 	uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	uv pip install --system --no-deps -e .
-	python setup.py build_python_protos --inplace
 
 install-python-ci-dependencies-uv-venv:
 	uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 	uv pip install --no-deps -e .
-	python setup.py build_python_protos --inplace
-
-install-protoc-dependencies:
-	pip install "protobuf<5" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
 
 lock-python-ci-dependencies:
 	uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
 
-package-protos:
-	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
-
-compile-protos-python: install-protoc-dependencies
-	python setup.py build_python_protos --inplace
+compile-protos-python:
+	python infra/scripts/generate_protos.py
 
 install-python:
 	python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt

docs/reference/alpha-web-ui.md

@@ -70,7 +70,7 @@ ReactDOM.render(
 );
 ```
 
-When you start the React app, it will look for `project-list.json` to find a list of your projects. The JSON should looks something like this.
+When you start the React app, it will look for `projects-list.json` to find a list of your projects. The JSON should look something like this.
 
 ```json
 {

examples/README.md

@@ -0,0 +1,16 @@
+# Feast Examples
+
+1. **[Quickstart Example](https://github.com/feast-dev/feast/tree/master/examples/quickstart)**: This is a step-by-step guide for getting started with Feast.
+
+2. **[Java Demo](https://github.com/feast-dev/feast/tree/master/examples/java-demo)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes.
+
+3. **[Python Helm Demo](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server.
+
+4. **[RBAC Local](https://github.com/feast-dev/feast/tree/master/examples/rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment.
+
+5. **[RBAC Remote](https://github.com/feast-dev/feast/tree/master/examples/rbac-local)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment.
+
+6. **[Remote Offline Store](https://github.com/feast-dev/feast/tree/master/examples/remote-offline-store)**: Demonstrates how to set up and use remote offline server.
+
+7. **[Podman/Podman Compose_local](https://github.com/feast-dev/feast/tree/master/examples/podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally.
+

examples/podman_local/README.md

@@ -0,0 +1,72 @@
+
+# Feast example using Podman and Podman Compose
+
+This guide explains how to deploy Feast remote server components using Podman Compose locally and run an example using the client.
+
+## Prerequisites
+
+1. **Podman**: [Podman installation guide](https://podman.io/).
+2. **Podman Compose**: [Podman Compose Installation guide](https://github.com/containers/podman-compose/tree/main?tab=readme-ov-file#installation]).
+3. **Python 3.9+ environment**
+4. **Feast CLI** 
+
+## Setup
+
+### 1. **Feast Project Setup**
+
+- The project [feature_repo](feature_repo) already created using `feast init` command
+
+### 2. **Run the Podman Compose File**
+
+- Use the [docker-compose.yml](docker-compose.yml) file to install and run the Feast feature servers (online, offline, and registry) on podman. The docker-compose file uses the `feastdev/feature-server:latest` image. Each respective service has specific port mappings and maps the volume from the  `./feature_repo` configuration.
+- To start the feature servers, run the following command:
+
+  ```bash
+  podman-compose up -d
+  ```
+
+- This will launch the necessary containers for online, offline, and registry feature servers.
+
+### 3. **Verify the Installation**
+
+- Use the `podman ps` command to verify that the containers are running:
+
+  ```bash
+  podman ps
+  ```
+
+  Example output:
+
+  ```
+  CONTAINER ID  IMAGE                                     COMMAND               CREATED        STATUS        PORTS                   NAMES
+  61442d6d6ef3  docker.io/feastdev/feature-server:latest  feast -c /feature...  2 minutes ago  Up 2 minutes  0.0.0.0:6566->6566/tcp  online-feature-server
+  1274c21716a6  docker.io/feastdev/feature-server:latest  feast -c /feature...  2 minutes ago  Up 2 minutes  0.0.0.0:8815->8815/tcp  offline-feature-server
+  4e38ca8c39db  docker.io/feastdev/feature-server:latest  feast -c /feature...  2 minutes ago  Up 2 minutes  0.0.0.0:6570->6570/tcp  registry-feature-server
+  ```
+
+- Alternatively, you can verify the running containers through **Podman Desktop**:
+  ![podman.png](podman.png)
+
+### 4. **Run Feast Apply**
+
+- To apply the feature store definitions to the remote registry, run the following command:
+
+  ```bash
+  podman exec registry-feature-server feast -c /feature_repo apply
+  ```
+
+### 5. **Run Client Examples**
+
+- The [client](client) folder contains example client-side configurations and code:
+   - [feature_store.yaml](client/feature_repo/feature_store.yaml): Configuration for the feature store.
+   - [test.py](client/feature_repo/test.py): Example Python script to interact with the Feast server.
+
+### 6. **Cleanup**
+
+- To stop and remove the running containers, run the following command:
+
+  ```bash
+  podman-compose down
+  ```
+
+- This will stop all the feature server containers and clean up the environment.

examples/podman_local/client/feature_repo/feature_store.yaml

@@ -0,0 +1,12 @@
+project: my_project
+registry:
+  registry_type: remote
+  path: localhost:6570
+offline_store:
+  type: remote
+  host: localhost
+  port: 8815
+online_store:
+  type: remote
+  path: http://localhost:6566
+

examples/podman_local/client/feature_repo/test.py

@@ -0,0 +1,123 @@
+import subprocess
+from datetime import datetime
+import pandas as pd
+from feast import FeatureStore
+from feast.data_source import PushMode
+
+def run_demo():
+    try:
+        store = FeatureStore(repo_path=".")
+
+        print("\n--- Historical features for training ---")
+        fetch_historical_features_entity_df(store, for_batch_scoring=False)
+
+        print("\n--- Historical features for batch scoring ---")
+        fetch_historical_features_entity_df(store, for_batch_scoring=True)
+
+        print("\n--- Load features into online store ---")
+        store.materialize_incremental(end_date=datetime.now())
+
+        print("\n--- Online features ---")
+        fetch_online_features(store)
+
+        print("\n--- Online features retrieved (instead) through a feature service---")
+        fetch_online_features(store, source="feature_service")
+
+        print(
+            "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---"
+        )
+        fetch_online_features(store, source="push")
+
+        print("\n--- Simulate a stream event ingestion of the hourly stats df ---")
+        event_df = pd.DataFrame.from_dict(
+            {
+                "driver_id": [1001],
+                "event_timestamp": [
+                    datetime.now(),
+                ],
+                "created": [
+                    datetime.now(),
+                ],
+                "conv_rate": [1.0],
+                "acc_rate": [1.0],
+                "avg_daily_trips": [1000],
+            }
+        )
+        print(event_df)
+        store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE)
+
+        print("\n--- Online features again with updated values from a stream push---")
+        fetch_online_features(store, source="push")
+    except Exception as e:
+        print(f"An error occurred in run_demo: {e}")
+
+def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: bool):
+    try:
+        entity_df = pd.DataFrame.from_dict(
+            {
+                "driver_id": [1001, 1002, 1003],
+                "event_timestamp": [
+                    datetime(2021, 4, 12, 10, 59, 42),
+                    datetime(2021, 4, 12, 8, 12, 10),
+                    datetime(2021, 4, 12, 16, 40, 26),
+                ],
+                "label_driver_reported_satisfaction": [1, 5, 3],
+                "val_to_add": [1, 2, 3],
+                "val_to_add_2": [10, 20, 30],
+            }
+        )
+        if for_batch_scoring:
+            entity_df["event_timestamp"] = pd.to_datetime("now", utc=True)
+
+        training_df = store.get_historical_features(
+            entity_df=entity_df,
+            features=[
+                "driver_hourly_stats:conv_rate",
+                "driver_hourly_stats:acc_rate",
+                "driver_hourly_stats:avg_daily_trips",
+                "transformed_conv_rate:conv_rate_plus_val1",
+                "transformed_conv_rate:conv_rate_plus_val2",
+            ],
+        ).to_df()
+        print(training_df.head())
+    except Exception as e:
+        print(f"An error occurred in fetch_historical_features_entity_df: {e}")
+
+def fetch_online_features(store, source: str = ""):
+    try:
+        entity_rows = [
+            {
+                "driver_id": 1001,
+                "val_to_add": 1000,
+                "val_to_add_2": 2000,
+            },
+            {
+                "driver_id": 1002,
+                "val_to_add": 1001,
+                "val_to_add_2": 2002,
+            },
+        ]
+        if source == "feature_service":
+            features_to_fetch = store.get_feature_service("driver_activity_v1")
+        elif source == "push":
+            features_to_fetch = store.get_feature_service("driver_activity_v3")
+        else:
+            features_to_fetch = [
+                "driver_hourly_stats:acc_rate",
+                "transformed_conv_rate:conv_rate_plus_val1",
+                "transformed_conv_rate:conv_rate_plus_val2",
+            ]
+        returned_features = store.get_online_features(
+            features=features_to_fetch,
+            entity_rows=entity_rows,
+        ).to_dict()
+        for key, value in sorted(returned_features.items()):
+            print(key, " : ", value)
+    except Exception as e:
+        print(f"An error occurred in fetch_online_features: {e}")
+
+if __name__ == "__main__":
+    try:
+        run_demo()
+    except Exception as e:
+        print(f"An error occurred in the main block: {e}")

examples/podman_local/docker-compose.yml

@@ -0,0 +1,33 @@
+version: '3.9'
+
+x-defaults: &default-settings
+  image: feastdev/feature-server:latest
+  restart: unless-stopped
+
+services:
+  online-feature-server:
+    <<: *default-settings
+    container_name: online-feature-server
+    command: feast -c /feature_repo serve -h 0.0.0.0
+    ports:
+      - "6566:6566"
+    volumes:
+      - ./feature_repo:/feature_repo
+
+  offline-feature-server:
+    <<: *default-settings
+    container_name: offline-feature-server
+    command: feast -c /feature_repo serve_offline -h 0.0.0.0
+    ports:
+      - "8815:8815"
+    volumes:
+      - ./feature_repo:/feature_repo
+
+  registry-feature-server:
+    <<: *default-settings
+    container_name: registry-feature-server
+    command: feast -c /feature_repo serve_registry
+    ports:
+      - "6570:6570"
+    volumes:
+      - ./feature_repo:/feature_repo