Feat temporal actor snapshots (eventsourced*)

Makefile

@@ -63,6 +63,9 @@ test-spawn:
 test-statestores_mysql:
 	cd spawn_statestores/statestores_mysql && MIX_ENV=test mix deps.get && MIX_ENV=test  PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
 
+test-statestores_mariadb:
+	cd spawn_statestores/statestores_mariadb && MIX_ENV=test mix deps.get && MIX_ENV=test  PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
+
 test-statestores_postgres:
 	cd spawn_statestores/statestores_postgres && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
 
@@ -144,6 +147,9 @@ run-proxy-local:
 run-proxy-local2:
 	ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_HTTP_PORT=9003 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name [email protected] -S mix
 
+run-proxy-local-3:
+	cd spawn_proxy/proxy && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_PORT=3307 PROXY_DATABASE_TYPE=mariadb PROXY_HTTP_PORT=9003 USER_FUNCTION_PORT=8091 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name [email protected] -S mix
+
 run-proxy-local-nodejs-test:
 	ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_HTTP_PORT=9001 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= PROXY_ACTOR_SYSTEM_NAME=SpawnSysTest SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=persistent iex --name [email protected] -S mix
 
@@ -156,6 +162,9 @@ run-sdk-local2:
 run-sdk-local3:
 	cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_TYPE=$(database) SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name [email protected] -S mix
 
+run-sdk-local-with-mariadb:
+	cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_PORT=3307 PROXY_DATABASE_TYPE=mariadb SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name [email protected] -S mix
+
 run-sdk-local-nats:
 	cd spawn_sdk/spawn_sdk_example && PROXY_CLUSTER_STRATEGY=epmd PROXY_DATABASE_TYPE=$(database) SPAWN_USE_INTERNAL_NATS=true SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name [email protected] -S mix
 

README.md

@@ -8,6 +8,31 @@
 
 ### **[Website](https://eigr.io)** • **[Getting Started](#getting-started)** • **[SDKs](#sdks)** • **[Documentation](https://eigr.io/docs/projects-spawn/spawn-introduction/)** • **[Blog](https://eigr.io/blog/)**
 
+
+# Table of Contents
+1. [Overview](#overview)
+   - [What problem Spawn solves](#what-problem-spawn-solves)
+   - [Spawn Architecture](#spawn-architecture)
+2. [Features](#features)
+3. [Install](#install)
+   - [Prerequisites](#prerequisites)
+   - [Instructions](#instructions)
+4. [Getting Started](#getting-started)
+   - [Examples](#examples)
+5. [SDKs](#sdks)
+6. [Custom Resources](#custom-resources)
+7. [Statestores](#statestores)
+   - [Actor State Checkpoints Restore](#actor-state-checkpoints-restore)
+   - [Statestore Features](#statestore-features)
+8. [Local Development](#local-development)
+8. [Main Concepts](#main-concepts)
+   - [The Protocol](#the-protocol)
+   - [The Actor Model](#the-actor-model)
+   - [The Sidecar Pattern](#the-sidecar-pattern)
+   - [Nats](#nats)
+9. [Talks](#talks)
+
+
 ## Overview
 
 **_What is Spawn?_**
@@ -57,7 +82,7 @@ Watch the video explaining how it works:
 
 > **_NOTE:_** This video was recorded with an old version of the SDK for Java. That's why errors are seen in Deployment
 
-## What problem Spawn solves
+### What problem Spawn solves
 
 The advancement of Cloud Computing, Edge computing, Containers, Orchestrators, Data-
 Oriented Services, and global-scale products aimed at serving audiences in various regions of
@@ -93,7 +118,7 @@ processing, real-time data ingestion, service integrations, financial or transac
 and logistics are some of the domains that can be mastered by the Eigr Functions Spawn
 platform.
 
-## Spawn Architecture
+### Spawn Architecture
 
 Spawn takes the Actor Model's distribution, fault tolerance, and high concurrent capability in
 its most famous implementation, the BEAM Erlang VM implementation, and adds to the
@@ -133,13 +158,17 @@ In turn, each Sidecar container within a POD organizes itself to form an Erlang
     - [x] Automatic renewal of certificates.
   - [x] Cross ActorSystem invocation Nats distribution.
 - [x] Configuration management via Kubernetes [CRDs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) and Envinronment Variables.
-- [x] Statestores. Adapters for persistent storage using multiple database providers.
-  - [x] Sqlite
-  - [x] MySql
-  - [x] Postgres
-  - [x] CockroachDB
-  - [x] MSSQL
-  - [ ] Native via Mnesia
+- [x] State Management. 
+  - [x] Supported database adapters for persistent storage using multiple database providers.
+    - [x] Sqlite
+    - [x] MariaDB
+    - [x] MySql
+    - [x] Postgres
+    - [x] CockroachDB
+    - [x] MSSQL
+    - [ ] Native via Mnesia
+  - [x] Write behind during execution and Write ahead during deactivation.
+  - [x] Point in time Recovery. See Statestore for more information.
 - [x] Automatic activation and deactivation of Actors.
 - [x] Horizontal Scalability
   - [x] automatically controlled by the Operator using Kubernetes HPA based on memory and cpu.
@@ -155,10 +184,10 @@ In turn, each Sidecar container within a POD organizes itself to form an Erlang
   - [x] Side effects. Sends an effect as a result of your computation for other Actors to handle.
   - [ ] Saga.
 - [x] SDKs
-  - [x] Elixir. All features implemented.
-  - [x] Node/Typescript. All features implemented.
-  - [x] Java. Partially implemented.
-  - [ ] Python. Under development.
+  - [x] Elixir.
+  - [x] Node/Typescript.
+  - [x] Java.
+  - [x] Python.
   - [ ] Go. Under development.
   - [ ] Rust. Under development.
   - [ ] .Net/C#. Under development.
@@ -338,12 +367,6 @@ You can find some examples of using Spawn in the links below:
 - **Fleet**: https://github.com/sleipnir/fleet-spawn-example
 - **Spawn Polyglot Example**: https://github.com/sleipnir/spawn-polyglot-ping-pong
 
-### Talks
-
-You can see some talks on Youtube about Eigr Community or Spawn in the links below:
-
-- **Marcel Lanz on Code Beam Europe 2022**: https://youtu.be/jgR7Oc_GXAg
-- **Adriano Santos on Code Beam BR 2022**: Link not yet released by the event organizers
 
 ## SDKs
 
@@ -357,10 +380,10 @@ abstract all the protocol specifics and expose an easy and intuitive API to deve
 | [Go SDK](https://github.com/eigr/spawn-go-sdk)                        | Go       |
 | [Spring Boot SDK](https://github.com/eigr/spawn-springboot-sdk)       | Java     |
 | [NodeJS/Typescript SDK](https://github.com/eigr/spawn-node-sdk)       | Node     |
-| [Python SDK](https://github.com/eigr-labs/spawn-python-sdk)           | Python   |
+| [Python SDK](https://github.com/eigr/spawn-python-sdk)                | Python   |
 | [Rust SDK](https://github.com/eigr-labs/spawn-rust-sdk)               | Rust     |
 
-### Custom Resources
+## Custom Resources
 
 Spawn defines some custom Resources for the user to interact with the API for deploying Spawn artifacts in Kubernetes. We'll talk more about these CRDs in the Getting Started section but for now we'll list each of these resources below for a general understanding of the concepts:
 
@@ -408,6 +431,28 @@ Below is a list of common global settings for all Statestores. For more details
 
 > **_NOTE:_** When running on top of Kubernetes you only need to set the CRD attributes of ActorSystem and Kubernetes secrets. The Operator will set the values of the environment variables according to the settings of these two mentioned places.
 
+### Actor State Checkpoints Restore
+
+Spawn provides the ability to start Actors from a certain point in time.
+For this we use the concept of revision.
+A review happens whenever a state change is detected by the actor and a recording of the new state is made during a snapshot event. Each time this occurs an increment in the revision number will occur marking the state to that moment in time.
+Developers can therefore start any Actor from a specific point in time which is marked by a revision.
+How developers will do this will depend on the APIs exposed by the SDK's for each specific language, so to learn more about this feature, check the desired SDK page.
+
+It is also worth mentioning that this feature depends on the implementation of each of our persistent storage adapters, so check the table in the section below to find out if the adapter for your database supports this feature.
+
+### Statestore Features
+
+| Feature                                   | CockroachDB | MariaDB | Mnesia | MSSQL | MySQL | Postgres | SQLite |
+| ------------------------------------------| ------------| --------| -------| ------| ------| ---------| -------|
+| Actor Fast key lookup                     |     [x]     |   [x]   |   [ ]  |  [x]  |  [x]  |   [x]    |   [x]  |
+| Actor State restore from Revision         |     [ ]     |   [x]   |   [ ]  |  [ ]  |  [ ]  |   [ ]    |   [ ]  |
+| Search by Actors Metadata                 |     [ ]     |   [ ]   |   [ ]  |  [ ]  |  [ ]  |   [ ]    |   [ ]  |
+| Search by all changes of Actor states     |     [ ]     |   [x]   |   [ ]  |  [ ]  |  [ ]  |   [ ]    |   [ ]  |
+| Search by all Actor state changes by date |     [ ]     |   [x]   |   [ ]  |  [ ]  |  [ ]  |   [ ]    |   [ ]  |
+| Snapshot Data Partition                   |     [ ]     |   [x]   |   [ ]  |  [ ]  |  [ ]  |   [ ]    |   [ ]  |
+| State AES Encryption                      |     [x]     |   [x]   |   [ ]  |  [x]  |  [x]  |   [x]    |   [x]  |
+
 ## Local Development
 
 > **_NOTE:_** All scripts will use a MySQL DB with a database called eigr-functions-db by default. Make sure you have a working instance on your localhost or you will have to change make tasks or run commands manually during testing.
@@ -492,3 +537,11 @@ https://medium.com/nerd-for-tech/microservice-design-pattern-sidecar-sidekick-pa
 We use [Nats](https://nats.io/) for communication between different systems like Activators or cross ActorSystems. According to the project page "NATS is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation (CNCF). NATS has over 40 client language implementations, and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems."
 
 Nats' ability to natively implement different topologies, as well as its minimalism, its cloud-native nature, and its capabilities to run on more constrained devices is what made us use Nats over other solutions. Nats allows Spawn to be able to provide strong isolation from an ActorSystem without limiting the user, allowing the user to still be able to communicate securely between different ActorSystems. Nats also facilitates the implementation of our triggers, called Activators, allowing those even without being part of an Erlang cluster to be able to invoke any actors.
+
+## Talks
+
+You can see some talks on Youtube about Eigr Community or Spawn in the links below:
+
+- **Marcel Lanz on Code Beam Europe 2022**: https://youtu.be/jgR7Oc_GXAg
+- **Adriano Santos on Code Beam BR 2022**: Link not yet released by the event organizers
+- **Adriano Santos ElugSP 2023**: https://www.youtube.com/watch?v=MKTqiAtpK1E

docker-compose.yaml

@@ -46,6 +46,18 @@ services:
     networks:
       - mysql-compose-network
 
+  mariadb:
+    image: mariadb
+    environment:
+      MYSQL_ROOT_PASSWORD: admin
+      MYSQL_DATABASE: eigr-functions-db
+      MYSQL_USER: admin
+      MYSQL_PASSWORD: admin
+    volumes:
+      - mariadb:/var/lib/mysql
+    ports:
+      - "3307:3306"
+
   adminer:
     image: adminer
     ports:
@@ -79,3 +91,4 @@ networks:
 volumes:
   mysql:
   postgres:
+  mariadb:

lib/actors/actor/entity/entity.ex

@@ -168,7 +168,7 @@ defmodule Actors.Actor.Entity do
   defp do_handle_info(
          message,
          %EntityState{
-           revisions: revisions,
+           revision: revision,
            actor: %Actor{id: %ActorId{name: name} = id, state: actor_state}
          } = state
        ) do
@@ -178,7 +178,7 @@ defmodule Actors.Actor.Entity do
 
     # what is the correct status here? For now we will use UNKNOWN
     if not is_nil(actor_state),
-      do: StateManager.save(id, actor_state, revision: revisions, status: "UNKNOWN")
+      do: StateManager.save(id, actor_state, revision: revision, status: "UNKNOWN")
 
     {:noreply, state, :hibernate}
   end

lib/actors/actor/entity/invocation.ex

@@ -8,7 +8,6 @@ defmodule Actors.Actor.Entity.Invocation do
 
   alias Actors.Actor.Entity.EntityState
   alias Actors.Exceptions.NotAuthorizedException
-  alias Actors.Security.Acl.DefaultAclManager
 
   alias Eigr.Functions.Protocol.Actors.{
     Actor,

lib/actors/actor/entity/lifecycle.ex

@@ -107,6 +107,7 @@ defmodule Actors.Actor.Entity.Lifecycle do
           actor:
             %Actor{settings: %ActorSettings{stateful: true}, id: %ActorId{name: name} = id} =
               actor,
+          revision: revision,
           opts: opts
         } = state
       ) do
@@ -118,7 +119,9 @@ defmodule Actors.Actor.Entity.Lifecycle do
     end
     |> Logger.debug()
 
-    case StateManager.load(id) do
+    loaded = get_state(id, revision)
+
+    case loaded do
       {:ok, current_state, current_revision, status, node} ->
         split_brain_detector =
           Keyword.get(opts, :split_brain_detector, Actors.Node.DefaultSplitBrainDetector)
@@ -129,7 +132,7 @@ defmodule Actors.Actor.Entity.Lifecycle do
            %EntityState{
              state
              | actor: %Actor{actor | state: current_state},
-               revisions: current_revision
+               revision: current_revision
            }, {:continue, :call_init_action}}
         else
           {:partition_check, {:error, :network_partition_detected}} ->
@@ -160,15 +163,15 @@ defmodule Actors.Actor.Entity.Lifecycle do
   def load_state(state), do: {:noreply, state, {:continue, :call_init_action}}
 
   def terminate(reason, %EntityState{
-        revisions: revisions,
+        revision: revision,
         actor:
           %Actor{
             id: %ActorId{name: name} = id,
             state: actor_state
           } = actor
       }) do
     if is_actor_valid?(actor) do
-      StateManager.save(id, actor_state, revision: revisions, status: @deactivated_status)
+      StateManager.save(id, actor_state, revision: revision, status: @deactivated_status)
     end
 
     Logger.debug("Terminating actor #{name} with reason #{inspect(reason)}")
@@ -211,7 +214,7 @@ defmodule Actors.Actor.Entity.Lifecycle do
         %EntityState{
           system: system,
           state_hash: old_hash,
-          revisions: revisions,
+          revision: revision,
           actor:
             %Actor{
               id: %ActorId{name: name} = id,
@@ -233,18 +236,18 @@ defmodule Actors.Actor.Entity.Lifecycle do
     new_state =
       if StateManager.is_new?(old_hash, actor_state.state) do
         Logger.debug("Snapshotting actor #{name}")
-        revision = revisions + 1
+        revision = revision + 1
 
         # Execute with timeout equals timeout strategy - 1 to avoid mailbox congestions
         case StateManager.save_async(id, actor_state, revision: revision, timeout: timeout - 1) do
           {:ok, _, hash} ->
-            %{state | state_hash: hash, revisions: revision}
+            %{state | state_hash: hash, revision: revision}
 
           {:error, _, _, hash} ->
-            %{state | state_hash: hash, revisions: revision}
+            %{state | state_hash: hash, revision: revision}
 
           {:error, :unsuccessfully, hash} ->
-            %{state | state_hash: hash, revisions: revision}
+            %{state | state_hash: hash, revision: revision}
 
           _ ->
             state
@@ -298,6 +301,29 @@ defmodule Actors.Actor.Entity.Lifecycle do
 
   def deactivate(state), do: {:noreply, state, :hibernate}
 
+  defp get_state(id, revision) do
+    initial = StateManager.load(id)
+
+    if revision <= 0 do
+      initial
+    else
+      case initial do
+        {:ok, _current_state, current_revision, _status, _node} ->
+          if current_revision != revision do
+            Logger.warning("""
+            It looks like you're looking to travel back in time. Starting state by review #{revision}.
+            Previously the review was #{current_revision}. Be careful as this type of operation can cause your actor to terminate if the attributes of its previous state schema is different from the current schema.
+            """)
+          end
+
+          StateManager.load(id, revision)
+
+        initial ->
+          initial
+      end
+    end
+  end
+
   defp is_actor_valid?(
          %Actor{
            settings: %ActorSettings{stateful: stateful},

lib/actors/actor/entity/state.ex

@@ -4,13 +4,13 @@ defmodule Actors.Actor.Entity.EntityState do
   """
   alias Eigr.Functions.Protocol.Actors.Actor
 
-  defstruct system: nil, actor: nil, state_hash: nil, revisions: 0, opts: []
+  defstruct system: nil, actor: nil, state_hash: nil, revision: 0, opts: []
 
   @type t :: %__MODULE__{
           system: String.t(),
           actor: Actor.t(),
           state_hash: binary(),
-          revisions: number(),
+          revision: number(),
           opts: Keyword.t()
         }
 

lib/actors/actor/entity/supervisor.ex

@@ -46,7 +46,14 @@ defmodule Actors.Actor.Entity.Supervisor do
   def lookup_or_create_actor(system, actor, opts \\ [])
 
   def lookup_or_create_actor(system, %Actor{} = actor, opts) when is_nil(system) do
-    entity_state = %EntityState{system: nil, actor: actor, opts: opts}
+    revision = Keyword.get(opts, :revision, 0)
+
+    entity_state = %EntityState{
+      system: nil,
+      actor: actor,
+      revision: revision,
+      opts: opts
+    }
 
     child_spec = %{
       id: Actors.Actor.Entity,
@@ -73,7 +80,14 @@ defmodule Actors.Actor.Entity.Supervisor do
         %Actor{} = actor,
         opts
       ) do
-    entity_state = %EntityState{system: actor_system, actor: actor, opts: opts}
+    revision = Keyword.get(opts, :revision, 0)
+
+    entity_state = %EntityState{
+      system: actor_system,
+      actor: actor,
+      revision: revision,
+      opts: opts
+    }
 
     child_spec = %{
       id: Actors.Actor.Entity,