Experimental Updated to Main

.env.example

@@ -0,0 +1,4 @@
+TOKEN=<token>
+RAVENVERSE_URL=http://0.0.0.0:8081
+RAVENVERSE_FTP_URL=0.0.0.0
+RAVENAUTH_URL=http://0.0.0.0:8000

.gitignore

@@ -1,2 +1,3 @@
 .idea
 .DS_Store
+.env

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.7.11
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+COPY . /
+
+RUN python -m ensurepip --upgrade
+RUN python -m pip install --upgrade pip
+
+RUN pip install -r requirements.txt
+
+CMD ["run_distributed_client.py"]
+ENTRYPOINT ["python"]

docker-compose.yml

@@ -0,0 +1,4 @@
+version: '3.3'
+services:
+  ravsock:
+    build: .

gui.py

@@ -0,0 +1,136 @@
+import eel
+import logging
+import os
+import psutil
+import shutil
+from hurry.filesize import size
+
+os.environ['RAVENVERSE_URL'] = "http://server.ravenverse.ai"
+os.environ['RAVENVERSE_FTP_HOST'] = "server.ravenverse.ai"
+os.environ['RAVENVERSE_FTP_URL'] = "server.ravenverse.ai"
+os.environ['RAVENAUTH_URL'] = "https://auth.ravenverse.ai"
+
+eel.init('web')
+
+
+@eel.expose
+def disconnect():
+    if g.client.connected:
+        g.logger.debug("Disconnecting...")
+        if g.client.connected:
+            g.client.emit("disconnect", namespace="/client")
+            g.logger.debug("Disconnected")
+            g.logger.debug("")
+
+    return True
+
+
+def close_callback(a, b):
+    disconnect()
+
+
+from ravpy.globals import g
+
+
+class CustomHandler(logging.Handler):
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+    def emit(self, record):
+        print("Custom", record)
+        eel.getLog({"asctime": record.asctime, "threadName": record.threadName, "levelname": record.levelname,
+                    "message": record.message})
+        return record
+
+
+log_formatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
+my_handler = CustomHandler()
+my_handler.setLevel(logging.DEBUG)
+my_handler.setFormatter(log_formatter)
+g.logger.addHandler(my_handler)
+
+
+@eel.expose
+def verify_access_token(access_token):
+    from ravpy.utils import verify_token
+    if verify_token(access_token):
+        return [access_token, "success", ""]
+    else:
+        return [access_token, "failure", "Invalid access token!"]
+
+
+@eel.expose
+def get_system_config():
+    ram_total = str(size(psutil.virtual_memory().total))
+    ram_available = str(size(psutil.virtual_memory().available))
+    cpu_count = psutil.cpu_count(logical=False)
+    cpu_percent = psutil.cpu_percent()
+    total, used, free = shutil.disk_usage("/")
+    storage_total = total // (2 ** 30)
+    storage_available = free // (2 ** 30)
+
+    return {"ram_total": ram_total, "ram_available": ram_available,
+            "cpu_count": cpu_count, "cpu_percent": cpu_percent, "storage_total": storage_total,
+            "storage_available": storage_available}
+
+
+@eel.expose
+def get_logs(skip, limit):
+    with open("debug.log", "r") as f:
+        logs = f.readlines()[skip:]
+        print(logs)
+
+
+@eel.expose
+def participate(token):
+    from ravpy.distributed.benchmarking import benchmark
+    from ravpy.utils import initialize_ftp_client
+    from ravpy.initialize import initialize
+
+    # Initialize
+    socket_client = initialize(ravenverse_token=token)
+
+    if socket_client is None:
+        disconnect()
+        eel.clientDisconnected()
+        return False
+    else:
+        eel.clientConnected()
+
+    # get ftp client
+    ftp_client = initialize_ftp_client()
+
+    if ftp_client is None:
+        disconnect()
+        eel.clientDisconnected()
+        return False
+
+    # do benchmark
+    benchmark()
+
+    g.logger.debug("")
+    g.logger.debug("Ravpy is waiting for graphs/subgraphs/ops...")
+    g.logger.debug("Warning: Do not close Ravpy if you like to "
+                   "keep participating and keep earning Raven tokens\n")
+
+    return True
+
+
+@eel.expose
+def get_subgraphs():
+    subgraphs = g.ravdb.get_subgraphs()
+    subgraphs = [sg.as_dict() for sg in subgraphs]
+    return subgraphs
+
+
+@eel.expose
+def delete_subgraphs():
+    g.ravdb.delete_subgraphs()
+    return True
+
+
+g.ravdb.create_database()
+g.ravdb.create_tables()
+
+
+eel.start('main.html', close_callback=close_callback)

ravpy/__init__.py

@@ -0,0 +1,2 @@
+from .initialize import initialize
+from .distributed.participate import participate

ravpy/config.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 
 BASE_DIR = os.path.join(str(Path.home()), "ravenverse/ravpy")
-
+PROJECT_DIR = pathlib.Path(__file__).parent.parent.resolve()
 CONTEXT_FOLDER = os.path.join(BASE_DIR, "contexts")
 PARAMS_DIR = os.path.join(BASE_DIR, "params")
 
@@ -13,12 +13,21 @@
 RAVENVERSE_URL = os.environ.get("RAVENVERSE_URL")
 RAVENVERSE_FTP_URL = os.environ.get("RAVENVERSE_FTP_URL")
 
-BENCHMARK_FILE_NAME = "ravpy/distributed/benchmark.json"
+BENCHMARK_FILE_NAME = os.path.join(PROJECT_DIR, "ravpy/distributed/benchmark.json")
 TYPE = "client"
 
 ENCRYPTION = False
 
-FTP_TEMP_FILES_FOLDER = os.path.join(os.getcwd(), "ravpy/distributed/temp_files")
-FTP_DOWNLOAD_FILES_FOLDER = os.path.join(os.getcwd(), "ravpy/distributed/downloads")
+FTP_TEMP_FILES_FOLDER = os.path.join(PROJECT_DIR, "ravpy/distributed/temp_files")
+FTP_DOWNLOAD_FILES_FOLDER = os.path.join(PROJECT_DIR, "ravpy/distributed/downloads")
+
+os.makedirs(FTP_TEMP_FILES_FOLDER, exist_ok=True)
+os.makedirs(FTP_DOWNLOAD_FILES_FOLDER, exist_ok=True)
+
+RAVPY_LOG_FILE = os.path.join(PROJECT_DIR, "debug.log")
+
+BENCHMARK_DOWNLOAD_PATH = os.path.join(PROJECT_DIR, "ravpy/distributed/downloads/")
+TEMP_FILES_PATH = os.path.join(PROJECT_DIR, "ravpy/distributed/temp_files/")
+RAVENAUTH_TOKEN_VERIFY_URL = "{}{}".format(os.environ.get("RAVENAUTH_URL"), "/api/token/verify/")
 
-RAVPY_LOG_FILE = os.path.join(pathlib.Path(__file__).parent.parent.resolve(), "debug.log")
+DATABASE_URI = "sqlite:///{}/{}".format(PROJECT_DIR, "database.db")

ravpy/db/__init__.py

@@ -0,0 +1,2 @@
+from .models import Subgraph
+from .manager import DBManager

ravpy/db/manager.py

@@ -0,0 +1,122 @@
+import sqlalchemy
+import sqlalchemy as db
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy_utils import create_database as cd
+from sqlalchemy_utils import database_exists, get_tables
+from sqlalchemy_utils import drop_database as dba
+
+from .models import Base, Subgraph
+from ..config import DATABASE_URI
+
+
+class DBManager:
+    def __init__(self):
+        self.create_database()
+        self.engine = self.connect()
+        self.logger = None
+
+    def set_logger(self, logger):
+        self.logger = logger
+
+    def get_session(self):
+        Session = sessionmaker(bind=self.engine, expire_on_commit=False)
+        return Session
+
+    def connect(self):
+        engine = db.create_engine(DATABASE_URI, isolation_level='READ UNCOMMITTED')
+        Base.metadata.bind = engine
+        return engine
+
+    def create_database(self):
+        if not database_exists(DATABASE_URI):
+            cd(DATABASE_URI)
+            print('Database created')
+
+    def drop_database(self):
+        if database_exists(DATABASE_URI):
+            dba(DATABASE_URI)
+            print('Database dropped')
+
+    def create_tables(self):
+        """
+        Create tables
+        """
+        Base.metadata.create_all(self.engine, checkfirst=True)
+
+    def add_subgraph(self, **kwargs):
+        """
+        Create a subgraph and add values
+        :param kwargs: subgraph details
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+
+            subgraph = self.find_subgraph(graph_id=kwargs['graph_id'], subgraph_id=kwargs['subgraph_id'])
+            if subgraph is None:
+                # create new subgraph
+                subgraph = Subgraph()
+                for key, value in kwargs.items():
+                    setattr(subgraph, key, value)
+                session.add(subgraph)
+                self.logger.debug("Subgraph created")
+            else:
+                self.logger.debug("Subgraph available")
+            return subgraph
+
+    def find_subgraph(self, graph_id, subgraph_id):
+        """
+        Find a subgraph
+        :param graph_id: Graph id
+        :param subgraph_id: subgraph id
+        :return: subgraph object
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+            subgraph = (
+                session.query(Subgraph).filter(
+                    Subgraph.graph_id == graph_id, Subgraph.subgraph_id == subgraph_id,
+                ).first()
+            )
+            return subgraph
+
+    def update_subgraph(self, subgraph, **kwargs):
+        """
+        Update a subgraph
+        :param subgraph: subgraph object
+        :param kwargs: details
+        :return: updated subgraph object
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+            for key, value in kwargs.items():
+                setattr(subgraph, key, value)
+            session.add(subgraph)
+            return subgraph
+
+    def delete_subgraph(self, obj):
+        """
+        Delete subgraph object
+        :param obj: subgraph object
+        :return: None
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+            session.delete(obj)
+
+    def get_subgraphs(self):
+        """
+        Fetch all subgraphs
+        :return: list of subgraphs
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+            return session.query(Subgraph).order_by(Subgraph.created_at.desc()).all()
+
+    def delete_subgraphs(self):
+        """
+        Delete all subgraphs
+        :return: None
+        """
+        Session = self.get_session()
+        with Session.begin() as session:
+            session.query(Subgraph).delete()

ravpy/db/models.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import datetime
+import sqlalchemy as sa
+from sqlalchemy import Column
+from sqlalchemy import DateTime
+from sqlalchemy import Float
+from sqlalchemy import Integer
+from sqlalchemy import String
+from sqlalchemy import orm
+
+metadata = sa.MetaData()
+Base = orm.declarative_base(metadata=metadata)
+
+
+class Subgraph(Base):
+    __tablename__ = 'subgraphs'
+    id = Column(Integer, primary_key=True)
+    graph_id = Column(Integer, nullable=False)
+    subgraph_id = Column(Integer, nullable=False)
+    status = Column(String(50), nullable=True, default=None)
+    progress = Column(Float, nullable=True, default=None)
+    tokens = Column(Float, nullable=True, default=None)
+    created_at = Column(DateTime, default=datetime.datetime.utcnow)
+
+    def as_dict(self):
+        values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
+        values['created_at'] = values['created_at'].strftime("%Y-%m-%d %H:%M:%S")
+        return values