Initial version of the DHT

.gitignore

@@ -1,8 +1,6 @@
 .DS_Store
 *.egg-info/
 .tox/
-local
-local/
 agent/tests/output/*
 .cache
 .idea

agent/agent.py

@@ -4,5 +4,5 @@
 
 app = create_app()
 
-#TODO Make the port configurable from the ENV
+# TODO Make the port configurable from the ENV
 web.run_app(app, port=8000)

agent/requirements.txt

@@ -13,10 +13,18 @@ pyaml
 requests
 
 pytest
+mock
 pytest-cov
 coveralls
 
 sphinx
 sphinx-autobuild
 sphinx_rtd_theme
 recommonmark
+
+msgpack-python
+ipaddress
+miniupnpc
+apscheduler
+m2crypto
+pynacl

agent/sn_agent/app.py

@@ -6,7 +6,7 @@
 
 from sn_agent.agent import setup_agent
 from sn_agent.log import setup_logging
-from sn_agent.network import setup_network
+from sn_agent.network import setup_network, join_network
 from sn_agent.ontology import setup_ontology
 from sn_agent.routes import setup_routes
 from sn_agent.service_adapter import setup_service_manager
@@ -27,6 +27,8 @@ def create_app():
     setup_service_manager(app)
     setup_agent(app)
 
+    join_network(app)
+
     app['name'] = 'SingularityNET Agent'
 
     return app

agent/sn_agent/network/__init__.py

@@ -1,9 +1,32 @@
+import logging
 
 from sn_agent.network.settings import NetworkSettings
 from sn_agent.utils import import_string
 
+logger = logging.getLogger(__name__)
+
 
 def setup_network(app):
     settings = NetworkSettings()
     klass = import_string(settings.CLASS)
+    logger.debug('Loading network class: %s', klass)
     app['network'] = klass(app)
+
+
+def join_network(app):
+    app['network'].join_network()
+
+
+class BadMessage(Exception):
+    """ Raised when a message can't be parsed or a timeout occurs """
+    pass
+
+
+class MaxSizeException(Exception):
+    """ Maximum size of something is reached """
+    pass
+
+
+class NetworkError(Exception):
+    """ Network error """
+    pass

agent/sn_agent/network/dht/__init__.py

@@ -1,9 +1,8 @@
 import logging
 
-from sn_agent.network.dht.dht import DHT
-
 from sn_agent.agent.base import AgentABC
 from sn_agent.network.base import NetworkABC
+from sn_agent.network.dht.dht import DHT
 from sn_agent.network.enum import NetworkStatus
 from sn_agent.ontology.service_descriptor import ServiceDescriptor
 
@@ -13,20 +12,8 @@
 class DHTNetwork(NetworkABC):
     def __init__(self, app):
         super().__init__(app)
-        import nacl.signing
-
-
-
-        host1, port1 = 'localhost', 3000
-        dht1 = DHT(key1)
-
-        key2 = nacl.signing.SigningKey.generate()
-
-        host2, port2 = 'localhost', 3001
-        dht2 = DHT(host2, port2, key2, boot_host=host1, boot_port=port1)
 
-        dht1["test2"] = ["My", "json-serializable", "Object"]
-        print(dht2["test2"])
+        self.dht = DHT()
 
     def update_ontology(self):
         super().update_ontology()
@@ -38,7 +25,9 @@ def leave_network(self) -> bool:
         return super().leave_network()
 
     def join_network(self) -> bool:
-        return super().join_network()
+        agent = self.app['agent']
+        agent_id = str(agent.agent_id)
+        self.dht[agent_id] = self.dht.my_id
 
     def get_network_status(self) -> NetworkStatus:
         return super().get_network_status()

agent/sn_agent/network/dht/dht.py

@@ -6,35 +6,32 @@
 import nacl.encoding
 import nacl.signing
 
-from .handler import DHTRequestHandler
-from .server import DHTServer
-from .settings import DHTSettings
 from .bucketset import BucketSet
+from .handler import DHTRequestHandler
 from .hashing import hash_function, random_id
 from .peer import Peer
+from .server import DHTServer
+from .settings import DHTSettings
 from .shortlist import Shortlist
 
 logger = logging.getLogger(__name__)
 
 
 class DHT(object):
-    def __init__(self, host=None, port=None, key=None, my_id=None):
+    def __init__(self):
         self.settings = DHTSettings()
 
-        if not my_id:
-            my_id = random_id()
+        self.my_id = random_id()
 
-        if not host:
-            host = "0.0.0.0"
+        host = self.settings.HOST
 
-        if not port:
+        port = self.settings.PORT
+        if port == 0:
             port = random.randint(5000, 10000)
 
-        if not key:
-            key = nacl.signing.SigningKey.generate()
-        self.my_key = key
+        self.my_key = nacl.signing.SigningKey.generate()
 
-        self.peer = Peer(host, port, my_id)
+        self.peer = Peer(host, port, self.my_id)
         self.data = {}
         self.buckets = BucketSet(self.settings.K, self.settings.ID_BITS, self.peer.id)
         self.rpc_ids = {}  # should probably have a lock for this
@@ -94,6 +91,9 @@ def iterative_find_value(self, key):
         return shortlist.completion_result()
 
     def bootstrap(self, boot_host, boot_port):
+
+        logger.debug("Bootstrapping for %s:%s", boot_host, boot_port)
+
         boot_peer = Peer(boot_host, boot_port, 0)
         self.iterative_find_nodes(self.peer.id, boot_peer=boot_peer)
 

agent/sn_agent/network/dht/server.py

@@ -8,13 +8,17 @@
 
 class DHTServer(socketserver.ThreadingMixIn, socketserver.UDPServer):
     def __init__(self, host_address, handler_cls):
+
+        logger.debug("Starting DHT Sever on %s", host_address)
+
         socketserver.UDPServer.__init__(self, host_address, handler_cls)
         self.send_lock = threading.Lock()
 
-    def try_upnp_portmap(self, host_address):
-        port = host_address[1]
+    def try_upnp_portmap(self, port):
+
         upnp = miniupnpc.UPnP()
         upnp.discover()
+
         try:
             upnp.selectigd()
             result = upnp.addportmapping(port, 'TCP', upnp.lanaddr, port, 'SN Agent dht port: %u' % port, '', )

agent/sn_agent/network/dht/settings.py

@@ -3,6 +3,7 @@
 
 class DHTSettings(SettingsBase):
     def __init__(self, **custom_settings):
+
         self.ITERATION_SLEEP = 1
         self.ID_BITS = 128
         self.ALPHA = 3
@@ -12,7 +13,10 @@ def __init__(self, **custom_settings):
         self.USE_UPNP = True
 
         self.NEEDS_BOOTING = True
-        self.BOOT_HOST = 'sn.jensenbox.com'
+        self.BOOT_HOST = 'localhost'
         self.BOOT_PORT = 6881
 
+        self.HOST = '0.0.0.0'
+        self.PORT = 6881
+
         super().__init__(**custom_settings)

agent/sn_agent/network/settings.py

@@ -4,5 +4,5 @@
 class NetworkSettings(SettingsBase):
     def __init__(self, **custom_settings):
         self._ENV_PREFIX = 'SN_NETWORK_'
-        self.CLASS = 'sn_agent.network.test.TestNetwork'
+        self.CLASS = 'sn_agent.network.dht.DHTNetwork'
         super().__init__(**custom_settings)

docker-compose.yml

@@ -9,8 +9,23 @@ services:
         - SN_AGENT_ID=b545478a-971a-48ec-bc56-4b9b7176799c
         - SN_SERVICE_ADAPTER_CONFIG_FILE=service_adapter_config_example.yml
         - PYTHONPATH=/code
+        - SN_DHT_BOOT_HOST=172.17.0.1
+        - SN_DHT_PORT=3000
       ports:
-        - "8000:8000"
+        - "3000:3000/udp"
+      volumes:
+        - ./agent-data:/data
+
+    agent2:
+      build: agent
+      environment:
+        - SN_AGENT_ID=c545478a-971a-48ec-bc56-aaaaaaaaaaaa
+        - SN_SERVICE_ADAPTER_CONFIG_FILE=service_adapter_config_example.yml
+        - PYTHONPATH=/code
+        - SN_DHT_NEEDS_BOOTING=False
+        - SN_DHT_PORT=6881
+      ports:
+        - "6881:6881/udp"
       volumes:
         - ./agent-data:/data
 

ipfs/Dockerfile

@@ -1,4 +1,3 @@
-FROM desmart/truffle
+FROM ipfs/go-ipfs
+
 
-WORKDIR /truffle
-ADD . /truffle

tools.sh

@@ -7,6 +7,7 @@ set -o nounset
 
 function recreate_agent_image {
     docker-compose create --build --force-recreate agent
+    docker-compose create --build --force-recreate agent2
 }
 
 case "$1" in
@@ -20,6 +21,11 @@ agent)
     docker-compose run --service-ports agent ./agent.sh run
     ;;
 
+agent2)
+    recreate_agent_image
+    docker-compose run --service-ports agent2 ./agent.sh run
+    ;;
+
 agent-docs)
     recreate_agent_image
     docker-compose run agent ./agent.sh docs
@@ -43,11 +49,15 @@ solc)
     ;;
 
 parity)
-    docker-compose run --service-ports parity parity --help
+    docker-compose run --service-ports parity
     ;;
 
 truffle)
-    docker-compose run --service-ports truffle truffle --help
+    docker-compose run --service-ports truffle
+    ;;
+
+ipfs)
+    docker-compose run --service-ports ipfs daemon
     ;;
 
 clean)
@@ -66,7 +76,13 @@ create-web-cookie)
     docker-compose run agent-web-cookie
     ;;
 
-
+gen-ssl)
+    openssl genrsa -des3 -passout pass:x -out server.pass.key 2048
+    openssl rsa -passin pass:x -in server.pass.key -out server.key
+    rm server.pass.key
+    openssl req -new -key server.key -out server.csr -subj "/C=UK/ST=Warwickshire/L=Leamington/O=OrgName/OU=IT Department/CN=example.com"
+    openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+    ;;
 
 *) echo 'No operation specified'
     exit 0;