diff --git a/.gitignore b/.gitignore index 1e8f52c..7bf766b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .DS_Store *.egg-info/ .tox/ -local -local/ agent/tests/output/* .cache .idea diff --git a/agent/agent.py b/agent/agent.py index fc40ed7..368b5d3 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -4,5 +4,5 @@ app = create_app() -#TODO Make the port configurable from the ENV +# TODO Make the port configurable from the ENV web.run_app(app, port=8000) diff --git a/agent/requirements.txt b/agent/requirements.txt index 44974b8..4be2f53 100644 --- a/agent/requirements.txt +++ b/agent/requirements.txt @@ -13,6 +13,7 @@ pyaml requests pytest +mock pytest-cov coveralls @@ -20,3 +21,10 @@ sphinx sphinx-autobuild sphinx_rtd_theme recommonmark + +msgpack-python +ipaddress +miniupnpc +apscheduler +m2crypto +pynacl diff --git a/agent/sn_agent/app.py b/agent/sn_agent/app.py index e0b7b17..3125608 100644 --- a/agent/sn_agent/app.py +++ b/agent/sn_agent/app.py @@ -6,7 +6,7 @@ from sn_agent.agent import setup_agent from sn_agent.log import setup_logging -from sn_agent.network import setup_network +from sn_agent.network import setup_network, join_network from sn_agent.ontology import setup_ontology from sn_agent.routes import setup_routes from sn_agent.service_adapter import setup_service_manager @@ -27,6 +27,8 @@ def create_app(): setup_service_manager(app) setup_agent(app) + join_network(app) + app['name'] = 'SingularityNET Agent' return app diff --git a/agent/sn_agent/network/__init__.py b/agent/sn_agent/network/__init__.py index fc83620..97912f0 100644 --- a/agent/sn_agent/network/__init__.py +++ b/agent/sn_agent/network/__init__.py @@ -1,9 +1,32 @@ +import logging from sn_agent.network.settings import NetworkSettings from sn_agent.utils import import_string +logger = logging.getLogger(__name__) + def setup_network(app): settings = NetworkSettings() klass = import_string(settings.CLASS) + logger.debug('Loading network class: %s', klass) app['network'] = klass(app) + + +def join_network(app): + app['network'].join_network() + + +class BadMessage(Exception): + """ Raised when a message can't be parsed or a timeout occurs """ + pass + + +class MaxSizeException(Exception): + """ Maximum size of something is reached """ + pass + + +class NetworkError(Exception): + """ Network error """ + pass diff --git a/agent/sn_agent/network/dht/__init__.py b/agent/sn_agent/network/dht/__init__.py index 886c63e..a104fef 100644 --- a/agent/sn_agent/network/dht/__init__.py +++ b/agent/sn_agent/network/dht/__init__.py @@ -1,9 +1,8 @@ import logging -from sn_agent.network.dht.dht import DHT - from sn_agent.agent.base import AgentABC from sn_agent.network.base import NetworkABC +from sn_agent.network.dht.dht import DHT from sn_agent.network.enum import NetworkStatus from sn_agent.ontology.service_descriptor import ServiceDescriptor @@ -13,20 +12,8 @@ class DHTNetwork(NetworkABC): def __init__(self, app): super().__init__(app) - import nacl.signing - - - - host1, port1 = 'localhost', 3000 - dht1 = DHT(key1) - - key2 = nacl.signing.SigningKey.generate() - - host2, port2 = 'localhost', 3001 - dht2 = DHT(host2, port2, key2, boot_host=host1, boot_port=port1) - dht1["test2"] = ["My", "json-serializable", "Object"] - print(dht2["test2"]) + self.dht = DHT() def update_ontology(self): super().update_ontology() @@ -38,7 +25,9 @@ def leave_network(self) -> bool: return super().leave_network() def join_network(self) -> bool: - return super().join_network() + agent = self.app['agent'] + agent_id = str(agent.agent_id) + self.dht[agent_id] = self.dht.my_id def get_network_status(self) -> NetworkStatus: return super().get_network_status() diff --git a/agent/sn_agent/network/dht/dht.py b/agent/sn_agent/network/dht/dht.py index c70d358..28abf85 100644 --- a/agent/sn_agent/network/dht/dht.py +++ b/agent/sn_agent/network/dht/dht.py @@ -6,35 +6,32 @@ import nacl.encoding import nacl.signing -from .handler import DHTRequestHandler -from .server import DHTServer -from .settings import DHTSettings from .bucketset import BucketSet +from .handler import DHTRequestHandler from .hashing import hash_function, random_id from .peer import Peer +from .server import DHTServer +from .settings import DHTSettings from .shortlist import Shortlist logger = logging.getLogger(__name__) class DHT(object): - def __init__(self, host=None, port=None, key=None, my_id=None): + def __init__(self): self.settings = DHTSettings() - if not my_id: - my_id = random_id() + self.my_id = random_id() - if not host: - host = "0.0.0.0" + host = self.settings.HOST - if not port: + port = self.settings.PORT + if port == 0: port = random.randint(5000, 10000) - if not key: - key = nacl.signing.SigningKey.generate() - self.my_key = key + self.my_key = nacl.signing.SigningKey.generate() - self.peer = Peer(host, port, my_id) + self.peer = Peer(host, port, self.my_id) self.data = {} self.buckets = BucketSet(self.settings.K, self.settings.ID_BITS, self.peer.id) self.rpc_ids = {} # should probably have a lock for this @@ -94,6 +91,9 @@ def iterative_find_value(self, key): return shortlist.completion_result() def bootstrap(self, boot_host, boot_port): + + logger.debug("Bootstrapping for %s:%s", boot_host, boot_port) + boot_peer = Peer(boot_host, boot_port, 0) self.iterative_find_nodes(self.peer.id, boot_peer=boot_peer) diff --git a/agent/sn_agent/network/dht/lazymq.py b/agent/sn_agent/network/dht/lazymq.py deleted file mode 100644 index e69de29..0000000 diff --git a/agent/sn_agent/network/dht/server.py b/agent/sn_agent/network/dht/server.py index 472678e..ce66a87 100644 --- a/agent/sn_agent/network/dht/server.py +++ b/agent/sn_agent/network/dht/server.py @@ -8,13 +8,17 @@ class DHTServer(socketserver.ThreadingMixIn, socketserver.UDPServer): def __init__(self, host_address, handler_cls): + + logger.debug("Starting DHT Sever on %s", host_address) + socketserver.UDPServer.__init__(self, host_address, handler_cls) self.send_lock = threading.Lock() - def try_upnp_portmap(self, host_address): - port = host_address[1] + def try_upnp_portmap(self, port): + upnp = miniupnpc.UPnP() upnp.discover() + try: upnp.selectigd() result = upnp.addportmapping(port, 'TCP', upnp.lanaddr, port, 'SN Agent dht port: %u' % port, '', ) diff --git a/agent/sn_agent/network/dht/settings.py b/agent/sn_agent/network/dht/settings.py index fe42cff..f3f4660 100644 --- a/agent/sn_agent/network/dht/settings.py +++ b/agent/sn_agent/network/dht/settings.py @@ -3,6 +3,7 @@ class DHTSettings(SettingsBase): def __init__(self, **custom_settings): + self.ITERATION_SLEEP = 1 self.ID_BITS = 128 self.ALPHA = 3 @@ -12,7 +13,10 @@ def __init__(self, **custom_settings): self.USE_UPNP = True self.NEEDS_BOOTING = True - self.BOOT_HOST = 'sn.jensenbox.com' + self.BOOT_HOST = 'localhost' self.BOOT_PORT = 6881 + self.HOST = '0.0.0.0' + self.PORT = 6881 + super().__init__(**custom_settings) diff --git a/agent/sn_agent/network/settings.py b/agent/sn_agent/network/settings.py index 75e018f..b8cbf00 100644 --- a/agent/sn_agent/network/settings.py +++ b/agent/sn_agent/network/settings.py @@ -4,5 +4,5 @@ class NetworkSettings(SettingsBase): def __init__(self, **custom_settings): self._ENV_PREFIX = 'SN_NETWORK_' - self.CLASS = 'sn_agent.network.test.TestNetwork' + self.CLASS = 'sn_agent.network.dht.DHTNetwork' super().__init__(**custom_settings) diff --git a/docker-compose.yml b/docker-compose.yml index 50508a8..bcfeab9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,23 @@ services: - SN_AGENT_ID=b545478a-971a-48ec-bc56-4b9b7176799c - SN_SERVICE_ADAPTER_CONFIG_FILE=service_adapter_config_example.yml - PYTHONPATH=/code + - SN_DHT_BOOT_HOST=172.17.0.1 + - SN_DHT_PORT=3000 ports: - - "8000:8000" + - "3000:3000/udp" + volumes: + - ./agent-data:/data + + agent2: + build: agent + environment: + - SN_AGENT_ID=c545478a-971a-48ec-bc56-aaaaaaaaaaaa + - SN_SERVICE_ADAPTER_CONFIG_FILE=service_adapter_config_example.yml + - PYTHONPATH=/code + - SN_DHT_NEEDS_BOOTING=False + - SN_DHT_PORT=6881 + ports: + - "6881:6881/udp" volumes: - ./agent-data:/data diff --git a/ipfs/Dockerfile b/ipfs/Dockerfile index b3422ed..f784971 100644 --- a/ipfs/Dockerfile +++ b/ipfs/Dockerfile @@ -1,4 +1,3 @@ -FROM desmart/truffle +FROM ipfs/go-ipfs + -WORKDIR /truffle -ADD . /truffle diff --git a/tools.sh b/tools.sh index d79c079..4591976 100755 --- a/tools.sh +++ b/tools.sh @@ -7,6 +7,7 @@ set -o nounset function recreate_agent_image { docker-compose create --build --force-recreate agent + docker-compose create --build --force-recreate agent2 } case "$1" in @@ -20,6 +21,11 @@ agent) docker-compose run --service-ports agent ./agent.sh run ;; +agent2) + recreate_agent_image + docker-compose run --service-ports agent2 ./agent.sh run + ;; + agent-docs) recreate_agent_image docker-compose run agent ./agent.sh docs @@ -43,11 +49,15 @@ solc) ;; parity) - docker-compose run --service-ports parity parity --help + docker-compose run --service-ports parity ;; truffle) - docker-compose run --service-ports truffle truffle --help + docker-compose run --service-ports truffle + ;; + +ipfs) + docker-compose run --service-ports ipfs daemon ;; clean) @@ -66,7 +76,13 @@ create-web-cookie) docker-compose run agent-web-cookie ;; - +gen-ssl) + openssl genrsa -des3 -passout pass:x -out server.pass.key 2048 + openssl rsa -passin pass:x -in server.pass.key -out server.key + rm server.pass.key + openssl req -new -key server.key -out server.csr -subj "/C=UK/ST=Warwickshire/L=Leamington/O=OrgName/OU=IT Department/CN=example.com" + openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt + ;; *) echo 'No operation specified' exit 0;