From fff5f4ce469f68d51a54c5907e3f12a6627959cc Mon Sep 17 00:00:00 2001 From: Christian Jensen Date: Wed, 29 Nov 2017 18:12:32 -0800 Subject: [PATCH] Testing various password storage techniques. --- agent/agent.py | 25 ---------- agent/agent.sh | 8 ++- agent/demo_config.yml | 17 +++++++ agent/requirements.txt | 8 +-- agent/sn_agent/cli.py | 76 +++++++++++++++++++++++++++++ agent/sn_agent/network/base.py | 7 +++ agent/sn_agent/network/settings.py | 6 ++- agent/sn_agent/network/sn.py | 58 +++++++++++++++------- agent/sn_agent/version.py | 10 ++++ data/vault-data/config/standard.hcl | 8 +++ docker-compose.demo.yml | 2 - docker-compose.yml | 64 ++++++++++++++++++++++++ geth/Dockerfile | 26 ++++++---- geth/config.toml | 16 ++++++ geth/geth.sh | 24 +++++++++ geth/password.txt | 1 + openDHT/Dockerfile | 68 ++++++++++++++++++++++++++ parity/Dockerfile | 8 ++- parity/config.toml | 7 +++ parity/parity.sh | 23 +++++++++ tools.sh | 23 +++++++-- vault/Dockerfile | 3 ++ 22 files changed, 425 insertions(+), 63 deletions(-) delete mode 100644 agent/agent.py create mode 100644 agent/demo_config.yml create mode 100755 agent/sn_agent/cli.py create mode 100644 agent/sn_agent/version.py create mode 100644 data/vault-data/config/standard.hcl create mode 100644 geth/config.toml create mode 100755 geth/geth.sh create mode 100644 geth/password.txt create mode 100644 openDHT/Dockerfile create mode 100644 parity/config.toml create mode 100755 parity/parity.sh create mode 100644 vault/Dockerfile diff --git a/agent/agent.py b/agent/agent.py deleted file mode 100644 index 574ff4a..0000000 --- a/agent/agent.py +++ /dev/null @@ -1,25 +0,0 @@ -from aiohttp import web - -from sn_agent.agent import AgentSettings -from sn_agent.app import create_app -import ssl - -import logging - -from sn_agent.network import NetworkSettings - -logger = logging.getLogger(__name__) - -app = create_app() - -# sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) -# sslcontext.load_cert_chain('server.crt', 'server.key') - -# TODO Make the port configurable from the ENV -# web.run_app(app, port=8000, ssl_context=sslcontext) - -settings = NetworkSettings() - -logger.info('Host setting: %s', settings.WEB_HOST) - -web.run_app(app, port=settings.WEB_PORT) diff --git a/agent/agent.sh b/agent/agent.sh index 7bd7fee..1319682 100755 --- a/agent/agent.sh +++ b/agent/agent.sh @@ -11,8 +11,14 @@ case "$1" in noop) ;; +vault) + export PYTHONPATH=/code + python3 sn_agent/cli.py vault + ;; + run) - python3 agent.py + export PYTHONPATH=/code + python3 sn_agent/cli.py run ;; docs) diff --git a/agent/demo_config.yml b/agent/demo_config.yml new file mode 100644 index 0000000..75bf7f2 --- /dev/null +++ b/agent/demo_config.yml @@ -0,0 +1,17 @@ +--- + +services: + +- service: deadbeef-aaaa-bbbb-cccc-000000000101 + module: examples.simple_adapter.SimpleAdapter + +- service: deadbeef-aaaa-bbbb-cccc-111111111101 + module: adapters.tensorflow.mnist.TensorflowMNIST + +- service: deadbeef-aaaa-bbbb-cccc-000000000202 + module: adapters.opencog.relex.RelexAdapter + +- service: deadbeef-aaaa-bbbb-cccc-100000000001 + module: adapters.aigents.AigentsAdapter + + diff --git a/agent/requirements.txt b/agent/requirements.txt index 86badb2..a7db188 100644 --- a/agent/requirements.txt +++ b/agent/requirements.txt @@ -9,14 +9,13 @@ jsonrpcserver rdflib pyaml -# For some reason jsonrpcclient is using this even though we are using the aiohttp implementation - perhaps under the covers? -requests - +# Testing pytest mock pytest-cov coveralls +# Documentation sphinx sphinx-autobuild sphinx_rtd_theme @@ -29,3 +28,6 @@ tensorflow web3 requests + +fire +hvac diff --git a/agent/sn_agent/cli.py b/agent/sn_agent/cli.py new file mode 100755 index 0000000..2246a07 --- /dev/null +++ b/agent/sn_agent/cli.py @@ -0,0 +1,76 @@ +import ssl +from getpass import getpass + +import fire +import sys + +from aiohttp import web + +from sn_agent.app import create_app +from sn_agent.network import NetworkSettings + +import logging + +import hvac +import os + +logger = logging.getLogger(__name__) + + +class Agent(object): + def run(self): + network_settings = NetworkSettings() + + app = create_app() + + sslcontext = None + + if network_settings.SSL_CERTIFICATE_FILE and network_settings.SSL_KEYFILE: + sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext.load_cert_chain('server.crt', 'server.key') + + logger.info('Host setting: %s', network_settings.WEB_HOST) + + web.run_app(app, port=network_settings.WEB_PORT, ssl_context=sslcontext) + + def vault_init(self, client): + + shares = 1 + threshold = 1 + + result = client.initialize(shares, threshold) + + print("The following is extremely important. It will never be shown again.") + print("-------------------------------------------------------------------") + print(result) + print("-------------------------------------------------------------------") + + def unseal(self, client): + + while True: + key = getpass() + client.unseal(key) + if not client.is_sealed(): + break + print("Try again") + + def vault(self): + + client = hvac.Client(url='http://vault:8200') + + if not client.is_initialized(): + self.vault_init(client) + return + + if client.is_sealed(): + self.unseal(client) + + client = hvac.Client(url='http://vault:8200', token=os.environ['VAULT_TOKEN']) + client.write('secret/foo', baz='bar', lease='1h') + print(client.read('secret/foo')) + client.delete('secret/foo') + +#| {'keys': ['86dfb323fc678f993af9d8876209297e56d6ab880dc056944f635b7949c816ca'], 'keys_base64': ['ht+zI/xnj5k6+diHYgkpflbWq4gNwFaUT2NbeUnIFso='], 'root_token': '71e608b7-ae2f-0c80-bb99-d1a4848cf549'} + +if __name__ == '__main__': + fire.Fire(Agent) diff --git a/agent/sn_agent/network/base.py b/agent/sn_agent/network/base.py index 8dc2131..f9a8ad4 100644 --- a/agent/sn_agent/network/base.py +++ b/agent/sn_agent/network/base.py @@ -16,6 +16,13 @@ from sn_agent.ontology.service_descriptor import ServiceDescriptor +class ResolverABC(ABC): + @abstractmethod + def resolve(self, agent_id): + """Resolve the url for an agent given only its ID""" + pass + + class NetworkABC(ABC): def __init__(self, app): self.app = app diff --git a/agent/sn_agent/network/settings.py b/agent/sn_agent/network/settings.py index 0430a4e..f95cf70 100644 --- a/agent/sn_agent/network/settings.py +++ b/agent/sn_agent/network/settings.py @@ -10,6 +10,8 @@ class NetworkSettings(SettingsBase): def __init__(self, **custom_settings): + + self._ENV_PREFIX = 'SN_NETWORK_' self.GATEWAY = '0.0.0.0' @@ -21,8 +23,10 @@ def __init__(self, **custom_settings): self.WEB_HOST = "0.0.0.0" self.WEB_PORT = 8000 + self.SSL_CERTIFICATE_FILE = None + self.SSL_KEYFILE = None + self.AGENT_URL_LOOKUP_FILE = os.path.join(THIS_DIR, 'data', 'agent_to_url_lookup.json') - self.COINBASE = '0x633a490e1d3022a90e49cfb79ff8789d264ae753' super().__init__(**custom_settings) diff --git a/agent/sn_agent/network/sn.py b/agent/sn_agent/network/sn.py index c129f16..e9001c5 100644 --- a/agent/sn_agent/network/sn.py +++ b/agent/sn_agent/network/sn.py @@ -7,13 +7,35 @@ from sn_agent.agent.base import AgentABC from sn_agent.network import NetworkSettings -from sn_agent.network.base import NetworkABC +from sn_agent.network.base import NetworkABC, ResolverABC from sn_agent.network.enum import NetworkStatus from sn_agent.ontology.service_descriptor import ServiceDescriptor logger = logging.getLogger(__name__) +class UnresolvedAgentException(Exception): + pass + + +class FileResolver(ResolverABC): + def __init__(self, lookup_file): + self.lookup_file = lookup_file + + def resolve(self, agent_id): + filename = self.lookup_file + + with open(filename, encoding='utf-8') as data_file: + agent_urls = json.loads(data_file.read()) + + return agent_urls.get(agent_id) + + +class DHTResolver(ResolverABC): + def resolve(self, agent_id): + return None + + class SNNetwork(NetworkABC): def __init__(self, app): super().__init__(app) @@ -23,17 +45,24 @@ def __init__(self, app): self.payload = None self.agent = None + self.resolvers = [] + self.resolvers.append(FileResolver(self.settings.AGENT_URL_LOOKUP_FILE)) + self.resolvers.append(DHTResolver()) async def startup(self): - logger.debug('Registering agent on DHT') + logger.debug('Starting up the network') self.agent = self.app['agent'] try: + account_address = self.client_connection.eth.coinbase + + logger.debug('Using account: %s', account_address) + self.payload = { - 'from': self.client_connection.eth.coinbase, + 'from': account_address, 'gas': 1500000, - 'gasPrice': 30000000000000 + 'gasPrice': 30000000 } current_block = self.client_connection.eth.blockNumber logger.debug('Current client blocknumber: %s', current_block) @@ -46,8 +75,8 @@ async def startup(self): # Implemented methods def join_network(self): logger.debug('Joining Network') - contract = self.get_agent_factory_contract() - contract.transact(self.payload).create() + # contract = self.get_agent_factory_contract() + # contract.transact(self.payload).create() logger.debug('Joined network') def advertise_service(self, service: ServiceDescriptor): @@ -66,19 +95,12 @@ def find_service_providers(self, service: ServiceDescriptor) -> list: def get_url_for_agent(self, agent_id): - filename = self.settings.AGENT_URL_LOOKUP_FILE - - with open(filename, encoding='utf-8') as data_file: - agent_urls = json.loads(data_file.read()) - - url = agent_urls.get(agent_id) - - if url is None: - # Fallback to blockchain if none specified in the lookup file - blockchain_result = None - # TODO implement grabbing from blockchain + for resolver in self.resolvers: + agent_url = resolver.resolve(agent_id) + if agent_url: + return agent_url - return url + raise UnresolvedAgentException(agent_id) # TODO: Unimplemented methods diff --git a/agent/sn_agent/version.py b/agent/sn_agent/version.py new file mode 100644 index 0000000..b2c5466 --- /dev/null +++ b/agent/sn_agent/version.py @@ -0,0 +1,10 @@ + +#: MAJOR, MINOR, RELEASE, STATUS [alpha, beta, final], VERSION +VERSION = (0, 0, 0, 'alpha', 0) + + +def get_version(): + """ + Returns a string representation of the version information of this project. + """ + return '.'.join([str(i) for i in VERSION]) diff --git a/data/vault-data/config/standard.hcl b/data/vault-data/config/standard.hcl new file mode 100644 index 0000000..9e1b627 --- /dev/null +++ b/data/vault-data/config/standard.hcl @@ -0,0 +1,8 @@ +backend "file" { + path = "/vault/file" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 +} diff --git a/docker-compose.demo.yml b/docker-compose.demo.yml index fe79876..ac8a237 100644 --- a/docker-compose.demo.yml +++ b/docker-compose.demo.yml @@ -22,8 +22,6 @@ services: - SN_WEB_COOKIE_SECRET=kubr6DvIuYj4GREdgXq5CCoL5qHQWglj8IECTsI79mY= - PYTHONPATH=/code - - parity: build: parity volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 2827ca3..ba042a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,26 @@ version: "3" services: + demo: + build: agent + environment: + - VAULT_TOKEN=71e608b7-ae2f-0c80-bb99-d1a4848cf549 + - SN_AGENT_ID=b545478a-FFFF-FFFF-FFFF-4b9b7176FFFF + - SN_NETWORK_WEB_PORT=9250 + - SN_SERVICE_ADAPTER_CONFIG_FILE=demo_config.yml + - PYTHONPATH=/code + - SN_DS_TEST_OUTPUT_DIRECTORY=/data + - SN_NETWORK_CLIENT_URL=http://parity:8545 + - SN_NETWORK_ACCOUNT=0x1934ab10860362eabcfaa111847bc75ecf43fcbd + ports: + - "9250:9250" + volumes: + - ./agent-data:/data + links: + - vault +# - parity +# - relex + alice: build: agent environment: @@ -49,6 +69,50 @@ services: links: - testrpc + opendht: + build: openDHT + ports: + - "4222:4222" + - "9100:9100" + + geth: + build: geth + volumes: + - ./data/geth-data:/geth-data + ports: + - "8080:8080" + - "8180:8180" + - "8545:8545" + - "8546:8546" + - "30303:30303" + - "30303:30303/udp" + - "30304:30304/udp" + + parity: + build: parity + ports: + - "8080:8080" + - "8180:8180" + - "8545:8545" + - "8546:8546" + - "30303:30303" + - "30303:30303/udp" + + volumes: + - ./data/parity-data:/parity-data + + vault: + build: vault + hostname: vault + ports: + - "8200:8200" + volumes: + - ./data/vault-data/file:/vault/file + - ./data/vault-data/config:/vault/config + - ./data/vault-data/logs:/vault/logs + cap_add: + - IPC_LOCK + test: build: agent environment: diff --git a/geth/Dockerfile b/geth/Dockerfile index bb08ccc..e2832a0 100644 --- a/geth/Dockerfile +++ b/geth/Dockerfile @@ -1,13 +1,21 @@ -FROM ubuntu:xenial +FROM ubuntu:17.04 ENV DEBIAN_FRONTEND noninteractive RUN \ -apt-get update && \ -apt-get -y -qq upgrade && \ -apt-get -y -qq install software-properties-common && \ -add-apt-repository ppa:ethereum/ethereum && \ -apt-get update && \ -apt-get -y -qq install geth solc && \ -apt-get clean && \ -rm -rf /var/lib/apt/lists/* + apt-get update && \ + apt-get -y -qq upgrade && \ + apt-get -y -qq install software-properties-common && \ + add-apt-repository ppa:ethereum/ethereum && \ + apt-get update && \ + apt-get -y -qq install geth solc && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /geth +COPY geth.sh . +COPY config.toml . +COPY password.txt . + +ENTRYPOINT ["/geth/geth.sh"] +CMD ["run"] diff --git a/geth/config.toml b/geth/config.toml new file mode 100644 index 0000000..c147ec2 --- /dev/null +++ b/geth/config.toml @@ -0,0 +1,16 @@ + +[Eth] +NetworkId = 3 +SyncMode = "fast" + +[Node] +DataDir = "/geth-data" +HTTPHost = "0.0.0.0" +HTTPModules = ["eth", "web3"] +WSModules = ["net", "web3", "eth", "shh"] + + +[Dashboard] +Host = "0.0.0.0" +Port = 8080 +Refresh = 3000000000 diff --git a/geth/geth.sh b/geth/geth.sh new file mode 100755 index 0000000..e424552 --- /dev/null +++ b/geth/geth.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +case "$1" in + +noop) + ;; + +create-account) + geth --config config.toml account new + ;; + +run) + geth --rpcapi admin,eth,web3,personal,net,shh, + db --rpc --config config.toml --password password.txt --unlock 0 + ;; + +*) echo 'No operation specified' + exit 0; + ;; + +esac diff --git a/geth/password.txt b/geth/password.txt new file mode 100644 index 0000000..d781113 --- /dev/null +++ b/geth/password.txt @@ -0,0 +1 @@ +abcdef1234 diff --git a/openDHT/Dockerfile b/openDHT/Dockerfile new file mode 100644 index 0000000..6387a71 --- /dev/null +++ b/openDHT/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:17.04 + +RUN \ + apt-get update && \ + apt-get -y -qq upgrade && \ + apt-get -y -qq install \ + build-essential \ + cmake \ + git \ + wget \ + libncurses5-dev \ + libreadline-dev \ + nettle-dev \ + libgnutls28-dev \ + libuv1-dev \ + libmsgpack-dev \ + libargon2-0-dev \ + cython3 \ + python3-dev \ + python3-setuptools \ + libssl-dev \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /jsoncpp + +RUN \ + git clone --recursive https://github.com/open-source-parsers/jsoncpp.git && \ + mkdir jsoncpp/build && \ + cd jsoncpp/build && \ + cmake .. \ + -G "Unix Makefiles" \ + && \ + make install + + +WORKDIR /restbed + +RUN \ + git clone --recursive https://github.com/corvusoft/restbed.git && \ + mkdir restbed/build && \ + cd restbed/build && \ + cmake .. \ + -DCMAKE_INSTALL_LIBDIR=/usr/local/lib \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_SHARED=On \ + && \ + make install + + + +WORKDIR /opendht + +RUN \ + git clone https://github.com/savoirfairelinux/opendht.git && \ + mkdir opendht/build && \ + cd opendht/build && \ + cmake .. \ + -DOPENDHT_PROXY_SERVER=On \ + -DOPENDHT_SHARED=Off \ + && \ + make install && \ + ldconfig + +ENTRYPOINT ["dhtnode"] +CMD ["-i", "-b", "bootstrap.ring.cx:4222", "-p", "4222", "--proxyserver", "9100"] diff --git a/parity/Dockerfile b/parity/Dockerfile index 29a9063..c0eb715 100644 --- a/parity/Dockerfile +++ b/parity/Dockerfile @@ -1,3 +1,9 @@ FROM parity/parity:stable -ENV DEBIAN_FRONTEND noninteractive +WORKDIR /parity +COPY config.toml . +COPY parity.sh . + +ENTRYPOINT ["/parity/parity.sh"] +CMD ["run"] + diff --git a/parity/config.toml b/parity/config.toml new file mode 100644 index 0000000..439587e --- /dev/null +++ b/parity/config.toml @@ -0,0 +1,7 @@ +[parity] +chain = "kovan" +base_path = "/parity-data" + +[rpc] +force = true +interface = "all" diff --git a/parity/parity.sh b/parity/parity.sh new file mode 100755 index 0000000..c0f4232 --- /dev/null +++ b/parity/parity.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +case "$1" in + +noop) + ;; + +create-account) + ./parity --config config.toml account new + ;; + +run) + ./parity --config config.toml + ;; + +*) echo 'No operation specified' + exit 0; + ;; + +esac diff --git a/tools.sh b/tools.sh index 70f1bd3..89eb54e 100755 --- a/tools.sh +++ b/tools.sh @@ -7,8 +7,13 @@ set -o nounset case "$1" in +init) + #https://www.vaultproject.io/intro/getting-started/deploy.html#initializing-the-vault+ + ;; + demo) - docker-compose up --build --force-recreate + docker-compose create --build --force-recreate demo + docker-compose run --service-ports demo ./agent.sh $2 ;; alice) @@ -40,8 +45,14 @@ agent-web) docker-compose run --service-ports agent-web ./agent-web.sh run ;; +opendht) + docker-compose create --build --force-recreate opendht + docker-compose run --service-ports opendht + ;; + geth) - docker-compose run --service-ports geth geth --datadir=/geth-data --metrics --shh --rpc --rpcaddr 0.0.0.0 --ws --wsaddr 0.0.0.0 --nat none --verbosity 5 --vmdebug --dev --maxpeers 0 --gasprice 0 --debug --pprof + docker-compose create --build --force-recreate geth + docker-compose run --service-ports geth $2 ;; solc) @@ -49,7 +60,13 @@ solc) ;; parity) - docker-compose run --service-ports parity + docker-compose create --build --force-recreate parity + docker-compose run --service-ports parity $2 + ;; + +vault) + docker-compose create --build --force-recreate vault + docker-compose run --service-ports vault $2 ;; relex) diff --git a/vault/Dockerfile b/vault/Dockerfile new file mode 100644 index 0000000..7ab05e9 --- /dev/null +++ b/vault/Dockerfile @@ -0,0 +1,3 @@ +FROM vault:latest + +CMD ["server"]