Merge pull request #2063 from fetchai/develop

Release v0.8.0

.github/workflows/workflow.yml

@@ -100,6 +100,10 @@ jobs:
           sudo apt-get autoremove
           sudo apt-get autoclean
           pip install tox
+          # install Protobuf compiler
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
+          unzip protoc-3.11.4-linux-x86_64.zip -d protoc
+          sudo mv protoc/bin/protoc /usr/local/bin/protoc
           # install IPFS
           sudo apt-get install -y wget
           wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
@@ -116,9 +120,27 @@ jobs:
         run: tox -e package_version_checks
       - name: Check package dependencies
         run: tox -e package_dependencies_checks
+      - name: Check generate protocols
+        run: tox -e check_generate_all_protocols
       - name: Generate Documentation
         run: tox -e docs
 
+  common_checks_5:
+    continue-on-error: False
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event_name == 'pull_request' && github.head_ref == 'master'
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/setup-python@master
+        with:
+          python-version: 3.6
+      - name: Check Docs links
+        run: tox -e check_doc_links
+      - name: Check API Docs updated
+        run: tox -e check_api_docs
+
+
   integration_checks:
     continue-on-error: True
     needs:
@@ -142,7 +164,10 @@ jobs:
         sudo apt-get autoremove
         sudo apt-get autoclean
         pip install tox
-        sudo apt-get install -y protobuf-compiler
+        # install Protobuf compiler
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
+        unzip protoc-3.11.4-linux-x86_64.zip -d protoc
+        sudo mv protoc/bin/protoc /usr/local/bin/protoc
     - name: Sync AEA loop integration tests
       run: |
         tox -e py3.8 -- -m 'sync'  # --aea-loop sync
@@ -171,9 +196,6 @@ jobs:
           pip install tox
       - name: Integration tests
         run: tox -e py3.8 -- -m 'integration and not unstable and ledger'
-        continue-on-error: true
-      - name: Force green exit
-        run: exit 0
 
   platform_checks:
     continue-on-error: True
@@ -203,22 +225,35 @@ jobs:
         sudo apt-get autoremove
         sudo apt-get autoclean
         pip install tox
-        sudo apt-get install -y protobuf-compiler
+        # install Protobuf compiler
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
+        unzip protoc-3.11.4-linux-x86_64.zip -d protoc
+        sudo mv protoc/bin/protoc /usr/local/bin/protoc
+    # sudo apt-get install -y protobuf-compiler
     # use sudo rm /var/lib/apt/lists/lock above in line above update if dependency install failures persist
     # use sudo apt-get dist-upgrade above in line below update if dependency install failures persist
     - if: matrix.os == 'macos-latest'
       name: Install dependencies (macos-latest)
       run: |
         pip install tox
-        brew install protobuf
+        brew install gcc
+        # brew install protobuf
+        # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-osx-x86_64.zip
+        unzip protoc-3.11.4-osx-x86_64.zip -d protoc
+        sudo mv protoc/bin/protoc /usr/local/bin/protoc
     - if: matrix.os == 'windows-latest'
       name: Install dependencies (windows-latest)
       env:
         ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       run: |
         pip install tox
         echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64"
-        choco install protoc
+        choco install protoc --version 3.11.4
+        choco install mingw -y
+        # wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-win64.zip
+        # unzip protoc-3.11.4-win64.zip -d protoc
+        # sudo mv protoc/bin/protoc /usr/local/bin/protoc
         python scripts/update_symlinks_cross_platform.py
     - if: matrix.os == 'ubuntu-latest' || matrix.python_version != '3.9'
       name: Unit tests
@@ -248,7 +283,9 @@ jobs:
         sudo apt-get autoremove
         sudo apt-get autoclean
         pip install tox
-        sudo apt-get install -y protobuf-compiler
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
+        unzip protoc-3.11.4-linux-x86_64.zip -d protoc
+        sudo mv protoc/bin/protoc /usr/local/bin/protoc
     - name: Unit tests with sync agent loop
       run: |
         tox -e py3.8 -- --aea-loop sync -m 'not integration and not unstable'
@@ -302,7 +339,11 @@ jobs:
         sudo apt-get autoremove
         sudo apt-get autoclean
         pip install tox
-        sudo apt-get install -y protobuf-compiler
+        # install Protobuf compiler
+        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
+        unzip protoc-3.11.4-linux-x86_64.zip -d protoc
+        sudo mv protoc/bin/protoc /usr/local/bin/protoc
+    # sudo apt-get install -y protobuf-compiler
     - name: Run all tests
       run: tox -e py3.7-cov -- --ignore=tests/test_docs --ignore=tests/test_examples --ignore=tests/test_packages/test_contracts --ignore=tests/test_packages/test_skills_integration -m 'not unstable'
       continue-on-error: true

.gitignore

@@ -122,6 +122,8 @@ output_file
 !packages/fetchai/contracts/erc1155/build
 !packages/fetchai/contracts/staking_erc20/build
 !packages/fetchai/contracts/oracle/build
+!packages/fetchai/contracts/oracle_client/build
+!packages/fetchai/contracts/fet_erc20/build
 packages/fetchai/connections/p2p_libp2p/libp2p_node
 
 !tests/data/dummy_contract/build

.pylintrc

@@ -1,5 +1,5 @@
 [MASTER]
-ignore-patterns=serialization.py,message.py,__main__.py,.*_pb2.py
+ignore-patterns=serialization.py,message.py,__main__.py,.*_pb2.py,tac.sh,tac_local.sh
 
 [MESSAGES CONTROL]
 disable=C0103,C0201,C0301,C0302,C0330,W0105,W0107,W0707,W1202,W1203,R0902,R0913,R0914,R0801,R0911,R0912,R0901,R0916,R1702,R0915
@@ -30,7 +30,7 @@ disable=C0103,C0201,C0301,C0302,C0330,W0105,W0107,W0707,W1202,W1203,R0902,R0913,
 # R0801: similar lines, # too granular
 
 [IMPORTS]
-ignored-modules=aiohttp,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,vyper,web3
+ignored-modules=aiohttp,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,vyper,web3,aioprometheus
 
 [DESIGN]
 min-public-methods=1

AUTHORS.md

@@ -13,3 +13,4 @@ This is the official list of Fetch.AI authors for copyright purposes.
 * Lokman Rahmani <[email protected]> [lrahmani](https://github.com/lrahmani)
 * Jiří Vestfál <[email protected]> [MissingNO57](https://github.com/MissingNO57)
 * Ed Fitzgerald <[email protected]> [ejfitzgerald](https://github.com/ejfitzgerald)
+* James Riehl <[email protected]> [jrriehl](https://github.com/jrriehl)

HISTORY.md

@@ -1,5 +1,31 @@
 # Release History
 
+## 0.8.0 (2020-12-17)
+
+- Adds support for protocol dialogue rules validation
+- Fixes url forwarding in http server connection
+- Revises protocols to correctly define terminal states
+- Adds a build command
+- Adds build command support for libp2p connection
+- Adds multiple fixes to libp2p connection
+- Adds prometheus connection and protocol
+- Adds tests for confirmation AW1 skill
+- Adds oracle demo docs
+- Replaces pickle with protobuf in all protocols
+- Refactors oef models to account for semantic irregularities
+- Updates docs for demos relying on Ganache
+- Adds generic storage support
+- Adds configurable dialogue offloading
+- Fixes transaction generation on confirmation bugs
+- Fixes transaction processing order in all buyer skills
+- Extends ledger api protocol to query ledger state
+- Adds remove-key command in CLI
+- Multiple tac stability fixes
+- Adds support for configurable error handler
+- Multiple additional tests to improve stability
+- Multiple docs updates based on user feedback
+- Multiple additional tests and test stability fixes
+
 ## 0.7.5 (2020-11-25)
 
 - Adds AW3 AEAs

MANIFEST.in

@@ -1,6 +1,6 @@
 include README.md LICENSE HISTORY.md AUTHORS.md SECURITY.md CODE_OF_CONDUCT.md Pipfile mkdocs.yml tox.ini pytest.ini strategy.ini
 
-recursive-include aea *.json *.yaml *.proto *.ico *png *.html *.js *.css *.md *.cfg
+recursive-include aea *.json *.yaml *.proto *.ico *png *.html *.js *.css *.md *.cfg json1.dll
 recursive-include docs *
 recursive-include examples *
 recursive-include packages *

Makefile

@@ -78,17 +78,17 @@ common_checks: security misc_checks lint static docs
 
 .PHONY: test
 test:
-	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub
 test-sub:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub-p
 test-sub-p:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 

Pipfile

@@ -10,6 +10,7 @@ name = "test-pypi"
 
 [dev-packages]
 aiohttp = "==3.6.2"
+aioprometheus = "==20.0.1"
 bandit = "==1.6.2"
 black = "==19.10b0"
 bs4 = "==0.0.1"

README.md

@@ -121,6 +121,8 @@ You can have more control on the installed dependencies by leveraging the setupt
 
 The following dependency is **only relevant if you intend to contribute** to the repository:
 
+- All Pull Requests should be opened against the `develop` branch. Do **not** open a Pull Request against `master`!
+
 - The project uses [Google Protocol Buffers](https://developers.google.com/protocol-buffers/) compiler for message serialization. A guide on how to install it is found [here](https://fetchai.github.io/oef-sdk-python/user/install.html#protobuf-compiler).
 
 The following steps are **only relevant if you intend to contribute** to the repository. They are **not required** for agent development.
@@ -137,34 +139,17 @@ The following steps are **only relevant if you intend to contribute** to the rep
 
       pip install -e ".[all]"
 
-- To run tests:
-
-      tox -e py3.7
-
-- To run linters (code style checks):
-
-      tox -e flake8
-      tox -e pylint
-
-- To run static type checks:
-
-      tox -e mypy
-
-- To run black code formatter:
-
-      tox -e black
+- To run tests: `tox -e py3.7` or `make test`. To only test specific modules try `make dir=PATH_TO_MODULE tdir=PATH_TO_TESTS test-sub` where  (e.g. `make dir=cli tdir=cli test-sub`).
 
-- To run isort code formatter:
+- To run linters (code style checks) and code formatters: `tox -e flake8` and `tox -e black` and ` tox -e isort` or `make lint`
 
-      tox -e isort
+- To run static type checks: `tox -e mypy` or `make static`
 
-- To run bandit security checks:
+- To run pylint: `tox -e pylint` or `make pylint`
 
-      tox -e bandit
+- To run security checks: `tox -e bandit` and `tox -e safety` or `make security`
 
-- To start a live-reloading docs server on localhost
-
-      mkdocs serve
+- To start a live-reloading docs server on localhost: `mkdocs serve`
 
 - To amend the docs, create a new documentation file in `docs/` and add a reference to it in `mkdocs.yml`.
 
@@ -179,7 +164,7 @@ consider to cite it with the following BibTex entry:
 
 ```
 @misc{agents-aea,
-  Author = {Marco Favorito and David Minarsch and Ali Hosseini and Aristotelis Triantafyllidis and Diarmid Campbell and Oleg Panasevych and Kevin Chen and Yuri Turchenkov and Lokman Rahmani and Jiří Vestfál},
+  Author = {Marco Favorito and David Minarsch and Ali Hosseini and Aristotelis Triantafyllidis and Diarmid Campbell and Oleg Panasevych and Kevin Chen and Yuri Turchenkov and Lokman Rahmani and Jiří Vestfál and James Riehl},
   Title = {Autonomous Economic Agent (AEA) Framework},
   Year = {2019},
 }

SECURITY.md

@@ -8,8 +8,8 @@ The following table shows which versions of `aea` are currently being supported
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 0.3.x   | :white_check_mark: |
-| < 0.3.0 | :x:                |
+| 0.7.x   | :white_check_mark: |
+| < 0.7.0 | :x:                |
 
 ## Reporting a Vulnerability
 

aea/__version__.py

@@ -22,7 +22,7 @@
 __title__ = "aea"
 __description__ = "Autonomous Economic Agent framework"
 __url__ = "https://github.com/fetchai/agents-aea.git"
-__version__ = "0.7.5"
+__version__ = "0.8.0"
 __author__ = "Fetch.AI Limited"
 __license__ = "Apache-2.0"
 __copyright__ = "2019 Fetch.AI Limited"

aea/abstract_agent.py

@@ -103,6 +103,10 @@ def get_multiplexer_setup_options(self) -> Optional[Dict]:
     def connections(self) -> List[Connection]:
         """Return list of connections."""
 
+    @abstractproperty
+    def storage_uri(self) -> Optional[str]:
+        """Return storage uri."""
+
     @abstractmethod
     def exception_handler(
         self, exception: Exception, function: Callable