Merge pull request #1269 from fetchai/develop

Release v0.3.3

.github/workflows/workflow.yml

@@ -8,6 +8,51 @@ on:
   pull_request:
 
 jobs:
+  sync_aea_loop_unit_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@master
+    - uses: actions/setup-python@master
+      with:
+        python-version: 3.8
+    - uses: actions/setup-go@master
+      with:
+        go-version: '^1.14.0'
+    - name: Install dependencies (ubuntu-latest)
+      run: |
+        sudo apt-get update --fix-missing
+        sudo apt-get autoremove
+        sudo apt-get autoclean
+        pip install pipenv
+        pip install tox
+        sudo apt-get install -y protobuf-compiler
+    - name: Unit tests and coverage
+      run: |
+        tox -e py3.8 -- --aea-loop sync -m 'not integration and not unstable'
+
+  sync_aea_loop_integrational_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@master
+    - uses: actions/setup-python@master
+      with:
+        python-version: 3.8
+    - uses: actions/setup-go@master
+      with:
+        go-version: '^1.14.0'
+    - name: Install dependencies (ubuntu-latest)
+      run: |
+        sudo apt-get update --fix-missing
+        sudo apt-get autoremove
+        sudo apt-get autoclean
+        pip install pipenv
+        pip install tox
+        sudo apt-get install -y protobuf-compiler
+    - name: Integrational tests and coverage
+      run: |
+        tox -e py3.8 -- --aea-loop sync -m 'integration and not unstable'
   common_checks:
     runs-on: ubuntu-latest
 
@@ -86,6 +131,9 @@ jobs:
     - uses: actions/setup-python@master
       with:
         python-version: ${{ matrix.python-version }}
+    - uses: actions/setup-go@master
+      with:
+        go-version: '^1.14.0'
     - if: matrix.os == 'ubuntu-latest'
       name: Install dependencies (ubuntu-latest)
       run: |

.gitignore

@@ -120,4 +120,5 @@ input_file
 output_file
 
 !packages/fetchai/contracts/erc1155/build
+packages/fetchai/connections/p2p_libp2p/libp2p_node
 

HISTORY.md

@@ -1,5 +1,23 @@
 # Release History
 
+## 0.3.3 (2020-05-24)
+
+- Adds option to pass ledger apis to aea builder
+- Refactors decision maker: separates interface and implementation; adds loading mechanisms so framework users can provide their own implementation
+- Adds async and sync agent loop implementations; agent can be run in both sync and async mode
+- Completes transition to atomic cli commands (fetch, generate, scaffold)
+- Refactors dialogue api: adds much simplified api; updates generator accordingly; updates skills
+- Adds support for crypto module extensions: framework users can register their own crypto module
+- Adds crypto module and ledger support for cosmos
+- Adds simple-oef (soef) connection
+- Adds p2p_libp2p connection for true p2p connectivity
+- Adds pypi dependency consistency checks for aea projects
+- Refactors cli for improved programmatic usage of components
+- Adds skill exception handling policies and configuration options
+- Adds comprehensive documentation of configuration files
+- Multiple docs updates
+- Multiple test stability fixes
+
 ## 0.3.2 (2020-05-07)
 
 - Adds dialogue generation functionality to protocol generator

Makefile

@@ -74,6 +74,11 @@ test:
 	pytest --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections tests/ --cov-report=html --cov-report=xml --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections
 	rm -fr .coverage*
 
+.PHONY: test-sub
+test-sub:
+	pytest --doctest-modules $(dir) $(tdir) --cov-report=html --cov-report=xml --cov-report=term --cov=$(dir)
+	rm -fr .coverage*
+
 .PHONY: test-all
 test-all:
 	tox

Pipfile

@@ -14,6 +14,7 @@ bandit = "==1.6.2"
 black = "==19.10b0"
 bs4 = "==0.0.1"
 colorlog = "==4.1.0"
+defusedxml = "==0.6.0"
 docker = "==4.2.0"
 fetch-p2p-api = {index = "https://test.pypi.org/simple/",version = "==0.0.2"}
 flake8 = "==3.7.9"
@@ -26,6 +27,7 @@ liccheck = "==0.4.3"
 markdown = ">=3.2.1"
 matplotlib = "==3.2.1"
 memory-profiler = "==0.57.0"
+mistune = "==2.0.0a4"
 mkdocs = "==1.1"
 mkdocs-material = "==4.6.3"
 mkdocs-mermaid-plugin = {git = "https://github.com/pugong/mkdocs-mermaid-plugin.git"}
@@ -39,16 +41,15 @@ psutil = "==5.7.0"
 pydocstyle = "==3.0.0"
 pygments = "==2.5.2"
 pymdown-extensions = "==6.3"
+pynacl = "==1.3.0"
 pytest = "==5.3.5"
 pytest-asyncio = "==0.10.0"
 pytest-cov = "==2.8.1"
 pytest-randomly = "==3.2.1"
 requests = ">=2.22.0"
 safety = "==1.8.5"
-tox = "==3.14.5"
+tox = "==3.15.1"
 vyper = "==0.1.0b12"
-mistune = "==2.0.0a4"
-pynacl = "==1.3.0"
 
 [packages]
 # we don't specify dependencies for the library here for intallation as per: https://pipenv-fork.readthedocs.io/en/latest/advanced.html#pipfile-vs-setuppy

aea/__version__.py

@@ -22,7 +22,7 @@
 __title__ = "aea"
 __description__ = "Autonomous Economic Agent framework"
 __url__ = "https://github.com/fetchai/agents-aea.git"
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 __author__ = "Fetch.AI Limited"
 __license__ = "Apache-2.0"
 __copyright__ = "2019 Fetch.AI Limited"

aea/aea.py

@@ -17,26 +17,31 @@
 #
 # ------------------------------------------------------------------------------
 """This module contains the implementation of an autonomous economic agent (AEA)."""
-
 import logging
 from asyncio import AbstractEventLoop
-from typing import List, Optional, cast
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type, cast
 
 from aea.agent import Agent
+from aea.agent_loop import AsyncAgentLoop, BaseAgentLoop, SyncAgentLoop
 from aea.configurations.constants import DEFAULT_SKILL
 from aea.connections.base import Connection
 from aea.context.base import AgentContext
 from aea.crypto.ledger_apis import LedgerApis
 from aea.crypto.wallet import Wallet
-from aea.decision_maker.base import DecisionMaker
-from aea.helpers.exec_timeout import ExecTimeoutThreadGuard
+from aea.decision_maker.base import DecisionMaker, DecisionMakerHandler
+from aea.decision_maker.default import (
+    DecisionMakerHandler as DefaultDecisionMakerHandler,
+)
+from aea.exceptions import AEAException
+from aea.helpers.exception_policy import ExceptionPolicyEnum
+from aea.helpers.exec_timeout import ExecTimeoutThreadGuard, TimeoutException
 from aea.identity.base import Identity
 from aea.mail.base import Envelope
 from aea.protocols.base import Message
 from aea.protocols.default.message import DefaultMessage
 from aea.registries.filter import Filter
 from aea.registries.resources import Resources
-from aea.skills.base import Behaviour, Handler
+from aea.skills.base import Behaviour, Handler, SkillComponent
 from aea.skills.error.handlers import ErrorHandler
 from aea.skills.tasks import TaskManager
 
@@ -46,6 +51,12 @@
 class AEA(Agent):
     """This class implements an autonomous economic agent."""
 
+    RUN_LOOPS: Dict[str, Type[BaseAgentLoop]] = {
+        "sync": SyncAgentLoop,
+        "async": AsyncAgentLoop,
+    }
+    DEFAULT_RUN_LOOP: str = "async"
+
     def __init__(
         self,
         identity: Identity,
@@ -58,6 +69,11 @@ def __init__(
         execution_timeout: float = 0,
         is_debug: bool = False,
         max_reactions: int = 20,
+        decision_maker_handler_class: Type[
+            DecisionMakerHandler
+        ] = DefaultDecisionMakerHandler,
+        skill_exception_policy: ExceptionPolicyEnum = ExceptionPolicyEnum.propagate,
+        loop_mode: Optional[str] = None,
         **kwargs,
     ) -> None:
         """
@@ -73,6 +89,9 @@ def __init__(
         :param exeution_timeout: amount of time to limit single act/handle to execute.
         :param is_debug: if True, run the agent in debug mode (does not connect the multiplexer).
         :param max_reactions: the processing rate of envelopes per tick (i.e. single loop).
+        :param decision_maker_handler_class: the class implementing the decision maker handler to be used.
+        :param skill_exception_policy: the skill exception policy enum
+        :param loop_mode: loop_mode to choose agent run loop.
         :param kwargs: keyword arguments to be attached in the agent context namespace.
 
         :return: None
@@ -83,27 +102,33 @@ def __init__(
             loop=loop,
             timeout=timeout,
             is_debug=is_debug,
+            loop_mode=loop_mode,
         )
 
         self.max_reactions = max_reactions
         self._task_manager = TaskManager()
-        self._decision_maker = DecisionMaker(identity, wallet, ledger_apis)
+        decision_maker_handler = decision_maker_handler_class(
+            identity=identity, wallet=wallet, ledger_apis=ledger_apis
+        )
+        self._decision_maker = DecisionMaker(
+            decision_maker_handler=decision_maker_handler
+        )
         self._context = AgentContext(
             self.identity,
             ledger_apis,
             self.multiplexer.connection_status,
             self.outbox,
             self.decision_maker.message_in_queue,
-            self.decision_maker.ownership_state,
-            self.decision_maker.preferences,
-            self.decision_maker.goal_pursuit_readiness,
+            decision_maker_handler.context,
             self.task_manager,
             **kwargs,
         )
         self._execution_timeout = execution_timeout
         self._resources = resources
         self._filter = Filter(self.resources, self.decision_maker.message_out_queue)
 
+        self._skills_exception_policy = skill_exception_policy
+
     @property
     def decision_maker(self) -> DecisionMaker:
         """Get decision maker."""
@@ -155,7 +180,7 @@ def act(self) -> None:
 
         :return: None
         """
-        for behaviour in self._filter.get_active_behaviours():
+        for behaviour in self._get_active_behaviours():
             self._behaviour_act(behaviour)
 
     def react(self) -> None:
@@ -216,6 +241,7 @@ def _handle(self, envelope: Envelope) -> None:
         try:
             msg = protocol.serializer.decode(envelope.message)
             msg.counterparty = envelope.sender
+            msg.is_incoming = True
         except Exception as e:
             error_handler.send_decoding_error(envelope)
             logger.warning("Decoding error. Exception: {}".format(str(e)))
@@ -238,15 +264,7 @@ def _handle_message_with_handler(self, message: Message, handler: Handler) -> No
         :param message: message to be handled.
         :param handler: handler suitable for this message protocol.
         """
-        with ExecTimeoutThreadGuard(self._execution_timeout) as timeout_result:
-            handler.handle(message)
-
-        if timeout_result.is_cancelled_by_timeout():
-            logger.warning(
-                "Handler `{}` was terminated as its execution exceeded the timeout of {} seconds. Please refactor your code!".format(
-                    handler, self._execution_timeout
-                )
-            )
+        self._execution_control(handler.handle, handler, [message])
 
     def _behaviour_act(self, behaviour: Behaviour) -> None:
         """
@@ -255,15 +273,59 @@ def _behaviour_act(self, behaviour: Behaviour) -> None:
         :param behaviour: behaviour already defined
         :return: None
         """
-        with ExecTimeoutThreadGuard(self._execution_timeout) as timeout_result:
-            behaviour.act_wrapper()
+        self._execution_control(behaviour.act_wrapper, behaviour)
 
-        if timeout_result.is_cancelled_by_timeout():
+    def _execution_control(
+        self,
+        fn: Callable,
+        component: SkillComponent,
+        args: Optional[Sequence] = None,
+        kwargs: Optional[Dict] = None,
+    ) -> Any:
+        """
+        Execute skill function in exception handling environment.
+
+        Logs error, stop agent or propagate excepion depends on policy defined.
+
+        :param fn: function to call
+        :param component: skill component function belongs to
+        :param args: optional sequence of arguments to pass to function on call
+        :param kwargs: optional dict of keyword arguments to pass to function on call
+
+        :return: same as function
+        """
+        # docstyle: ignore
+        def log_exception(e, fn, component):
+            logger.exception(f"<{e}> raised during `{fn}` call of `{component}`")
+
+        try:
+            with ExecTimeoutThreadGuard(self._execution_timeout):
+                return fn(*(args or []), **(kwargs or {}))
+        except TimeoutException:
             logger.warning(
-                "Act of `{}` was terminated as its execution exceeded the timeout of {} seconds. Please refactor your code!".format(
-                    behaviour, self._execution_timeout
+                "`{}` of `{}` was terminated as its execution exceeded the timeout of {} seconds. Please refactor your code!".format(
+                    fn, component, self._execution_timeout
                 )
             )
+        except Exception as e:
+            if self._skills_exception_policy == ExceptionPolicyEnum.propagate:
+                raise
+            elif self._skills_exception_policy == ExceptionPolicyEnum.just_log:
+                log_exception(e, fn, component)
+            elif self._skills_exception_policy == ExceptionPolicyEnum.stop_and_exit:
+                log_exception(e, fn, component)
+                self.stop()
+                raise AEAException(
+                    f"AEA was terminated cause exception `{e}` in skills {component} {fn}! Please check logs."
+                )
+            else:
+                raise AEAException(
+                    f"Unsupported exception policy: {self._skills_exception_policy}"
+                )
+
+    def _get_active_behaviours(self) -> List[Behaviour]:
+        """Get all active behaviours to use in act."""
+        return self._filter.get_active_behaviours()
 
     def update(self) -> None:
         """