Merge remote-tracking branch 'gnosis/master'

.github/workflows/cla.yml

@@ -0,0 +1,36 @@
+name: "CLA Assistant"
+on:
+  issue_comment:
+    types: [ created ]
+  pull_request_target:
+    types: [ opened,closed,synchronize ]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "CLA Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        # Beta Release
+        uses: cla-assistant/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # the below token should have repo scope and must be manually added by you in the repository's secret
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        with:
+          path-to-signatures: 'signatures/version1/cla.json'
+          path-to-document: 'https://gnosis-safe.io/cla/'
+          # branch should not be protected
+          branch: 'cla-signatures'
+          allowlist: jpalvarezl,fmrsabino,luarx,rmeissner,Uxio0,*bot # may need to update this expression if we add new bots
+
+          #below are the optional inputs - If the optional inputs are not given, then default values will be taken
+          #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #remote-repository-name:  enter the  remote repository name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #create-file-commit-message: 'For example: Creating file for storing CLA Signatures'
+          #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo'
+          #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign'
+          #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA'
+          #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.'
+          #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true)
+          #use-dco-flag: true - If you are using DCO instead of CLA

config/settings/base.py

@@ -83,6 +83,7 @@
     "rest_framework",
     "drf_yasg",
     "django_s3_storage",
+    "rest_framework.authtoken",
 ]
 LOCAL_APPS = [
     "safe_transaction_service.contracts.apps.ContractsConfig",
@@ -228,7 +229,10 @@
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_always_eager
 CELERY_ALWAYS_EAGER = False
 # https://docs.celeryproject.org/en/latest/userguide/configuration.html#task-default-priority
-CELERY_TASK_DEFAULT_PRIORITY = 5  # Higher = more priority
+# Higher = more priority on RabbitMQ, opposite on Redis ¯\_(ツ)_/¯
+CELERY_TASK_DEFAULT_PRIORITY = 3
+# https://docs.celeryproject.org/en/stable/userguide/configuration.html#task-queue-max-priority
+CELERY_TASK_QUEUE_MAX_PRIORITY = 10
 # https://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-transport-options
 CELERY_BROKER_TRANSPORT_OPTIONS = {
     "queue_order_strategy": "priority",
@@ -264,6 +268,9 @@
     "disable_existing_loggers": False,
     "filters": {
         "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"},
+        "ignore_succeeded_none": {
+            "()": "safe_transaction_service.utils.loggers.IgnoreSucceededNone"
+        },
     },
     "formatters": {
         "short": {"format": "%(asctime)s %(message)s"},
@@ -293,6 +300,7 @@
         },
         "celery_console": {
             "level": "DEBUG",
+            "filters": [] if DEBUG else ["ignore_succeeded_none"],
             "class": "logging.StreamHandler",
             "formatter": "celery_verbose",
         },
@@ -414,5 +422,10 @@
 )
 
 ETHERSCAN_API_KEY = env("ETHERSCAN_API_KEY", default=None)
-IPFS_GATEWAY = env("IPFS_GATEWAY", default="https://cloudflare-ipfs.com/")
-ENABLE_OWNERS_ENDPOINT = env.bool("ENABLE_OWNERS_ENDPOINT", default=True)
+IPFS_GATEWAY = env("IPFS_GATEWAY", default="https://cloudflare-ipfs.com/ipfs/")
+
+SWAGGER_SETTINGS = {
+    "SECURITY_DEFINITIONS": {
+        "api_key": {"type": "apiKey", "in": "header", "name": "Authorization"}
+    },
+}

requirements-test.txt

@@ -2,8 +2,8 @@
 coverage==6.2
 django-stubs==1.9.0
 factory-boy==3.2.1
-faker==10.0.0
-mypy==0.930
+faker==11.3.0
+mypy==0.931
 pytest==6.2.5
 pytest-celery==0.0.0
 pytest-django==4.5.2

requirements.txt

@@ -1,5 +1,5 @@
 cachetools==4.2.4
-celery==5.2.1
+celery==5.2.3
 django==3.2.10
 django-authtools==1.7.0
 django-cache-memoize==0.1.10
@@ -20,13 +20,13 @@ docutils==0.18.1
 drf-yasg[validation]==1.20.0
 ethereum==2.3.2
 firebase-admin==5.2.0
-gnosis-py[django]==3.7.5
+gnosis-py[django]==3.7.6
 gunicorn[gevent]==20.1.0
 hexbytes==0.2.2
 packaging>=21.0
-pillow==8.4.0
+pillow==9.0.0
 psycogreen==1.0.2
-psycopg2==2.9.2
-redis==4.0.2
-requests==2.26.0
+psycopg2==2.9.3
+redis==4.1.0
+requests==2.27.1
 web3==5.24.0

safe_transaction_service/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.4.23"
+__version__ = "4.0.0"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

safe_transaction_service/contracts/tasks.py

@@ -36,7 +36,7 @@ def create_missing_contracts_with_metadata_task() -> int:
     for address in addresses:
         logger.info("Detected missing contract %s", address)
         create_or_update_contract_with_metadata_task.apply_async(
-            (address,), priority=0
+            (address,), priority=5
         )  # Lowest priority
         i += 1
     return i
@@ -55,7 +55,7 @@ def reindex_contracts_without_metadata_task() -> int:
         Contract.objects.without_metadata().values_list("address", flat=True).iterator()
     ):
         logger.info("Reindexing contract %s", address)
-        create_or_update_contract_with_metadata_task.apply_async((address,), priority=0)
+        create_or_update_contract_with_metadata_task.apply_async((address,), priority=5)
         i += 1
     return i
 

safe_transaction_service/history/admin.py

@@ -6,6 +6,7 @@
 from django.db.transaction import atomic
 
 from hexbytes import HexBytes
+from rest_framework.authtoken.admin import TokenAdmin
 
 from gnosis.eth import EthereumClientProvider
 
@@ -30,6 +31,14 @@
 )
 from .services import IndexServiceProvider
 
+# By default, TokenAdmin doesn't allow key edition
+# IFF you have a service that requests from multiple safe-transaction-service
+# you might want to share that key for convenience between instances.
+TokenAdmin.fields = (
+    "user",
+    "key",
+)
+
 
 # Inline objects ------------------------------
 class ERC20TransferInline(admin.TabularInline):

safe_transaction_service/history/services/collectibles_service.py

@@ -41,6 +41,7 @@ class MetadataRetrievalException(CollectiblesServiceException):
 
 def ipfs_to_http(uri: Optional[str]) -> Optional[str]:
     if uri and uri.startswith("ipfs://"):
+        uri = uri.replace("ipfs://ipfs/", "ipfs://")
         return urljoin(
             settings.IPFS_GATEWAY, uri.replace("ipfs://", "", 1)
         )  # Use ipfs gateway
@@ -159,6 +160,7 @@ def _retrieve_metadata_from_uri(self, uri: str) -> Dict[Any, Any]:
         """
         Get metadata from uri. Maybe at some point support IPFS or another protocols. Currently just http/https is
         supported
+
         :param uri: Uri starting with the protocol, like http://example.org/token/3
         :return: Metadata as a decoded json
         """

safe_transaction_service/history/signals.py

@@ -247,9 +247,13 @@ def process_webhook(
     )
     for payload in payloads:
         if address := payload.get("address"):
-            send_webhook_task.delay(address, payload)
+            send_webhook_task.apply_async(
+                args=(address, payload), priority=4
+            )  # Almost the lowest priority
             if is_relevant_notification(sender, instance, created):
-                send_notification_task.apply_async(args=(address, payload), countdown=5)
+                send_notification_task.apply_async(
+                    args=(address, payload), countdown=5, priority=4
+                )
             else:
                 logger.debug(
                     "Notification will not be sent for created=%s object=%s",

safe_transaction_service/history/tasks.py

@@ -355,7 +355,15 @@ def check_reorgs_task(self) -> Optional[int]:
 @cache
 def get_webhook_http_session(webhook_url: str) -> requests.Session:
     logger.debug("Getting http session for url=%s", webhook_url)
-    return requests.Session()
+    session = requests.Session()
+    adapter = requests.adapters.HTTPAdapter(
+        pool_connections=1,  # Doing all the connections to the same url
+        pool_maxsize=100,  # Number of concurrent connections
+        pool_block=False,
+    )
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
 
 
 @app.shared_task(

safe_transaction_service/history/tests/test_collectibles_service.py

@@ -38,7 +38,13 @@ def test_ipfs_to_http(self):
         ipfs_url = "ipfs://testing-url/path/?arguments"
         result = ipfs_to_http(ipfs_url)
         self.assertTrue(result.startswith("http"))
-        self.assertIn("testing-url/path/?arguments", result)
+        self.assertIn("ipfs/testing-url/path/?arguments", result)
+
+        ipfs_with_path_url = "ipfs://ipfs/testing-url/path/?arguments"
+        result = ipfs_to_http(ipfs_with_path_url)
+        self.assertTrue(result.startswith("http"))
+        self.assertNotIn("ipfs/ipfs", result)
+        self.assertIn("ipfs/testing-url/path/?arguments", result)
 
     def test_get_collectibles(self):
         mainnet_node = just_test_if_mainnet_node()

safe_transaction_service/history/tests/test_signals.py

@@ -79,7 +79,7 @@ def test_build_webhook_payload(self):
     @factory.django.mute_signals(post_save)
     def test_process_webhook(self):
         multisig_confirmation = MultisigConfirmationFactory()
-        with mock.patch.object(send_webhook_task, "delay") as webhook_task_mock:
+        with mock.patch.object(send_webhook_task, "apply_async") as webhook_task_mock:
             with mock.patch.object(
                 send_notification_task, "apply_async"
             ) as send_notification_task_mock:
@@ -88,7 +88,7 @@ def test_process_webhook(self):
                 send_notification_task_mock.assert_called()
 
         multisig_confirmation.created -= timedelta(minutes=45)
-        with mock.patch.object(send_webhook_task, "delay") as webhook_task_mock:
+        with mock.patch.object(send_webhook_task, "apply_async") as webhook_task_mock:
             with mock.patch.object(
                 send_notification_task, "apply_async"
             ) as send_notification_task_mock:

safe_transaction_service/history/views.py

@@ -1029,21 +1029,11 @@ def get(self, request, address, *args, **kwargs):
                 },
             )
 
-        if settings.ENABLE_OWNERS_ENDPOINT:
-            return self.get_owners(address)
-        else:
-            return self.get_owners_empty()
-
-    def get_owners(self, address):
         safes_for_owner = SafeStatus.objects.addresses_for_owner(address)
         serializer = self.serializer_class(data={"safes": safes_for_owner})
         assert serializer.is_valid()
         return Response(status=status.HTTP_200_OK, data=serializer.data)
 
-    def get_owners_empty(self):
-        serializer = self.serializer_class(data={"safes": []})
-        return Response(status=status.HTTP_200_OK, data=serializer.data)
-
 
 class DataDecoderView(GenericAPIView):
     def get_serializer_class(self):

safe_transaction_service/utils/tests/test_loggers.py

@@ -2,8 +2,6 @@
 
 from django.test import TestCase
 
-import pytest
-
 from ..loggers import IgnoreCheckUrl, IgnoreSucceededNone
 
 
@@ -23,7 +21,6 @@ def test_ignore_check_url(self):
         self.assertFalse(ignore_check_url.filter(check_log))
         self.assertTrue(ignore_check_url.filter(other_log))
 
-    @pytest.mark.skip(reason="Filter is disabled temporarily")  # TODO
     def test_ignore_succeeded_none(self):
         name = "name"
         level = 1