Merge pull request #230 from OpenUpSA/sys/background-search-update

Use celery rabbit queue for Haystack Elasticsearch upates

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 
 services:
   app:
@@ -9,6 +9,7 @@ services:
     depends_on:
       - db
       - elasticsearch
+      - rabbitmq
     volumes:
       - ./data:/app-data
       - .:/app
@@ -26,6 +27,7 @@ services:
       - ENVIRONMENT=development
       - SENTRY_DSN=https://[email protected]/5246237
       - GOOGLE_ANALYTICS_ID=G-T25P07Y7G2
+      - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq//
     ports:
       - "8000:8000"
     command: bin/wait-for-deps.sh python manage.py runserver 0.0.0.0:8000
@@ -53,6 +55,20 @@ services:
     ports:
       - 9200
 
+  cerebro:
+    image: lmenezes/cerebro
+    ports:
+      - "8001:9000"
+
+  rabbitmq:
+    image: rabbitmq:3.7.28
+    volumes:
+      - rabbitmq-data:/var/lib/rabbitmq/
+    ports:
+      - 5672:5672
+      - 15672:15672
+
 volumes:
   db-data:
   es-data:
+  rabbitmq-data:

pombola/search/search_indexes.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 
 from haystack import indexes
+from celery_haystack.indexes import CelerySearchIndex
 
 from pombola.core import models as core_models
 
@@ -16,7 +17,7 @@
 # Note - these indexes could be specified in the individual apps, which might
 # well be cleaner.
 
-class BaseIndex(indexes.SearchIndex):
+class BaseIndex(CelerySearchIndex):
     text = indexes.CharField(document=True, use_template=True)
 
 class PersonIndex(BaseIndex, indexes.Indexable):

pombola/settings/base.py

@@ -13,6 +13,7 @@
 
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration
+from sentry_sdk.integrations.celery import CeleryIntegration
 
 from urlparse import urlparse
 
@@ -324,7 +325,13 @@
     }
 
 
-HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"
+# HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"
+HAYSTACK_SIGNAL_PROCESSOR = "celery_haystack.signals.CelerySignalProcessor"
+CELERY_HAYSTACK_TRANSACTION_SAFE = True
+CELERY_HAYSTACK_DEFAULT_ALIAS = None
+CELERY_HAYSTACK_RETRY_DELAY = 5 * 60
+CELERY_HAYSTACK_MAX_RETRIES = 1
+CELERY_HAYSTACK_DEFAULT_TASK = "celery_haystack.tasks.CeleryHaystackSignalHandler"
 
 # Admin autocomplete
 AJAX_LOOKUP_CHANNELS = {
@@ -436,6 +443,7 @@
     # Pombola templates do {% load thumbnail %}
     "easy_thumbnails",
     "sorl.thumbnail",
+    "celery_haystack",
     "haystack",
     "slug_helpers",
     "info",
@@ -683,12 +691,12 @@ def show_toolbar(request):
 if os.environ.get("SENTRY_DSN"):
     sentry_sdk.init(
         dsn=os.environ.get("SENTRY_DSN"),
-        enable_tracing=True,
-        integrations=[DjangoIntegration()],
+        #enable_tracing=True,
+        integrations=[DjangoIntegration(), CeleryIntegration()],
         environment=os.environ.get("ENVIRONMENT"),
-        traces_sample_rate=os.environ.get("SENTRY_TRACES_SAMPLE_RATE", 1.0),
-        profiles_sample_rate=os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", 1.0),
+        #traces_sample_rate=os.environ.get("SENTRY_TRACES_SAMPLE_RATE", 1.0),
+        #profiles_sample_rate=os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", 1.0),
         send_default_pii=True,
     )
 
-DATA_UPLOAD_MAX_NUMBER_FIELDS=None
+DATA_UPLOAD_MAX_NUMBER_FIELDS = None

requirements.txt

@@ -124,16 +124,18 @@ anyjson==0.3.3
 audioread==1.0.3
 billiard==3.3.0.19
 bleach==3.3.0
-celery==3.1.17
+django-appconf==1.0.2
+celery==3.1.25
+celery-haystack==0.10
+kombu==3.0.37
 chardet==2.3.0
 cssselect==0.9.1
 django-bleach==0.5.2
-django-celery==3.1.16
+django-celery==3.1.17
 django-qmethod==0.0.3
 django-subdomain-instances==3.0.2
 django-tastypie==0.12.1
 html5lib==0.999
-kombu==3.0.24
 mechanize==0.4.6
 mimeparse==0.1.3
 mock==1.3.0
@@ -206,5 +208,6 @@ zipstream==1.1.4
 xlrd==1.2.0
 
 # Sentry
-sentry-sdk[django]==1.39.2
+#sentry-sdk[django]==1.39.2
+sentry-sdk==0.14.3