[writers] Remove ElasticSearch writer

README.md

@@ -18,15 +18,14 @@ it. Once a job is finished, if the result is successful, the server will
 re-schedule it to retrieve new data.
 
 By default, items fetched by each job will be published using a Redis queue.
-Additionally, they can be written to an Elastic Search index.
 
 
 ## Usage
 
 ### arthurd
 ```
 usage: arthurd [-c <file>] [-g] [-h <host>] [-p <port>] [-d <database>]
-               [--es-index <index>] [--log-path <path>] [--archive-path <cpath>]
+               [--log-path <path>] [--archive-path <cpath>]
                [--no-archive] [--no-daemon] | --help
 
 King Arthur commands his loyal knight Perceval on the quest
@@ -46,7 +45,6 @@ optional arguments:
   -p, --port            set listening TCP port (default: 8080)
   -d, --database        URL database connection (default: 'redis://localhost/8')
   -s, --sync            work in synchronous mode (without workers)
-  --es-index            output ElasticSearch server index
   --log-path            path where logs are stored
   --archive-path        path to archive manager directory
   --no-archive          do not archive fetched raw data
@@ -121,13 +119,12 @@ $ python3 setup.py install
 ## How to run it
 
 The first step is to run a Redis server that will be used for communicating
-Arthur's components. Moreover, an Elastic Search server can be used to store
-the items generated by jobs. Please refer to their documentation to know how to
-install and run them both.
+Arthur's components. Please refer to its documentation to know how to
+install and run it.
 
 To run Arthur server:
 ```
-$ arthurd -g -d redis://localhost/8 --es-index http://localhost:9200/items --log-path /tmp/logs/arthud --no-archive
+$ arthurd -g -d redis://localhost/8 --log-path /tmp/logs/arthud --no-archive
 ```
 
 To run a worker:
@@ -186,9 +183,6 @@ Then, send this JSON stream to the server calling `add` method.
 $ curl -H "Content-Type: application/json" --data @tasks.json http://127.0.0.1:8080/add
 ```
 
-For this example, items will be stored in the `items` index on the
-Elastic Search server (http://localhost:9200/items).
-
 ## Listing tasks
 
 The list of tasks currently scheduled can be obtained using the method `tasks`.

arthur/server.py

@@ -22,8 +22,6 @@
 #
 
 import logging
-import threading
-import time
 
 import cherrypy
 
@@ -49,37 +47,13 @@ class ArthurServer(Arthur):
     """Arthur REST server"""
 
     def __init__(self, *args, **kwargs):
-        if 'writer' in kwargs:
-            writer = kwargs.pop('writer')
-
         super().__init__(*args, **kwargs)
-
-        if writer:
-            self.writer_th = threading.Thread(target=self.write_items,
-                                              args=(writer, self.items))
-        else:
-            self.writer_th = None
-
         cherrypy.engine.subscribe('start', self.start, 100)
 
     def start(self):
         """Start the server and the writer"""
 
         super().start()
-        if self.writer_th:
-            self.writer_th.start()
-
-    @classmethod
-    def write_items(cls, writer, items_generator):
-        """Write items to the queue
-
-        :param writer: the writer object
-        :param items_generator: items to be written in the queue
-        """
-        while True:
-            items = items_generator()
-            writer.write(items)
-            time.sleep(1)
 
     @cherrypy.expose
     @cherrypy.tools.json_in()

bin/arthurd

@@ -33,7 +33,6 @@ import redis
 
 from arthur.common import ARCHIVES_DEFAULT_PATH
 from arthur.server import ArthurServer
-from arthur.writers import ElasticItemsWriter
 
 
 ARTHURD_USAGE_MSG = \
@@ -59,7 +58,6 @@ optional arguments:
   -p, --port            set listening TCP port (default: 8080)
   -d, --database        URL database connection (default: 'redis://localhost/8')
   -s, --sync            work in synchronous mode (without workers)
-  --es-index            output ElasticSearch server index
   --log-path            path where logs are stored
   --archive-path        path to archive manager directory
   --no-archive          do not archive fetched raw data
@@ -81,17 +79,11 @@ def main():
 
     conn = connect_to_redis(args.database)
 
-    if args.es_index:
-        writer = ElasticItemsWriter(args.es_index)
-    else:
-        writer = None
-
     # Set archive manager directory
     base_archive_path = None if args.no_archive else args.archive_path
 
     app = ArthurServer(conn, base_archive_path,
-                       async_mode=args.sync_mode,
-                       writer=writer)
+                       async_mode=args.sync_mode)
 
     run_daemon = not args.no_daemon
 
@@ -196,8 +188,6 @@ def create_common_arguments_parser(defaults):
     parser.add_argument('-s', '--sync', dest='sync_mode',
                         action='store_false',
                         help=argparse.SUPPRESS)
-    parser.add_argument('--es-index', dest='es_index',
-                        help=argparse.SUPPRESS)
     parser.add_argument('--log-path', dest='log_path',
                         default=os.path.expanduser('~/.arthur/logs/'),
                         help=argparse.SUPPRESS)

tests/test_server.py

@@ -36,7 +36,7 @@
 def run_server(conn):
     """Start CherryPy server for the tests in the background and shut it down once a test is finished"""
 
-    server = ArthurServer(conn, None, async_mode=False, writer=None)
+    server = ArthurServer(conn, None, async_mode=False)
     cherrypy.tree.mount(server, '/')
     cherrypy.engine.start()
     cherrypy.engine.wait(cherrypy.engine.states.STARTED)