updated default config with production release sample

root/defaults/diskover.cfg

@@ -52,6 +52,8 @@ indexrefresh = 30s
 disablereplicas = True
 ; transaction log flush threshold size (default 512mb)
 translogsize = 1gb
+; search scroll size (default 100 docs)
+scrollsize = 1000
 
 [redis]
 host = {{REDIS_HOST}}
@@ -63,46 +65,49 @@ port = {{REDIS_PORT}}
 cachedirtimes = False
 ; how long in seconds directory keys lives in Redis (default 1 day)
 dirtimesttl = 604800
-
-[treethreads]
-; number of threads to use for tree walking down directories in rootdir (cores x 2 might be a good start)
-threads = 8
+; database to use (default is 0)
+db = 0
+; rq job default time out in sec (default 180)
+timeout = 3600
+; rq default ttl for key/results (default 500)
+ttl = 500
+; rq queue names to use (default is diskover, diskover_crawl, diskover_calcdir)
+queue = diskover
+queuecrawl = diskover_crawl
+queuecalcdir = diskover_calcdir
 
 [adaptivebatch]
 ; adaptive batch settings when using -a (intelligent crawling)
 ; batchsize (numbers of dirs) to start at
 startsize = 50
 ; maximum size of batch
 maxsize = 500
-; when adjusting batch size use this for +/- (increase when queues is > 0, decrease when 0)
+; when adjusting batch size use this for +/- (increases when queue is 0, decreases when > 0)
 stepsize = 10
 
-[workerbot]
-; enable bot logs (True or False), bot logs will slow down crawl, use for debugging only
-botlogs = False
-; log file directory to store worker logs
-; log files are named diskover_bot_worker_<workername>_<time>_log
-logfiledir = /tmp
-
 [paths]
 ; used by diskover socket server
 ; path to diskover.py (default is ./diskover.py)
-diskoverpath = /app/diskover/diskover.py
+diskoverpath = ./diskover.py
 ; path to python executable (default is python)
 pythonpath = python
 
 [socketlistener]
 ; hostname and port (TCP) for diskover socket server for remote commands
 host = 0.0.0.0
 port = 9999
+; max connections for diskover socket server
+maxconnections = 5
+; port (TCP) for diskover socket server for messages from diskover treewalk client
+twcport = 9998
 
 [dupescheck]
-; read size (bytes) for md5 sum check
+; read size (bytes) for md5 sum check (how many bytes to read in at a time when md5 checking, default 64 KB)
 readsize = 65536
-; max size (bytes) of files to check
+; max size (bytes) of files to check (files larger than this will be skipped, default 1 GB)
 maxsize = 1073741824
-; bytes to check at start and end of file before doing md5 sum check
-checkbytes = 8
+; bytes to check at start and end of file before doing md5 sum check (set large enough to account for file header info, default is 64)
+checkbytes = 64
 
 [crawlbot]
 ; continuous scanner