[Draft]Initial pass at py3 compatibility.

README.md

@@ -36,7 +36,7 @@ By providing an easy to use interactive command line application, users are able
 
 This project has taken motivation from the greats that came before it such as SpiderFoot, Harpoon, and DataSploit. Much thanks to those great authors for contributing to the world of open source.
 
-The application is written with Python 2.7 in mind and has been successfully tested on OSX and Ubuntu 16.04 environments.
+The application is written with Python 3.6 in mind and has been successfully tested on MacOS.
 
 This is a beta of the final application and as such there may be some bugs or other weirdness during usage. For the most part Omnibus is fully functional and can be used to begin OSINT investigation right away.
 

lib/cache.py

@@ -8,28 +8,27 @@
 
 from redis import Redis
 
-from common import error
+from .common import error
 
-from common import get_option
+from .common import get_option
 
-from common import utf_decode
-from common import utf_encode
+from .common import utf_decode
+from .common import utf_encode
 
 
 class RedisCache(object):
+
     def __init__(self, config):
         self.host = get_option('redis', 'host', config)
         self.port = int(get_option('redis', 'port', config))
         self.database = int(get_option('redis', 'db', config))
         self.ttl = 999999
 
         try:
-            self.db = Redis(db=self.database, host=self.host,
-                port=self.port, socket_timeout=None)
+            self.db = Redis(db=self.database, host=self.host, port=self.port, socket_timeout=None)
         except:
             self.db = None
 
-
     def receive(self, queue_name):
         """ Return most recent message from a given Redis queue"""
         try:
@@ -41,28 +40,24 @@ def receive(self, queue_name):
             error('[redis] failed to receive message from queue %s (error: %s)' % (queue_name, str(err)))
             pass
 
-
     def delete(self, names):
         """ Remove one or more keys by name """
         try:
             self.db.delete(names)
         except Exception as err:
             error('[redis] failed to delete artifacts (error: %s)' % str(err))
 
-
     def exists(self, key):
         """ Check if value exists by key """
         return self.db.exists(key)
 
-
     def get(self, key):
         """ Get a value from redis by key """
         retval = self.db.get(key)
         if isinstance(retval, bytes):
             return utf_decode(retval)
         return retval
 
-
     def set(self, key, value, ttl=None):
         """ Set a value in cache with optional TTL """
         if ttl is None:
@@ -73,7 +68,6 @@ def set(self, key, value, ttl=None):
         self.db.setnx(key, value)
         self.db.expire(key, ttl)
 
-
     def flush(self):
         """ Flush opened database entirely """
         self.db.flushdb()

lib/common.py

@@ -5,7 +5,7 @@
 import json
 import string
 import datetime
-import ConfigParser
+import configparser
 
 from pygments import lexers
 from pygments import highlight
@@ -72,12 +72,15 @@ def pp_json(data):
     if data is None:
         warning('No data returned from module.')
     else:
-        print(highlight(unicode(json.dumps(data, indent=4, default=jsondate), 'UTF-8'),
-            lexers.JsonLexer(), formatters.TerminalFormatter()))
+        print(highlight(
+            json.dumps(data, indent=4, default=jsondate),
+            lexers.JsonLexer(),
+            formatters.TerminalFormatter(),
+        ))
 
 
 def get_option(section, name, conf):
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     if not os.path.exists(conf):
         error('configuration file %s does not exist!' % conf)
         return None
@@ -165,7 +168,7 @@ def mkdir(path):
     else:
         try:
             os.mkdir(path)
-            os.chmod(path, 0777)
+            os.chmod(path, 0x1FF) # 0777 permissions
             return True
         except:
             return False
@@ -177,7 +180,7 @@ def lookup_key(session, artifact):
     valid_key = False
 
     if session is None:
-        return (valid_key, value)
+        return valid_key, value
 
     try:
         artifact = int(artifact)
@@ -186,7 +189,7 @@ def lookup_key(session, artifact):
     except:
         pass
 
-    return (valid_key, value)
+    return valid_key, value
 
 
 def utf_decode(data):
@@ -238,32 +241,31 @@ def is_email(address):
     return bool(re.match(re_email, address))
 
 
-def is_hash(string):
+def is_hash(string_):
     """ Check if string is a valid hash and if so what kind """
-    if re.match(re_md5, string):
+    if re.match(re_md5, string_):
         return 'md5'
-    elif re.match(re_sha1, string):
+    elif re.match(re_sha1, string_):
         return 'sha1'
-    elif re.match(re_sha256, string):
+    elif re.match(re_sha256, string_):
         return 'sha256'
-    elif re.match(re_sha512, string):
+    elif re.match(re_sha512, string_):
         return 'sha512'
     else:
         return False
 
 
-def is_btc_addr(string):
+def is_btc_addr(string_):
     """Check if string is matches as a Bitcoin address.
 
     @note: This does not verify that the string is a VALID BTC address,
     only that it matches the regex pattern of BTC addresses.
     """
-    if re.match(re_btc, string):
+    if re.match(re_btc, string_):
         return 'btc'
     return False
 
 
-
 def detect_type(artifact):
     """ Determine type of given argument """
     if is_ipv4(artifact):

lib/dispatch.py

@@ -5,15 +5,15 @@
 ##
 import importlib
 
-from common import info
-from common import error
-from common import success
-from common import warning
+from .common import info
+from .common import error
+from .common import success
+from .common import warning
 
-from common import lookup_key
-from common import detect_type
+from .common import lookup_key
+from .common import detect_type
 
-from models import create_artifact
+from .models import create_artifact
 
 
 class Dispatch(object):
@@ -44,9 +44,9 @@ def __init__(self, db):
             ],
         }
 
-
     def machine(self, session, artifact):
         """ Run all modules against an artifact of a given type """
+        modules = []
         is_key, value = lookup_key(session, artifact)
 
         if is_key and value is None:
@@ -77,8 +77,13 @@ def machine(self, session, artifact):
                     if self.db.exists(artifact['type'], {'name': artifact['name']}):
 
                         for child in result['children']:
-                            child_artifact = create_artifact(child['name'], parent=artifact['name'],
-                                _type=child['type'], source=child['source'], subtype=child['subtype'])
+                            child_artifact = create_artifact(
+                                child['name'],
+                                parent=artifact['name'],
+                                _type=child['type'],
+                                source=child['source'],
+                                subtype=child['subtype'],
+                            )
 
                             if not self.db.exists(child['type'], {'name': child['name']}):
                                 self.db.insert_one(child['type'], child_artifact)
@@ -97,7 +102,6 @@ def machine(self, session, artifact):
 
         success('Machine completed')
 
-
     def submit(self, session, module, artifact, no_argument=False):
         """ Run a single module against an artifact """
         if no_argument:
@@ -135,8 +139,13 @@ def submit(self, session, module, artifact, no_argument=False):
                 if self.db.exists(artifact['type'], {'name': artifact['name']}):
 
                     for child in result['children']:
-                        child_artifact = create_artifact(child['name'], parent=artifact['name'],
-                            _type=child['type'], source=child['source'], subtype=child['subtype'])
+                        child_artifact = create_artifact(
+                            child['name'],
+                            parent=artifact['name'],
+                            _type=child['type'],
+                            source=child['source'],
+                            subtype=child['subtype'],
+                        )
 
                         if not self.db.exists(child['type'], {'name': child['name']}):
                             self.db.insert_one(child['type'], child_artifact)
@@ -155,11 +164,8 @@ def submit(self, session, module, artifact, no_argument=False):
         else:
             warning('Failed to get module results (%s)' % module)
 
-
     def run(self, module, artifact):
         """ Load Python library from modules directory and execute main function """
-        results = None
-
         try:
             ptr = importlib.import_module('lib.modules.%s' % module)
         except Exception as err:

lib/http.py

@@ -32,12 +32,12 @@ def post(*args, **kwargs):
         try:
             req = requests.post(*args, **kwargs)
         except:
-            return (False, None)
+            return False, None
 
         if req.status_code == 200:
-            return (True, req)
+            return True, req
         else:
-            return (False, req)
+            return False, req
 
 
 def get(*args, **kwargs):
@@ -52,9 +52,9 @@ def get(*args, **kwargs):
         try:
             req = requests.get(*args, **kwargs)
         except:
-            return (False, None)
+            return False, None
 
         if req.status_code == 200:
-            return (True, req)
+            return True, req
         else:
-            return (False, req)
+            return False, req

lib/models.py

@@ -4,24 +4,24 @@
 # starter data models for host, email, and user artifacts
 ##
 
-from common import is_ipv4
-from common import is_ipv6
-from common import is_fqdn
-from common import is_hash
+from .common import is_ipv4
+from .common import is_ipv6
+from .common import is_fqdn
+from .common import is_hash
 
-from common import warning
-from common import timestamp
-from common import detect_type
+from .common import warning
+from .common import timestamp
+from .common import detect_type
 
 
 artifact_types = ['host', 'email', 'user', 'bitcoin', 'hash']
 
 
 class Artifact(object):
-    def __init__(self, name, type, source=None, subtype=None, case_id=None):
+    def __init__(self, name, type_, source=None, subtype=None, case_id=None):
         self.name = name
         self.created = timestamp()
-        self.type = type
+        self.type = type_
         self.subtype = subtype
         self.source = source
         self.parent = None
@@ -69,7 +69,7 @@ def create_artifact(artifact_name, _type=None, source=None, subtype=None, parent
         warning('Artifact must be one of: email, ipv4, fqdn, user, hash, bitcoin address')
         return None
 
-    created = Artifact(name=artifact_name, type=artifact_type, subtype=subtype, source=source)
+    created = Artifact(name=artifact_name, type_=artifact_type, subtype=subtype, source=source)
 
     if parent is not None:
         created.parent = parent

lib/modules/blockchain.py

@@ -3,18 +3,18 @@
 # omnibus - deadbits
 # blockchain.info address lookup
 ##
-from http import get
 
-from common import warning
+from ..common import warning
+from ..http import get
 
 
 class Plugin(object):
+
     def __init__(self, artifact):
         self.artifact = artifact
         self.artifact['data']['blockchain'] = None
         self.headers = {'User-Agent': 'OSINT Omnibus (https://github.com/InQuest/Omnibus)'}
 
-
     def run(self):
         url = 'https://blockchain.info/rawaddr/%s' % self.artifact['name']
 

lib/modules/censys.py

@@ -4,13 +4,13 @@
 # censys.io module
 ##
 
-from http import get
-
-from common import get_apikey
-from common import warning
+from ..common import get_apikey
+from ..common import warning
+from ..http import get
 
 
 class Plugin(object):
+
     def __init__(self, artifact):
         self.artifact = artifact
         self.artifact['data']['censys'] = None
@@ -19,7 +19,6 @@ def __init__(self, artifact):
             raise TypeError('API keys cannot be left blank | set all keys in etc/apikeys.json')
         self.headers = {'User-Agent': 'OSINT Omnibus (https://github.com/InQuest/Omnibus)'}
 
-
     def run(self):
         url = 'https://censys.io/api/v1/view/ipv4/%s' % self.artifact['name']
 

lib/modules/clearbit.py

@@ -3,13 +3,14 @@
 # omnibus - deadbits.
 # clearbit email lookup module
 ##
-from http import get
 
-from common import warning
-from common import get_apikey
+from ..common import get_apikey
+from ..common import warning
+from ..http import get
 
 
 class Plugin(object):
+
     def __init__(self, artifact):
         self.artifact = artifact
         self.artifact['data']['clearbit'] = None