add service domainbigdata lookup

domainbigdata_service/CHANGELOG

@@ -0,0 +1,4 @@
+Version 0.0.1
+-------------
+
+This is an attempt at integrating a crits service for use domainbigdata (domainbigdata.com) on indicators email/domain & domains.

domainbigdata_service/DEPENDENCIES

@@ -0,0 +1 @@
+bs4

domainbigdata_service/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017, Lionel PRAT. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

domainbigdata_service/README

@@ -0,0 +1,6 @@
+Version 0.0.1
+-------------
+
+This is an attempt at integrating a crits service for use domainbigdata (domainbigdata.com) on indicators email/domain & domains.
+
+

domainbigdata_service/__init__.py

@@ -0,0 +1,114 @@
+# (c) 2017, Lionel PRAT <[email protected]>
+# domainbigdata lookup on indicators email & domaines (https://domainbigdata.com)
+# based on service preview of Adam Polkosnik
+# use modified source code of Roberto Sponchioni - https://github.com/Ptr32Void/OSTrICa/blob/master/ostrica/Plugins/DomainBigData/__init__.py
+# All rights reserved.
+
+import logging
+import os
+import io
+import StringIO
+
+import IndicatorTypes
+
+# for adding the extracted files
+from django.conf import settings
+from django.template.loader import render_to_string
+from crits.services.core import Service, ServiceConfigError
+
+from . import domainbigdata
+
+logger = logging.getLogger(__name__)
+
+
+class DomainBigDataService(Service):
+
+    name = "DomainBigData"
+    version = '0.0.1'
+    supported_types = ['Domain', 'Indicator']
+    description = "DomainBigData lookup on indicators email & domains"
+
+    @staticmethod
+    def valid_for(obj):
+        # Only run on indictors: URI, DOMAIN, IP, FILE HASH
+        # or on IP object == IP lookup
+        # or on domain object == Domain lookup
+        # or sample object == HASH lookup
+        if obj._meta['crits_type'] == 'Indicator' and (obj['ind_type'] == 'Email Address' or obj['ind_type'] == 'Email Address From' or obj['ind_type'] == 'Email Reply-To' or obj['ind_type'] == 'Domain'):
+            return True
+        elif obj._meta['crits_type'] == 'Domain':
+            return True
+        return False
+
+    def run(self, obj, config):
+        self.config = config
+        self.obj = obj
+        user = self.current_task.user
+        #create DomainBigData object
+        self._info('RUN DomainBigData lookup')    
+        dbd = domainbigdata.DomainBigData()
+        result = None
+        value_obj = None
+        if (obj._meta['crits_type'] == 'Indicator' and obj['ind_type'] == 'Domain') or obj._meta['crits_type'] == 'Domain':
+            if obj._meta['crits_type'] == 'Domain':
+                value_obj = obj.domain
+            else:
+                value_obj = obj['value']
+            #run domain information
+            try:
+                self._info('Send request type domain on DomainBigData')
+                result = dbd.domain_information(value_obj, self)
+            except Exception as e:
+                self._error('Query DomainBigData error:' + str(e))
+        elif (obj._meta['crits_type'] == 'Indicator') and (obj['ind_type'] == 'Email Address' or obj['ind_type'] == 'Email Address From' or obj['ind_type'] == 'Email Reply-To' or obj['ind_type'] == 'Domain'):
+            value_obj = obj['value']
+            #email indicator
+            try:
+                self._info('Send request type email on DomainBigData')
+                result = dbd.email_information(value_obj, self)
+            except Exception as e:
+                self._error('Query DomainBigData error:' + str(e))
+        else:
+            self._error('This object type cannot use service DomainBigData lookup.')
+            return False
+        if not result:
+            self._info('Result is empty')
+            return
+        self._info('Processing results:' + str(result))
+        #self._info(str(result))
+        #add result
+        if type(result) is dict:
+            for k,v in result.iteritems():
+                if not type(v) is dict:
+                        if type(v) is list:
+                            count=1
+                            for item in v:
+                                if type(item) is dict:
+                                    self._add_result('Result of DomainBigData on ' + value_obj, k+' -> '+str(count), item)
+                                elif type(item) is list:
+                                     self._add_result('Result of DomainBigData on ' + value_obj, k+' -> '+str(count), {'value': str(item)})
+                                else:
+                                     self._add_result('Result of DomainBigData on ' + value_obj, k+' -> '+str(count), {'value': item})
+                                count+=1
+                        else:
+                            self._add_result('Result of DomainBigData on ' + value_obj, k, {'value': v})
+        if type(result) is dict:
+            for k,v in result.iteritems():
+                if type(v) is dict:
+                    for kx,vx in v.iteritems():
+                        if type(vx) is dict:
+                            self._add_result('Result of DomainBigData on ' + value_obj + ' -- Result: ' + k, kx, {'value': vx})
+                        elif type(vx) is list:
+                            count=1
+                            for item in vx:
+                                if type(item) is dict:
+                                    self._add_result('Result of DomainBigData on ' + value_obj + ' -- Result: ' + k, kx+' -> '+str(count), item)
+                                elif type(item) is list:
+                                     self._add_result('Result of DomainBigData on ' + value_obj + ' -- Result: ' + k, kx+' -> '+str(count), {'value': str(item)})
+                                else:
+                                     self._add_result('Result of DomainBigData on ' + value_obj + ' -- Result: ' + k, kx+' -> '+str(count), {'value': item})
+                                count+=1
+                        else:
+                            self._add_result('Result of DomainBigData on ' + value_obj + ' -- Result: ' + k, kx, {'value': vx})
+        self._info('END')
+

domainbigdata_service/bootstrap

@@ -0,0 +1,147 @@
+#!/bin/sh
+# (c) 2016, The MITRE Corporation. All rights reserved.
+# Source code distributed pursuant to license agreement.
+#
+# Usage: bootstrap
+# This script is designed to install all of the necessary dependencies for the
+# service.
+
+. ../funcs.sh
+
+ubuntu_install()
+{
+    printf "${HEAD}Installing dependencies with apt-get${END}\n"
+    #sudo apt-add-repository universe
+    #sudo apt-get update
+    #sudo apt-get install -y --fix-missing phantomjs
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Ubuntu Install Complete${END}\n"
+    else
+      printf "${FAIL}Ubuntu Install Failed${END}\n"
+    fi
+    sudo ldconfig 
+}
+
+debian_install()
+{
+    printf "${HEAD}Installing dependencies with apt-get${END}\n"
+    #sudo apt-add-repository universe
+    #sudo apt-get update
+    #sudo apt-get install -y --fix-missing phantomjs
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Debian Install Complete${END}\n"
+    else
+      printf "${FAIL}Debian Install Failed${END}\n"
+    fi
+    sudo ldconfig
+}
+
+darwin_install()
+{
+    command -v brew >/dev/null 2>&1 || {
+    printf "${HEAD}Installation for OSX requires Homebrew. Please visit http://brew.sh/.${END}\n"
+    exit
+    }
+    #brew install chmlib clamav wireshark upx
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Homebrew Install Complete${END}\n"
+    else
+      printf "${FAIL}Homebrew Install Failed${END}\n"
+    fi
+}
+
+freebsd_install()
+{
+    #printf "${HEAD}Installing Ports${END}\n"
+    #sudo pkg install phantomjs
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Ports Install Complete${END}\n"
+    else
+      printf "${FAIL}Ports Install Failed${END}\n"
+    fi
+}
+
+red_hat_install()
+{
+    #printf "${HEAD}Installing Yum Packages${END}\n"
+    #sudo yum install phantomjs
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Yum Install Complete${END}\n"
+    else
+      printf "${FAIL}Yum Install Failed${END}\n"
+    fi
+}
+
+centos_install()
+{
+    #printf "${HEAD}Installing Yum Packages${END}\n"
+    #sudo yum install phantomjs
+    if [ $? -eq 0 ]
+    then
+      printf "${PASS}Yum Install Complete${END}\n"
+    else
+      printf "${FAIL}Yum Install Failed${END}\n"
+    fi
+}
+#===============================================================
+# This is the Beginning of the Script
+#===============================================================
+# Sees if there is an argument
+if [ -z $1 ];
+then
+    STEP=1
+else
+    STEP=$1
+fi
+
+while [ $STEP -lt 2 ]
+do
+    case $STEP in
+        1)
+            verify
+            if [ "$OS" = 'ubuntu' ]
+            then
+                #printf "${PASS}ubuntu is Supported!${END}\n"
+                ubuntu_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            elif [ "$OS" = 'debian' ]
+            then
+                #printf "${PASS}Debian is Supported!${END}\n"
+                debian_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            elif [ "$OS" = 'darwin' ]
+            then
+                #printf "${PASS}OS X is Supported!${END}\n"
+                darwin_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            elif [ "$OS" = "centos" ]
+            then
+                #printf "${PASS}CentOS is Supported!${END}\n"
+                centos_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            elif [ "$OS" = "red hat" ]
+            then
+                #printf "${PASS}Red Hat is Supported!${END}\n"
+                red_hat_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            elif [ "$OS" = 'freebsd' ]
+            then
+                #printf "${PASS}FreeBSD is Supported${END}\n"
+                freebsd_install || exit_restart $STEP
+                depend_crits ||exit_restart $STEP
+            else
+                printf "${FAIL}OS: $OS, need Ubuntu, Debian, Darwin (OS X), CentOS, Red Hat, or FreeBSD${END}\n"
+                exit
+            fi
+            ;;
+        *)
+            exit
+            ;;
+    esac
+    STEP=$((STEP+1))
+done