diff --git a/.travis.yml b/.travis.yml index 8bbbe010a..a2296e311 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ before_script: - python manage.py db upgrade script: + - sh env_tests/test_dart.sh - py.test security_monkey/tests || exit 1 notifications: diff --git a/dart/pubspec.yaml b/dart/pubspec.yaml index ecec8dea7..22a5dff96 100644 --- a/dart/pubspec.yaml +++ b/dart/pubspec.yaml @@ -1,6 +1,6 @@ name: security_monkey description: An AWS Policy Monitoring and Alerting Tool -version: 0.4.0 +version: 0.4.1 dependencies: angular: ">=1.1.0 <2.0.0" angular_ui: '0.6.8' diff --git a/docs/api/index.rst b/docs/api/index.rst index c8991e1fa..1743e3379 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -28,7 +28,7 @@ included, and users are free to add their own rules. >>> import security_monkey >>> security_monkey.__version__ - u'0.4.0' + u'0.4.1' Class and method level definitions and documentation diff --git a/docs/changelog.rst b/docs/changelog.rst index 481a49a78..171b0d295 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,28 @@ Changelog ********* -v0.4.0 (2016-11-20) +v0.4.1 (2015-12-22) +=================== +- PR #269 - mikegrima - TravisCI now ensures that dart builds. +- PR #270 - monkeysecurity - Refactored sts_connect to dynamically import boto resources. +- PR #271 - OllyTheNinja-Xero - Fixed indentation mistake in auditor.py +- PR #275 - AlexCline - Added elb logging to ELB watcher and auditor. +- PR #279 - mikegrima - Added ElasticSearch Watcher and Auditor (with tests). +- PR #280 - monkeysecurity - PolicyDiff better handling of changes to primitives (like ints) in dictionay values and added explicit escaping instead of relying on Angular. +- PR #282 - mikegrima - Documentation Fixes to configuration.rst and quickstart.rst adding es: permissions and other fixes. + +Hotfixes: + +- Added OSSMETADATA file to master/develop for internal Netflix tracking. + +Contributors: + +- @mikegrima +- @monkeysecurity +- @OllyTheNinja-Xero +- @AlexCline + +v0.4.0 (2015-11-20) =================== - PR #228 - jeremy-h - IAM check misses '*' when found within a list. (Issue #223) - PR #230 - markofu - New error and echo functions to simplify code for scripts/secmonkey_auto_install.sh diff --git a/docs/configuration.rst b/docs/configuration.rst index 4e96848db..b315745ab 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -143,7 +143,9 @@ SM-ReadOnly "sqs:ListQueues", "sqs:ReceiveMessage", "storagegateway:List*", - "storagegateway:Describe*" + "storagegateway:Describe*", + "es:Describe*", + "es:List*" ], "Effect": "Allow", "Resource": "*" diff --git a/docs/quickstart.rst b/docs/quickstart.rst index fe90affb3..acfb8af4b 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -50,7 +50,7 @@ Paste in this JSON with the name "SecurityMonkeyLaunchPerms": { "Effect": "Allow", "Action": "sts:AssumeRole", - "Resource": "*" + "Resource": "arn:aws:iam::*:role/SecurityMonkey" } ] } @@ -143,7 +143,9 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly": "sns:listtopics", "sqs:getqueueattributes", "sqs:listqueues", - "sqs:receivemessage" + "sqs:receivemessage", + "es:Describe*, + "es:List*" ], "Effect": "Allow", "Resource": "*" @@ -231,9 +233,9 @@ Now may also be a good time to edit the "launch-wizard-1" security group to rest Keypair ------- -You may be prompted to download a keypair. You should protect this keypair; it is used to provide ssh access to the new instance. Put it in a safe place. You will need to change the permissions on the keypair to 600:: +You may be prompted to download a keypair. You should protect this keypair; it is used to provide ssh access to the new instance. Put it in a safe place. You will need to change the permissions on the keypair to 400:: - $ chmod 600 SecurityMonkeyKeypair.pem + $ chmod 400 SecurityMonkeyKeypair.pem Connecting to your new instance: -------------------------------- @@ -315,6 +317,7 @@ Next we'll clone and install the package:: # Build the Web UI $ cd /usr/local/src/security_monkey/dart + $ sudo /usr/lib/dart/bin/pub get $ sudo /usr/lib/dart/bin/pub build # Copy the compiled Web UI to the appropriate destination diff --git a/env_tests/test_dart.sh b/env_tests/test_dart.sh new file mode 100644 index 000000000..e8e529d0c --- /dev/null +++ b/env_tests/test_dart.sh @@ -0,0 +1,37 @@ +#!/bin/bash +########################################## +# Copyright 2015 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################## +# +# Script to test Dart installation for Security Monkey +# +########################################## + +DART_DOWNLOAD_LOCATION="https://storage.googleapis.com/dart-archive/channels/stable/release/1.12.2/sdk/dartsdk-linux-x64-release.zip" + +echo "Getting Dart..." +wget $DART_DOWNLOAD_LOCATION -O 'dartsdk-linux-x64-release.zip' + +echo "Unzipping Dart..." +unzip "dartsdk-linux-x64-release.zip" > /dev/null + +echo Setting up the environment variables +export DART_SDK="$PWD/dart-sdk" +export PATH="$DART_SDK/bin:$PATH" + +echo "Building the dart deps..." +cd dart +pub get +pub build diff --git a/security_monkey/auditor.py b/security_monkey/auditor.py index 42101bef2..5e28ceee6 100644 --- a/security_monkey/auditor.py +++ b/security_monkey/auditor.py @@ -59,8 +59,7 @@ def __init__(self, accounts=None, debug=False): for account in self.accounts: users = User.query.filter(User.daily_audit_email==True).filter(User.accounts.any(name=account)).all() - - self.emails.extend([user.email for user in users]) + self.emails.extend([user.email for user in users]) def add_issue(self, score, issue, item, notes=None): """ diff --git a/security_monkey/auditors/elasticsearch_service.py b/security_monkey/auditors/elasticsearch_service.py new file mode 100644 index 000000000..cedcd728f --- /dev/null +++ b/security_monkey/auditors/elasticsearch_service.py @@ -0,0 +1,181 @@ +# Copyright 2015 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.auditors.elb + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Mike Grima + +""" +from security_monkey.auditor import Auditor +from security_monkey.common.arn import ARN +from security_monkey.datastore import NetworkWhitelistEntry +from security_monkey.watchers.elasticsearch_service import ElasticSearchService + +import ipaddr + + +class ElasticSearchServiceAuditor(Auditor): + index = ElasticSearchService.index + i_am_singular = ElasticSearchService.i_am_singular + i_am_plural = ElasticSearchService.i_am_plural + + def __init__(self, accounts=None, debug=False): + super(ElasticSearchServiceAuditor, self).__init__(accounts=accounts, debug=debug) + + def prep_for_audit(self): + self.network_whitelist = NetworkWhitelistEntry.query.all() + + def _parse_arn(self, arn_input, account_numbers, es_domain): + if arn_input == '*': + notes = "An ElasticSearch Service domain policy where { 'Principal': { 'AWS': '*' } } must also have" + notes += " a {'Condition': {'IpAddress': { 'AWS:SourceIp': '' } } }" + notes += " or it is open to any AWS account." + self.add_issue(20, 'ES cluster open to all AWS accounts', es_domain, notes=notes) + return + + arn = ARN(arn_input) + if arn.error: + self.add_issue(3, 'Auditor could not parse ARN', es_domain, notes=arn_input) + return + + if arn.tech == 's3': + notes = "The ElasticSearch Service domain allows access from S3 bucket [{}]. ".format(arn.name) + notes += "Security Monkey does not yet have the capability to determine if this is " + notes += "a friendly S3 bucket. Please verify manually." + self.add_issue(3, 'ES cluster allows access from S3 bucket', es_domain, notes=notes) + else: + account_numbers.append(arn.account_number) + + def check_es_access_policy(self, es_domain): + policy = es_domain.config["policy"] + + for statement in policy.get("Statement", []): + effect = statement.get("Effect") + # We only care about "Allows" + if effect.lower() == "deny": + continue + + account_numbers = [] + princ = statement.get("Principal", {}) + if isinstance(princ, dict): + princ_aws = princ.get("AWS", "error") + else: + princ_aws = princ + + if princ_aws == "*": + condition = statement.get('Condition', {}) + + # Get the IpAddress subcondition: + ip_addr_condition = condition.get("IpAddress") + + if ip_addr_condition: + source_ip_condition = ip_addr_condition.get("aws:SourceIp") + + if not ip_addr_condition or not source_ip_condition: + tag = "ElasticSearch Service domain open to everyone" + notes = "An ElasticSearch Service domain policy where { 'Principal': { '*' } } OR" + notes += " { 'Principal': { 'AWS': '*' } } must also have a" + notes += " {'Condition': {'IpAddress': { 'AWS:SourceIp': '' } } }" + notes += " or it is open to the world. In this case, anyone is allowed to perform " + notes += " this action(s): {}".format(statement.get("Action")) + self.add_issue(20, tag, es_domain, notes=notes) + + else: + # Check for "aws:SourceIp" as a condition: + if isinstance(source_ip_condition, list): + for cidr in source_ip_condition: + self._check_proper_cidr(cidr, es_domain, statement.get("Action")) + + else: + self._check_proper_cidr(source_ip_condition, es_domain, statement.get("Action")) + + else: + if isinstance(princ_aws, list): + for entry in princ_aws: + arn = ARN(entry) + if arn.error: + self.add_issue(3, 'Auditor could not parse ARN', es_domain, notes=entry) + continue + + if arn.root: + message = "ALL IAM Roles/users/groups in account can perform the following actions:\n" + message += "{}".format(statement.get("Action")) + self.add_issue(3, message, es_domain, notes="Root IAM ARN") + + account_numbers.append(arn.account_number) + else: + arn = ARN(princ_aws) + if arn.error: + self.add_issue(3, 'Auditor could not parse ARN', es_domain, notes=princ_aws) + else: + if arn.root: + message = "ALL IAM Roles/users/groups in account can perform the following actions:\n" + message += "{}".format(statement.get("Action")) + self.add_issue(3, message, es_domain, notes="Root IAM ARN") + + account_numbers.append(arn.account_number) + + for account_number in account_numbers: + self._check_cross_account(account_number, es_domain, 'policy') + + def _check_proper_cidr(self, cidr, es_domain, actions): + try: + any, ip_cidr = self._check_for_any_ip(cidr, es_domain, actions) + if any: + return + + if not self._check_inclusion_in_network_whitelist(cidr): + message = "A CIDR that is not in the whitelist has access to this ElasticSearch Service domain:\n" + message += "CIDR: {}, Actions: {}".format(cidr, actions) + self.add_issue(5, message, es_domain, notes=cidr) + + # Check if the CIDR is in a large subnet (and not whitelisted): + # Check if it's 10.0.0.0/8 + if ip_cidr == ipaddr.IPNetwork("10.0.0.0/8"): + message = "aws:SourceIp Condition contains a very large IP range: 10.0.0.0/8" + self.add_issue(7, message, es_domain, notes=cidr) + else: + mask = int(ip_cidr.exploded.split('/')[1]) + if 0 < mask < 24: + message = "aws:SourceIp contains a large IP Range: {}".format(cidr) + self.add_issue(3, message, es_domain, notes=cidr) + + + except ValueError as ve: + self.add_issue(3, 'Auditor could not parse CIDR', es_domain, notes=cidr) + + def _check_for_any_ip(self, cidr, es_domain, actions): + if cidr == '*': + self.add_issue(20, 'Any IP can perform the following actions against this ElasticSearch Service ' + 'domain:\n{}'.format(actions), + es_domain, notes=cidr) + return True, None + + zero = ipaddr.IPNetwork("0.0.0.0/0") + ip_cidr = ipaddr.IPNetwork(cidr) + if zero == ip_cidr: + self.add_issue(20, 'Any IP can perform the following actions against this ElasticSearch Service ' + 'domain:\n{}'.format(actions), + es_domain, notes=cidr) + return True, None + + return False, ip_cidr + + def _check_inclusion_in_network_whitelist(self, cidr): + for entry in self.network_whitelist: + if ipaddr.IPNetwork(cidr) in ipaddr.IPNetwork(str(entry.cidr)): + return True + return False diff --git a/security_monkey/auditors/elb.py b/security_monkey/auditors/elb.py index ee17a9e52..246f4dfea 100644 --- a/security_monkey/auditors/elb.py +++ b/security_monkey/auditors/elb.py @@ -178,6 +178,14 @@ def check_listener_reference_policy(self, elb_item): if not reference_policy: self._process_custom_listener_policy(policy, listener['load_balancer_port'], elb_item) + def check_logging(self, elb_item): + """ + Alert when elb logging is not enabled + """ + logging = elb_item.config.get('is_logging') + if not logging: + self.add_issue(1, 'ELB is not configured for logging.', elb_item) + def _process_reference_policy(self, reference_policy, policy_name, port, elb_item): notes = "Policy {0} on port {1}".format(policy_name, port) if reference_policy is None: diff --git a/security_monkey/common/PolicyDiff.py b/security_monkey/common/PolicyDiff.py index 619cd8d08..0f4ca82d5 100644 --- a/security_monkey/common/PolicyDiff.py +++ b/security_monkey/common/PolicyDiff.py @@ -28,6 +28,11 @@ import json import sys import collections +from cgi import escape as cgi_escape + + +def escape(data): + return cgi_escape(unicode(data)) def i(indentlevel): @@ -45,9 +50,9 @@ def i(indentlevel): # Type Change # Regular Change # DELETED -def processSubDict(key, sda, sdb, indentlevel): +def process_sub_dict(key, sda, sdb, indentlevel): if type(sda) is not type(sdb): - raise ValueError("processSubDict requires that both items have the same type.") + raise ValueError("process_sub_dict requires that both items have the same type.") # BUG: What if going from None to 'vpc-1de23c' retstr = '' @@ -63,20 +68,18 @@ def processSubDict(key, sda, sdb, indentlevel): retstr += same("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentlevel))) else: retstr += deleted("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentlevel))) - retstr += added("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentlevel))) + retstr += added("{2}\"{0}\": {1},".format(key, json.dumps(sdb), i(indentlevel))) elif type(sda) is dict: retstr += same("{4}\"{0}\": {2}
\n{1}{4}{3},".format(key, diffdict(sda, sdb, indentlevel+1), brackets[0], brackets[1], i(indentlevel))) elif type(sda) is list: retstr += same("{4}\"{0}\": {2}
\n{1}{4}{3},".format(key, difflist(sda, sdb, indentlevel+1), brackets[0], brackets[1], i(indentlevel))) else: - print "processSubDict - Unexpected diffdict type {}".format(type(sda)) + print "process_sub_dict - Unexpected diffdict type {}".format(type(sda)) return retstr def formbrack(value, indentlevel): - brackets = {} - brackets['open'] = '' - brackets['close'] = '' + brackets = {'open': '', 'close': ''} if type(value) is str or type(value) is unicode: brackets['open'] = '"' @@ -99,7 +102,7 @@ def printlist(structure, action, indentlevel): brackets = formbrack(value, indentlevel) new_value = "" if type(value) is str or type(value) is unicode: - new_value = value + new_value = escape(value) elif type(value) is dict: new_value = printdict(value, action, indentlevel+1) elif type(value) is list: @@ -115,7 +118,7 @@ def printlist(structure, action, indentlevel): retstr += deleted(content) elif action is 'added': retstr += added(content) - return removeLastComma(retstr) + return remove_last_comma(retstr) def printdict(structure, action, indentlevel): @@ -125,7 +128,7 @@ def printdict(structure, action, indentlevel): brackets = formbrack(value, indentlevel) new_value = '' if type(value) is str or type(value) is unicode or type(value) is int or type(value) is float: - new_value = value + new_value = escape(value) elif type(value) is bool or type(value) is type(None): new_value = json.dumps(value) elif type(value) is dict: @@ -135,7 +138,13 @@ def printdict(structure, action, indentlevel): else: print "printdict - Unexpected diffdict type {}".format(type(value)) - content = "{4}\"{0}\": {2}{1}{3},".format(key, new_value, brackets['open'], brackets['close'], i(indentlevel)) + content = "{4}\"{0}\": {2}{1}{3},".format( + escape(key), + new_value, + brackets['open'], + brackets['close'], + i(indentlevel) + ) if action is 'same': retstr += same(content) @@ -143,12 +152,12 @@ def printdict(structure, action, indentlevel): retstr += deleted(content) elif action is 'added': retstr += added(content) - return removeLastComma(retstr) + return remove_last_comma(retstr) def printsomething(value, action, indentlevel): if type(value) is str or type(value) is unicode or type(value) is int or type(value) is float: - return value + return escape(value) elif type(value) is bool or type(value) is type(None): new_value = json.dumps(value) elif type(value) is dict: @@ -180,7 +189,7 @@ def diffdict(dicta, dictb, indentlevel): brackets = charfortype(dicta[keya]) retstr += added("{4}\"{0}\": {2}{1}{3},".format(keya, dicta[keya], brackets[0], brackets[1], i(indentlevel))) else: - retstr += processSubDict(keya, dicta[keya], dictb[keya], indentlevel) + retstr += process_sub_dict(keya, dicta[keya], dictb[keya], indentlevel) for keyb in dictb.keys(): if not keyb in dicta: brackets = charfortype(dictb[keyb]) @@ -188,13 +197,12 @@ def diffdict(dicta, dictb, indentlevel): retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(keyb, printsomething(dictb[keyb], 'deleted', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) if type(dictb[keyb]) is list or type(dictb[keyb]) is dict: retstr += deleted("{4}\"{0}\": {2}
\n{1}{4}{3},".format(keyb, printsomething(dictb[keyb], 'deleted', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) - return removeLastComma(retstr) + return remove_last_comma(retstr) -def removeLastComma(str): +def remove_last_comma(str): position = str.rfind(',') - retstr = str[:position] + str[position+1:] - return retstr + return str[:position] + str[position+1:] def difflist(lista, listb, indentlevel): @@ -226,7 +234,7 @@ def difflist(lista, listb, indentlevel): if item in listb: brackets = charfortype(item) if type(item) is str or type(item) is unicode: - retstr += same("{3}{1}{0}{2},".format(item, brackets[0], brackets[1], i(indentlevel))) + retstr += same("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) else: # Handle lists and dicts here: diffstr = '' @@ -243,7 +251,7 @@ def difflist(lista, listb, indentlevel): brackets = charfortype(item) if None is bestmatch: if type(item) is str or type(item) is unicode: - retstr += added("{3}{1}{0}{2},".format(item, brackets[0], brackets[1], i(indentlevel))) + retstr += added("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) else: # Handle lists and dicts here: diffstr = '' @@ -252,8 +260,8 @@ def difflist(lista, listb, indentlevel): retstr += added("{3}{1}
\n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) else: if type(item) is str or type(item) is unicode: - retstr += deleted("{3}{1}{0}{2},".format(bestmatch, brackets[0], brackets[1], i(indentlevel))) - retstr += added("{3}{1}{0}{2},".format(item, brackets[0], brackets[1], i(indentlevel))) + retstr += deleted("{3}{1}{0}{2},".format(escape(bestmatch), brackets[0], brackets[1], i(indentlevel))) + retstr += added("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) else: # Handle lists and dicts here: diffstr = '' @@ -266,14 +274,14 @@ def difflist(lista, listb, indentlevel): for item in deletedlist: brackets = charfortype(item) if type(item) is str or type(item) is unicode: - retstr += deleted("{3}{1}{0}{2},".format(item, brackets[0], brackets[1], i(indentlevel))) + retstr += deleted("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) else: # Handle lists and dicts here: diffstr = '' if type(item) is list or type(item) is dict: diffstr = printsomething(item, 'deleted', indentlevel+1) retstr += deleted("{3}{1}
\n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) - return removeLastComma(retstr) + return remove_last_comma(retstr) # levenshtein - http://hetland.org/coding/python/levenshtein.py @@ -291,7 +299,7 @@ def strdistance(a, b): add, delete = previous[j]+1, current[j-1]+1 change = previous[j-1] if a[j-1] != b[i-1]: - change = change + 1 + change += 1 current[j] = min(add, delete, change) return current[n] @@ -419,5 +427,5 @@ def produceDiffHTML(self): } """ - pdiddy = PolicyDiff(old_pol, new_pol) + pdiddy = PolicyDiff(new_pol, old_pol) print pdiddy.produceDiffHTML() diff --git a/security_monkey/common/arn.py b/security_monkey/common/arn.py index 81b308805..ee44bb4ba 100644 --- a/security_monkey/common/arn.py +++ b/security_monkey/common/arn.py @@ -31,10 +31,14 @@ class ARN(object): name = None partition = None error = False + root = False def __init__(self, input): arn_match = re.search('^arn:([^:]*):([^:]*):([^:]*):(|[\d]{12}):(.+)$', input) if arn_match: + if arn_match.group(2) == "iam" and arn_match.group(5) == "root": + self.root = True + return self._from_arn(arn_match, input) acct_number_match = re.search('^(\d{12})+$', input) diff --git a/security_monkey/common/sts_connect.py b/security_monkey/common/sts_connect.py index 5c76d6aa3..dbb7fa15b 100644 --- a/security_monkey/common/sts_connect.py +++ b/security_monkey/common/sts_connect.py @@ -20,17 +20,10 @@ """ from security_monkey.datastore import Account -import boto -import boto.ec2 -import boto.ses -import boto.iam -import boto.sns -import boto.sqs -import boto.rds -import boto.redshift -import boto.vpc import botocore.session import boto3 +import boto + def connect(account_name, connection_type, **args): """ @@ -69,62 +62,6 @@ def connect(account_name, connection_type, **args): ) return botocore_session - if connection_type == 'ec2': - return boto.connect_ec2( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'elb': - if 'region' in args: - region = args['region'] - del args['region'] - else: - region = 'us-east-1' - - return boto.ec2.elb.connect_to_region( - region, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 's3': - if 'region' in args: - region = args['region'] - # drop region key-val pair from args or you'll get an exception - del args['region'] - return boto.s3.connect_to_region( - region, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_s3( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'ses': - if 'region' in args: - region = args['region'] - del args['region'] - return boto.ses.connect_to_region( - region, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_ses( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - if connection_type == 'iam_boto3': session = boto3.Session( aws_access_key_id=role.credentials.access_key, @@ -133,122 +70,29 @@ def connect(account_name, connection_type, **args): ) return session.resource('iam') - if connection_type == 'iam': - if 'region' in args: - region = args['region'] - # drop region key-val pair from args or you'll get an exception - del args['region'] - return boto.iam.connect_to_region( - region, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_iam( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'route53': - return boto.connect_route53( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'sns': - if 'region' in args: - region = args['region'] - del args['region'] - return boto.sns.connect_to_region( - region.name, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_sns( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'sqs': - if 'region' in args: - region = args['region'] - del args['region'] - return boto.sqs.connect_to_region( - region.name, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_sqs( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'vpc': - return boto.connect_vpc( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'rds': - if 'region' in args: - reg = args['region'] - rds_region = None - for boto_region in boto.rds.regions(): - if reg.name == boto_region.name: - rds_region = boto_region - - if rds_region is None: - raise Exception('The supplied region {0} is not in boto.rds.regions. {1}'.format(reg, boto.rds.regions())) - - return boto.connect_rds( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'redshift': - if 'region' in args: - region = args['region'] - del args['region'] - return boto.redshift.connect_to_region( - region.name, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) + region = 'us-east-1' + if args.has_key('region'): + region = args.pop('region') + if hasattr(region, 'name'): + region = region.name - return boto.connect_redshift( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - if connection_type == 'vpc': - if 'region' in args: - region = args['region'] - del args['region'] - return boto.vpc.connect_to_region( - region.name, - aws_access_key_id=role.credentials.access_key, - aws_secret_access_key=role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - return boto.connect_vpc( - role.credentials.access_key, - role.credentials.secret_key, - security_token=role.credentials.session_token, - **args) - - err_msg = 'The connection_type supplied (%s) is not implemented.' % connection_type - raise Exception(err_msg) + # ElasticSearch Service: + if connection_type == 'es': + session = boto3.Session( + aws_access_key_id=role.credentials.access_key, + aws_secret_access_key=role.credentials.secret_key, + aws_session_token=role.credentials.session_token, + region_name=region + ) + return session.client('es') + + module = __import__("boto.{}".format(connection_type)) + for subm in connection_type.split('.'): + module = getattr(module, subm) + + return module.connect_to_region( + region, + aws_access_key_id=role.credentials.access_key, + aws_secret_access_key=role.credentials.secret_key, + security_token=role.credentials.session_token + ) diff --git a/security_monkey/monitors.py b/security_monkey/monitors.py index a6b05dbc8..a507377d9 100644 --- a/security_monkey/monitors.py +++ b/security_monkey/monitors.py @@ -40,6 +40,8 @@ from security_monkey.watchers.vpc.vpc import VPC from security_monkey.watchers.vpc.subnet import Subnet from security_monkey.watchers.vpc.route_table import RouteTable +from security_monkey.watchers.elasticsearch_service import ElasticSearchService +from security_monkey.auditors.elasticsearch_service import ElasticSearchServiceAuditor class Monitor(object): @@ -89,7 +91,9 @@ def has_auditor(self): RouteTable.index: Monitor(RouteTable.index, RouteTable, None), ManagedPolicy.index: - Monitor(ManagedPolicy.index, ManagedPolicy, ManagedPolicyAuditor) + Monitor(ManagedPolicy.index, ManagedPolicy, ManagedPolicyAuditor), + ElasticSearchService.index: + Monitor(ElasticSearchService.index, ElasticSearchService, ElasticSearchServiceAuditor) } diff --git a/security_monkey/tests/test_arn.py b/security_monkey/tests/test_arn.py index 9fee60e21..f7e5aa386 100644 --- a/security_monkey/tests/test_arn.py +++ b/security_monkey/tests/test_arn.py @@ -11,6 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +.. module: security_monkey.tests.test_arn + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Mike Grima + +""" from security_monkey.common.arn import ARN from security_monkey.tests import SecurityMonkeyTestCase from security_monkey import app @@ -39,6 +47,10 @@ def test_from_arn(self): arn_obj = ARN(arn) self.assertFalse(arn_obj.error) + if "root" in arn: + self.assertTrue(arn_obj.root) + else: + self.assertFalse(arn_obj.root) bad_arns = [ 'arn:aws:iam::012345678910', diff --git a/security_monkey/tests/test_elasticsearch_service.py b/security_monkey/tests/test_elasticsearch_service.py new file mode 100644 index 000000000..64cd51e47 --- /dev/null +++ b/security_monkey/tests/test_elasticsearch_service.py @@ -0,0 +1,333 @@ +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.tests.test_elasticsearch_service + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Mike Grima + +""" +import json + +from security_monkey.datastore import NetworkWhitelistEntry +from security_monkey.tests import SecurityMonkeyTestCase + +# TODO: Make a ES test for spulec/moto, then make test cases that use it. +from security_monkey.watchers.elasticsearch_service import ElasticSearchServiceItem + +CONFIG_ONE = { + "name": "es_test", + "policy": json.loads(b"""{ + "Statement": [ + { + "Action": "es:*", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Resource": "arn:aws:es:us-east-1:012345678910:domain/es_test/*", + "Sid": "" + } + ], + "Version": "2012-10-17" + } + """) +} + +CONFIG_TWO = { + "name": "es_test_2", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:*", + "Resource": "arn:aws:es:us-west-2:012345678910:domain/es_test_2/*" + } + ] + } + """) +} + +CONFIG_THREE = { + "name": "es_test_3", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::012345678910:root" + }, + "Action": "es:*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_3/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:ESHttp*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_3/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "192.168.1.1/32", + "10.0.0.1/8" + ] + } + } + } + ] + } + """) +} + +CONFIG_FOUR = { + "name": "es_test_4", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::012345678910:root" + }, + "Action": "es:*", + "Resource": "arn:aws:es:us-east-1:012345678910:domain/es_test_4/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:ESHttp*", + "Resource": "arn:aws:es:us-east-1:012345678910:domain/es_test_4/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "0.0.0.0/0" + ] + } + } + } + ] + } + """) +} + +CONFIG_FIVE = { + "name": "es_test_5", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::012345678910:root" + }, + "Action": "es:*", + "Resource": "arn:aws:es:us-east-1:012345678910:domain/es_test_5/*" + }, + { + "Sid": "", + "Effect": "Deny", + "Principal": { + "AWS": "arn:aws:iam::012345678910:role/not_this_role" + }, + "Action": "es:*", + "Resource": "arn:aws:es:us-east-1:012345678910:domain/es_test_5/*" + } + ] + } + """) +} + +CONFIG_SIX = { + "name": "es_test_6", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::012345678910:role/a_good_role" + }, + "Action": "es:*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_6/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:ESHttp*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_6/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "192.168.1.1/32", + "100.0.0.1/16" + ] + } + } + } + ] + } + """) +} + +CONFIG_SEVEN = { + "name": "es_test_7", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::012345678910:role/a_good_role" + }, + "Action": "es:*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_7/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:ESHttp*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_7/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "192.168.1.200/32", + "10.0.0.1/8" + ] + } + } + } + ] + } + """) +} + +CONFIG_EIGHT = { + "name": "es_test_8", + "policy": json.loads(b"""{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": "es:*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_8/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": "*", + "Action": "es:ESHttp*", + "Resource": "arn:aws:es:eu-west-1:012345678910:domain/es_test_8/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "192.168.1.1/32", + "100.0.0.1/16" + ] + } + } + } + ] + } + """) +} + +WHITELIST_CIDRS = [ + ("Test one", "192.168.1.1/32"), + ("Test two", "100.0.0.1/16"), +] + + +class ElasticSearchServiceTestCase(SecurityMonkeyTestCase): + def setUp(self): + self.es_items = [ + ElasticSearchServiceItem(region="us-east-1", account="012345678910", name="es_test", config=CONFIG_ONE), + ElasticSearchServiceItem(region="us-west-2", account="012345678910", name="es_test_2", config=CONFIG_TWO), + ElasticSearchServiceItem(region="eu-west-1", account="012345678910", name="es_test_3", config=CONFIG_THREE), + ElasticSearchServiceItem(region="us-east-1", account="012345678910", name="es_test_4", config=CONFIG_FOUR), + ElasticSearchServiceItem(region="us-east-1", account="012345678910", name="es_test_5", config=CONFIG_FIVE), + ElasticSearchServiceItem(region="eu-west-1", account="012345678910", name="es_test_6", config=CONFIG_SIX), + ElasticSearchServiceItem(region="eu-west-1", account="012345678910", name="es_test_7", config=CONFIG_SEVEN), + ElasticSearchServiceItem(region="eu-west-1", account="012345678910", name="es_test_8", config=CONFIG_EIGHT), + ] + + def test_es_auditor(self): + from security_monkey.auditors.elasticsearch_service import ElasticSearchServiceAuditor + es_auditor = ElasticSearchServiceAuditor(accounts=["012345678910"]) + + # Add some test network whitelists into this: + es_auditor.network_whitelist = [] + for cidr in WHITELIST_CIDRS: + whitelist_cidr = NetworkWhitelistEntry() + whitelist_cidr.cidr = cidr[1] + whitelist_cidr.name = cidr[0] + + es_auditor.network_whitelist.append(whitelist_cidr) + + for es_domain in self.es_items: + es_auditor.check_es_access_policy(es_domain) + + # Check for correct number of issues located: + # CONFIG ONE: + self.assertEquals(len(self.es_items[0].audit_issues), 1) + self.assertEquals(self.es_items[0].audit_issues[0].score, 20) + + # CONFIG TWO: + self.assertEquals(len(self.es_items[1].audit_issues), 1) + self.assertEquals(self.es_items[1].audit_issues[0].score, 20) + + # CONFIG THREE: + self.assertEquals(len(self.es_items[2].audit_issues), 3) + self.assertEquals(self.es_items[2].audit_issues[0].score, 3) + self.assertEquals(self.es_items[2].audit_issues[1].score, 5) + self.assertEquals(self.es_items[2].audit_issues[2].score, 7) + + # CONFIG FOUR: + self.assertEquals(len(self.es_items[3].audit_issues), 2) + self.assertEquals(self.es_items[3].audit_issues[0].score, 3) + self.assertEquals(self.es_items[3].audit_issues[1].score, 20) + + # CONFIG FIVE: + self.assertEquals(len(self.es_items[4].audit_issues), 1) + self.assertEquals(self.es_items[4].audit_issues[0].score, 3) + + # CONFIG SIX: + self.assertEquals(len(self.es_items[5].audit_issues), 0) + + # CONFIG SEVEN: + self.assertEquals(len(self.es_items[6].audit_issues), 3) + self.assertEquals(self.es_items[6].audit_issues[0].score, 5) + self.assertEquals(self.es_items[6].audit_issues[1].score, 5) + self.assertEquals(self.es_items[6].audit_issues[2].score, 7) + + # CONFIG EIGHT: + self.assertEquals(len(self.es_items[7].audit_issues), 1) + self.assertEquals(self.es_items[7].audit_issues[0].score, 20) diff --git a/security_monkey/watchers/elasticsearch_service.py b/security_monkey/watchers/elasticsearch_service.py new file mode 100644 index 000000000..e40f607f5 --- /dev/null +++ b/security_monkey/watchers/elasticsearch_service.py @@ -0,0 +1,109 @@ +# Copyright 2015 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.watchers.keypair + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Mike Grima + +""" +import json + +from security_monkey.constants import TROUBLE_REGIONS +from security_monkey.exceptions import BotoConnectionIssue +from security_monkey.watcher import Watcher, ChangeItem +from security_monkey import app + +from boto.ec2 import regions + + +class ElasticSearchService(Watcher): + index = 'elasticsearchservice' + i_am_singular = 'ElasticSearch Service Access Policy' + i_am_singular = 'ElasticSearch Service Access Policies' + + def __init__(self, accounts=None, debug=False): + super(ElasticSearchService, self).__init__(accounts=accounts, debug=debug) + + def slurp(self): + """ + :returns: item_list - list of ElasticSearchService Items + :return: exception_map - A dict where the keys are a tuple containing the + location of the exception and the value is the actual exception + + """ + self.prep_for_slurp() + + item_list = [] + exception_map = {} + for account in self.accounts: + for region in regions(): + try: + if region.name in TROUBLE_REGIONS: + continue + + (client, domains) = self.get_all_es_domains_in_region(account, region) + except Exception as e: + if region.name not in TROUBLE_REGIONS: + exc = BotoConnectionIssue(str(e), 'es', account, region.name) + self.slurp_exception((self.index, account, region.name), exc, exception_map) + continue + + app.logger.debug("Found {} {}".format(len(domains), ElasticSearchService.i_am_plural)) + for domain in domains: + if self.check_ignore_list(domain["DomainName"]): + continue + + # Fetch the policy: + item = self.build_item(domain["DomainName"], client, region.name, account, exception_map) + if item: + item_list.append(item) + + return item_list, exception_map + + def get_all_es_domains_in_region(self, account, region): + from security_monkey.common.sts_connect import connect + client = connect(account, "es", region=region) + app.logger.debug("Checking {}/{}/{}".format(ElasticSearchService.index, account, region.name)) + # No need to paginate according to: client.can_paginate("list_domain_names") + domains = self.wrap_aws_rate_limited_call(client.list_domain_names)["DomainNames"] + + return client, domains + + def build_item(self, domain, client, region, account, exception_map): + config = {} + + try: + domain_config = self.wrap_aws_rate_limited_call(client.describe_elasticsearch_domain_config, + DomainName=domain) + config['policy'] = json.loads(domain_config["DomainConfig"]["AccessPolicies"]["Options"]) + config['name'] = domain + + except Exception as e: + self.slurp_exception((domain, client, region), e, exception_map) + return None + + return ElasticSearchServiceItem(region=region, account=account, name=domain, config=config) + + +class ElasticSearchServiceItem(ChangeItem): + def __init__(self, region=None, account=None, name=None, config={}): + super(ElasticSearchServiceItem, self).__init__( + index=ElasticSearchService.index, + region=region, + account=account, + name=name, + new_config=config + ) diff --git a/security_monkey/watchers/elb.py b/security_monkey/watchers/elb.py index fea3cd9b4..c92bea564 100644 --- a/security_monkey/watchers/elb.py +++ b/security_monkey/watchers/elb.py @@ -119,7 +119,7 @@ def slurp(self): self._setup_botocore(account) for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) - elb_conn = connect(account, 'elb', region=region.name) + elb_conn = connect(account, 'ec2.elb', region=region.name) botocore_client = self.botocore_session.create_client('elb', region_name=region.name) botocore_operation = botocore_client.describe_load_balancer_policies @@ -170,6 +170,9 @@ def slurp(self): elb_map['source_security_group'] = elb.source_security_group.name elb_map['subnets'] = list(elb.subnets) elb_map['vpc_id'] = elb.vpc_id + elb_map['is_logging'] = self.wrap_aws_rate_limited_call( + lambda: elb.get_attributes().access_log.enabled + ) backends = [] for be in elb.backends: diff --git a/setup.py b/setup.py index 1b289b8ae..098dc6740 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name='security_monkey', - version='0.4.0', + version='0.4.1', long_description=__doc__, packages=['security_monkey'], include_package_data=True,