Merge pull request #1678 from edx/feanil/rc-lavash-0

Feanil/rc lavash 0

CHANGELOG.md

@@ -1,10 +1,29 @@
+- Role: edxapp
+  - A new var was added to make it easy ot invalidate the default
+    memcache store to make it easier to invalidate sessions. Updating
+    the edxapp env.json files will result in all users getting logged
+    out.  This is a one time penalty as long as the value of `EDXAPP_DEFAULT_CACHE_VERSION`
+    is not explicitly changed.
+
+- Role: nginx
+  - New html templates for server errors added.
+    Defaults for a ratelimiting static page and server error static page.
+    CMS/LMS are set to use them by default, wording can be changed in the
+    Nginx default vars.
+
+- Role: edxapp
+  - We now have an all caps variable override for celery workers
 - Role: common
   - We now remove the default syslog.d conf file (50-default.conf) this will 
   break people who have hand edited that file.
 
 - Role: edxapp
   - Updated the module store settings to match the new settings format.
 
+- Update, possible breaking change: the edxapp role vars edxapp_lms_env and edxapp_cms_env have
+  been changed to EDXAPP_LMS_ENV and EDXAPP_CMS_ENV to indicate, via our convention,
+  that overridding them is expected.  The default values remain the same.
+
 - Role: analytics-api
   - Added a new role for the analytics-api Django app.  Currently a private repo
 
@@ -29,3 +48,7 @@
 - Role: Mongo
   - Fixed case of variable used in if block that breaks cluster configuration
     by changing mongo_clustered to MONGO_CLUSTERED.
+
+- Role: Edxapp
+  - Added EDXAPP_LMS_AUTH_EXTRA and EDXAPP_CMS_AUTH_EXTRA for passing unique AUTH_EXTRA configurations to the LMS and CMS. 
+    Both variables default to EDXAPP_AUTH_EXTRA for backward compatibility

playbooks/callback_plugins/hipchat_plugin.py

@@ -1,9 +1,14 @@
 import os
-import prettytable
-import hipchat
 import time
-import random
 from ansible import utils
+try:
+    import prettytable
+except ImportError:
+    prettytable = None
+try:
+    import hipchat
+except ImportError:
+    hipchat = None
 
 
 class CallbackModule(object):
@@ -24,30 +29,40 @@ class CallbackModule(object):
     """
 
     def __init__(self):
-
-        if 'HIPCHAT_TOKEN' in os.environ:
-            self.start_time = time.time()
-            self.task_report = []
-            self.last_task = None
-            self.last_task_changed = False
-            self.last_task_count = 0
-            self.last_task_delta = 0
-            self.last_task_start = time.time()
-            self.condensed_task_report = (os.getenv('HIPCHAT_CONDENSED', True) == True)
-            self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
-            self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
-            self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')
-            try:
-                self.hipchat_conn = hipchat.HipChat(token=os.getenv('HIPCHAT_TOKEN'))
-            except Exception as e:
-                utils.warning("Unable to connect to hipchat: {}".format(e))
-            self.hipchat_msg_prefix = os.getenv('HIPCHAT_MSG_PREFIX', '')
-            self.hipchat_msg_color = os.getenv('HIPCHAT_MSG_COLOR', '')
-            self.printed_playbook = False
-            self.playbook_name = None
-            self.enabled = True
-        else:
-            self.enabled = False
+        self.enabled = "HIPCHAT_TOKEN" in os.environ
+        if not self.enabled:
+            return
+
+        # make sure we got our imports
+        if not hipchat:
+            raise ImportError(
+                "The hipchat plugin requires the hipchat Python module, "
+                "which is not installed or was not found."
+            )
+        if not prettytable:
+            raise ImportError(
+                "The hipchat plugin requires the prettytable Python module, "
+                "which is not installed or was not found."
+            )
+        self.start_time = time.time()
+        self.task_report = []
+        self.last_task = None
+        self.last_task_changed = False
+        self.last_task_count = 0
+        self.last_task_delta = 0
+        self.last_task_start = time.time()
+        self.condensed_task_report = (os.getenv('HIPCHAT_CONDENSED', True) == True)
+        self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
+        self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
+        self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')
+        try:
+            self.hipchat_conn = hipchat.HipChat(token=os.getenv('HIPCHAT_TOKEN'))
+        except Exception as e:
+            utils.warning("Unable to connect to hipchat: {}".format(e))
+        self.hipchat_msg_prefix = os.getenv('HIPCHAT_MSG_PREFIX', '')
+        self.hipchat_msg_color = os.getenv('HIPCHAT_MSG_COLOR', '')
+        self.printed_playbook = False
+        self.playbook_name = None
 
     def _send_hipchat(self, message, room=None, from_name=None, color=None, message_format='text'):
 
@@ -221,7 +236,7 @@ def playbook_on_stats(self, stats):
                 summary_output = "<b>{}</b>: <i>{}</i> - ".format(self.hipchat_msg_prefix, host)
                 for summary_item in ['ok', 'changed', 'unreachable', 'failures']:
                     if stats[summary_item] != 0:
-                       summary_output += "<b>{}</b> - {} ".format(summary_item, stats[summary_item])
+                        summary_output += "<b>{}</b> - {} ".format(summary_item, stats[summary_item])
                 summary_all_host_output.append(summary_output)
             self._send_hipchat("<br />".join(summary_all_host_output), message_format='html')
             msg = "<b>{description}</b>: Finished Ansible run for <b><i>{play}</i> in {min:02} minutes, {sec:02} seconds</b><br /><br />".format(

playbooks/callback_plugins/sqs.py

@@ -22,11 +22,12 @@
 import json
 import socket
 try:
+    import boto
+except ImportError:
+    boto = None
+else:
     import boto.sqs
     from boto.exception import NoAuthHandlerFound
-except ImportError:
-    print "Boto is required for the sqs_notify callback plugin"
-    raise
 
 
 class CallbackModule(object):
@@ -47,36 +48,42 @@ class CallbackModule(object):
         - START events
     """
     def __init__(self):
+        self.enable_sqs = 'ANSIBLE_ENABLE_SQS' in os.environ
+        if not self.enable_sqs:
+            return
+
+        # make sure we got our imports
+        if not boto:
+            raise ImportError(
+                "The sqs callback module requires the boto Python module, "
+                "which is not installed or was not found."
+            )
 
         self.start_time = time.time()
 
-        if 'ANSIBLE_ENABLE_SQS' in os.environ:
-            self.enable_sqs = True
-            if not 'SQS_REGION' in os.environ:
-                print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
-                      'not defined in environment'
-                sys.exit(1)
-            self.region = os.environ['SQS_REGION']
-            try:
-                self.sqs = boto.sqs.connect_to_region(self.region)
-            except NoAuthHandlerFound:
-                print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
-                      'to AWS due invalid credentials'
-                sys.exit(1)
-            if not 'SQS_NAME' in os.environ:
-                print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
-                      'defined in environment'
-                sys.exit(1)
-            self.name = os.environ['SQS_NAME']
-            self.queue = self.sqs.create_queue(self.name)
-            if 'SQS_MSG_PREFIX' in os.environ:
-                self.prefix = os.environ['SQS_MSG_PREFIX']
-            else:
-                self.prefix = ''
-
-            self.last_seen_ts = {}
+        if not 'SQS_REGION' in os.environ:
+            print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
+                  'not defined in environment'
+            sys.exit(1)
+        self.region = os.environ['SQS_REGION']
+        try:
+            self.sqs = boto.sqs.connect_to_region(self.region)
+        except NoAuthHandlerFound:
+            print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
+                  'to AWS due invalid credentials'
+            sys.exit(1)
+        if not 'SQS_NAME' in os.environ:
+            print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
+                  'defined in environment'
+            sys.exit(1)
+        self.name = os.environ['SQS_NAME']
+        self.queue = self.sqs.create_queue(self.name)
+        if 'SQS_MSG_PREFIX' in os.environ:
+            self.prefix = os.environ['SQS_MSG_PREFIX']
         else:
-            self.enable_sqs = False
+            self.prefix = ''
+
+        self.last_seen_ts = {}
 
     def runner_on_failed(self, host, res, ignore_errors=False):
         if self.enable_sqs:

playbooks/edx-east/aide.yml

@@ -2,6 +2,9 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - aide
     - role: datadog

playbooks/edx-east/alton.yml

@@ -3,5 +3,8 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - alton

playbooks/edx-east/antivirus.yml

@@ -0,0 +1,12 @@
+- name: Deploy Antivirus Scanner
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - antivirus
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/edx-east/aws.yml

@@ -2,5 +2,8 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - aws

playbooks/edx-east/bastion.yml

@@ -2,5 +2,8 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - bastion

playbooks/edx-east/certs.yml

@@ -2,6 +2,9 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - aws
     - certs

playbooks/edx-east/cluster_rabbitmq.yml

@@ -1,5 +1,5 @@
 
-# ansible-playbook -i ec2.py commoncluster.yml --limit tag_Name_stage-edx-commoncluster -e@/path/to/vars/env-deployment.yml -T 30 --list-hosts
+# ansible-playbook -i ec2.py cluster_rabbitmq.yml --limit tag_Name_stage-edx-commoncluster -e@/path/to/vars/env-deployment.yml -T 30 --list-hosts
 
 - hosts: all
   sudo: True
@@ -28,14 +28,9 @@
   tasks:
     - debug: msg="{{ ansible_ec2_local_ipv4 }}"
       with_items: list.results
-    - shell: echo "rabbit@ip-{{ item|replace('.', '-') }}"
-      when: item != ansible_ec2_local_ipv4
-      with_items: hostvars.keys()
-      register: list
     - command: rabbitmqctl stop_app
-    - command: rabbitmqctl join_cluster {{ item.stdout }}
-      when: item.stdout is defined
-      with_items: list.results
+    - command: rabbitmqctl join_cluster rabbit@ip-{{ hostvars.keys()[0]|replace('.', '-') }}
+      when: hostvars.keys()[0] != ansible_ec2_local_ipv4
     - command: rabbitmqctl start_app
   post_tasks:
     - debug: var="{{ ansible_ec2_instance_id }}"

playbooks/edx-east/common.yml

@@ -2,6 +2,9 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - common
     - role: datadog

playbooks/edx-east/connect_sandbox.yml

@@ -31,7 +31,7 @@
         - "EDXAPP_MONGO_HOSTS: {{ EDXAPP_MONGO_HOSTS }}"
         - "EDXAPP_MONGO_DB_NAME: {{ EDXAPP_MONGO_DB_NAME }}"
         - "EDXAPP_MONGO_USER: {{ EDXAPP_MONGO_USER }}"
-        - "EDXAPP_MONGO_PASS: {{ EDXAPP_MONGO_PASS }}"
+        - "EDXAPP_MONGO_PASSWORD: {{ EDXAPP_MONGO_PASSWORD }}"
       tags: update_edxapp_mysql_host
 
     - name: call update on edx-platform

playbooks/edx-east/create_dbs.yml

@@ -40,6 +40,10 @@
       sudo: yes
       with_items:
         - python-mysqldb
+      # When this is run on jenkins the package will already
+      # exist and can't run as the jenkins user because it
+      # does not have sudo privs.
+      when: ansible_ssh_user != 'jenkins'
     - name: create mysql databases for the edX stack
       mysql_db: >
         db={{ item[0] }}{{ item[1].db_name }}

playbooks/edx-east/demo.yml

@@ -2,6 +2,9 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - demo
     - role: datadog

playbooks/edx-east/devpi.yml

@@ -2,5 +2,8 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - devpi

playbooks/edx-east/discern.yml

@@ -2,6 +2,9 @@
   hosts: all
   sudo: True
   gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - aws
     - role: nginx

playbooks/edx-east/edx_ansible.yml

@@ -1,6 +1,9 @@
 - name: Deploy the edx_ansible role
   hosts: all
   sudo: True
-  gather_facts: False
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
   roles:
     - edx_ansible

playbooks/edx-east/edx_continuous_integration.yml

@@ -12,7 +12,6 @@
       nginx_sites:
       - cms
       - lms
-      - ora
       - xqueue
       - xserver
       - certs
@@ -31,7 +30,6 @@
     - forum
     - { role: "xqueue", update_users: True }
     - xserver
-    - ora
     - certs
     - edx_ansible
     - analytics-api

playbooks/edx-east/edx_provision.yml

@@ -1,7 +1,7 @@
 - name: Create ec2 instance
   hosts: localhost
   connection: local
-  gather_facts: False
+  gather_facts: True
   vars:
     keypair: continuous-integration
     instance_type: t2.medium