diff --git a/.wiki-images/download-certificate.png b/.wiki-images/download-certificate.png
deleted file mode 100644
index e80daba3ffd..00000000000
Binary files a/.wiki-images/download-certificate.png and /dev/null differ
diff --git a/.wiki-images/set-end-date-studio.png b/.wiki-images/set-end-date-studio.png
deleted file mode 100644
index 87ed4c7ac8d..00000000000
Binary files a/.wiki-images/set-end-date-studio.png and /dev/null differ
diff --git a/.wiki-images/wrapping-up-dashboard.png b/.wiki-images/wrapping-up-dashboard.png
deleted file mode 100644
index 01772e8c160..00000000000
Binary files a/.wiki-images/wrapping-up-dashboard.png and /dev/null differ
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e3ff6a43fa0..fbe87057127 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,18 @@
+- Role: edxapp
+ - A new var was added to make it easy ot invalidate the default
+ memcache store to make it easier to invalidate sessions. Updating
+ the edxapp env.json files will result in all users getting logged
+ out. This is a one time penalty as long as the value of `EDXAPP_DEFAULT_CACHE_VERSION`
+ is not explicitly changed.
+
+- Role: nginx
+ - New html templates for server errors added.
+ Defaults for a ratelimiting static page and server error static page.
+ CMS/LMS are set to use them by default, wording can be changed in the
+ Nginx default vars.
+
+- Role: edxapp
+ - We now have an all caps variable override for celery workers
- Role: common
- We now remove the default syslog.d conf file (50-default.conf) this will
break people who have hand edited that file.
@@ -5,6 +20,10 @@
- Role: edxapp
- Updated the module store settings to match the new settings format.
+- Update, possible breaking change: the edxapp role vars edxapp_lms_env and edxapp_cms_env have
+ been changed to EDXAPP_LMS_ENV and EDXAPP_CMS_ENV to indicate, via our convention,
+ that overridding them is expected. The default values remain the same.
+
- Role: analytics-api
- Added a new role for the analytics-api Django app. Currently a private repo
@@ -29,3 +48,7 @@
- Role: Mongo
- Fixed case of variable used in if block that breaks cluster configuration
by changing mongo_clustered to MONGO_CLUSTERED.
+
+- Role: Edxapp
+ - Added EDXAPP_LMS_AUTH_EXTRA and EDXAPP_CMS_AUTH_EXTRA for passing unique AUTH_EXTRA configurations to the LMS and CMS.
+ Both variables default to EDXAPP_AUTH_EXTRA for backward compatibility
diff --git a/playbooks/callback_plugins/hipchat_plugin.py b/playbooks/callback_plugins/hipchat_plugin.py
index 16a2f94a207..63c4716c75b 100644
--- a/playbooks/callback_plugins/hipchat_plugin.py
+++ b/playbooks/callback_plugins/hipchat_plugin.py
@@ -1,9 +1,14 @@
import os
-import prettytable
-import hipchat
import time
-import random
from ansible import utils
+try:
+ import prettytable
+except ImportError:
+ prettytable = None
+try:
+ import hipchat
+except ImportError:
+ hipchat = None
class CallbackModule(object):
@@ -24,30 +29,40 @@ class CallbackModule(object):
"""
def __init__(self):
-
- if 'HIPCHAT_TOKEN' in os.environ:
- self.start_time = time.time()
- self.task_report = []
- self.last_task = None
- self.last_task_changed = False
- self.last_task_count = 0
- self.last_task_delta = 0
- self.last_task_start = time.time()
- self.condensed_task_report = (os.getenv('HIPCHAT_CONDENSED', True) == True)
- self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
- self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
- self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')
- try:
- self.hipchat_conn = hipchat.HipChat(token=os.getenv('HIPCHAT_TOKEN'))
- except Exception as e:
- utils.warning("Unable to connect to hipchat: {}".format(e))
- self.hipchat_msg_prefix = os.getenv('HIPCHAT_MSG_PREFIX', '')
- self.hipchat_msg_color = os.getenv('HIPCHAT_MSG_COLOR', '')
- self.printed_playbook = False
- self.playbook_name = None
- self.enabled = True
- else:
- self.enabled = False
+ self.enabled = "HIPCHAT_TOKEN" in os.environ
+ if not self.enabled:
+ return
+
+ # make sure we got our imports
+ if not hipchat:
+ raise ImportError(
+ "The hipchat plugin requires the hipchat Python module, "
+ "which is not installed or was not found."
+ )
+ if not prettytable:
+ raise ImportError(
+ "The hipchat plugin requires the prettytable Python module, "
+ "which is not installed or was not found."
+ )
+ self.start_time = time.time()
+ self.task_report = []
+ self.last_task = None
+ self.last_task_changed = False
+ self.last_task_count = 0
+ self.last_task_delta = 0
+ self.last_task_start = time.time()
+ self.condensed_task_report = (os.getenv('HIPCHAT_CONDENSED', True) == True)
+ self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
+ self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
+ self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')
+ try:
+ self.hipchat_conn = hipchat.HipChat(token=os.getenv('HIPCHAT_TOKEN'))
+ except Exception as e:
+ utils.warning("Unable to connect to hipchat: {}".format(e))
+ self.hipchat_msg_prefix = os.getenv('HIPCHAT_MSG_PREFIX', '')
+ self.hipchat_msg_color = os.getenv('HIPCHAT_MSG_COLOR', '')
+ self.printed_playbook = False
+ self.playbook_name = None
def _send_hipchat(self, message, room=None, from_name=None, color=None, message_format='text'):
@@ -221,7 +236,7 @@ def playbook_on_stats(self, stats):
summary_output = "{}: {} - ".format(self.hipchat_msg_prefix, host)
for summary_item in ['ok', 'changed', 'unreachable', 'failures']:
if stats[summary_item] != 0:
- summary_output += "{} - {} ".format(summary_item, stats[summary_item])
+ summary_output += "{} - {} ".format(summary_item, stats[summary_item])
summary_all_host_output.append(summary_output)
self._send_hipchat("
".join(summary_all_host_output), message_format='html')
msg = "{description}: Finished Ansible run for {play} in {min:02} minutes, {sec:02} seconds
".format(
diff --git a/playbooks/callback_plugins/sqs.py b/playbooks/callback_plugins/sqs.py
index 70124eb7140..39ad206b7d4 100644
--- a/playbooks/callback_plugins/sqs.py
+++ b/playbooks/callback_plugins/sqs.py
@@ -22,11 +22,12 @@
import json
import socket
try:
+ import boto
+except ImportError:
+ boto = None
+else:
import boto.sqs
from boto.exception import NoAuthHandlerFound
-except ImportError:
- print "Boto is required for the sqs_notify callback plugin"
- raise
class CallbackModule(object):
@@ -47,36 +48,42 @@ class CallbackModule(object):
- START events
"""
def __init__(self):
+ self.enable_sqs = 'ANSIBLE_ENABLE_SQS' in os.environ
+ if not self.enable_sqs:
+ return
+
+ # make sure we got our imports
+ if not boto:
+ raise ImportError(
+ "The sqs callback module requires the boto Python module, "
+ "which is not installed or was not found."
+ )
self.start_time = time.time()
- if 'ANSIBLE_ENABLE_SQS' in os.environ:
- self.enable_sqs = True
- if not 'SQS_REGION' in os.environ:
- print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
- 'not defined in environment'
- sys.exit(1)
- self.region = os.environ['SQS_REGION']
- try:
- self.sqs = boto.sqs.connect_to_region(self.region)
- except NoAuthHandlerFound:
- print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
- 'to AWS due invalid credentials'
- sys.exit(1)
- if not 'SQS_NAME' in os.environ:
- print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
- 'defined in environment'
- sys.exit(1)
- self.name = os.environ['SQS_NAME']
- self.queue = self.sqs.create_queue(self.name)
- if 'SQS_MSG_PREFIX' in os.environ:
- self.prefix = os.environ['SQS_MSG_PREFIX']
- else:
- self.prefix = ''
-
- self.last_seen_ts = {}
+ if not 'SQS_REGION' in os.environ:
+ print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
+ 'not defined in environment'
+ sys.exit(1)
+ self.region = os.environ['SQS_REGION']
+ try:
+ self.sqs = boto.sqs.connect_to_region(self.region)
+ except NoAuthHandlerFound:
+ print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
+ 'to AWS due invalid credentials'
+ sys.exit(1)
+ if not 'SQS_NAME' in os.environ:
+ print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
+ 'defined in environment'
+ sys.exit(1)
+ self.name = os.environ['SQS_NAME']
+ self.queue = self.sqs.create_queue(self.name)
+ if 'SQS_MSG_PREFIX' in os.environ:
+ self.prefix = os.environ['SQS_MSG_PREFIX']
else:
- self.enable_sqs = False
+ self.prefix = ''
+
+ self.last_seen_ts = {}
def runner_on_failed(self, host, res, ignore_errors=False):
if self.enable_sqs:
diff --git a/playbooks/edx-east/aide.yml b/playbooks/edx-east/aide.yml
index 96dfb35cc08..f3886ed2659 100644
--- a/playbooks/edx-east/aide.yml
+++ b/playbooks/edx-east/aide.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aide
- role: datadog
diff --git a/playbooks/edx-east/alton.yml b/playbooks/edx-east/alton.yml
index 4bc81a3744e..2236c6e1987 100644
--- a/playbooks/edx-east/alton.yml
+++ b/playbooks/edx-east/alton.yml
@@ -3,5 +3,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- alton
diff --git a/playbooks/edx-east/antivirus.yml b/playbooks/edx-east/antivirus.yml
new file mode 100644
index 00000000000..2e8206eaac2
--- /dev/null
+++ b/playbooks/edx-east/antivirus.yml
@@ -0,0 +1,12 @@
+- name: Deploy Antivirus Scanner
+ hosts: all
+ sudo: True
+ gather_facts: True
+ roles:
+ - antivirus
+ - role: datadog
+ when: COMMON_ENABLE_DATADOG
+ - role: splunkforwarder
+ when: COMMON_ENABLE_SPLUNKFORWARDER
+ - role: newrelic
+ when: COMMON_ENABLE_NEWRELIC
diff --git a/playbooks/edx-east/aws.yml b/playbooks/edx-east/aws.yml
index 48bd8f833b1..ea77ca006e8 100644
--- a/playbooks/edx-east/aws.yml
+++ b/playbooks/edx-east/aws.yml
@@ -2,5 +2,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
diff --git a/playbooks/edx-east/bastion.yml b/playbooks/edx-east/bastion.yml
index 9933dce7da2..9392434a74e 100644
--- a/playbooks/edx-east/bastion.yml
+++ b/playbooks/edx-east/bastion.yml
@@ -2,5 +2,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- bastion
diff --git a/playbooks/edx-east/certs.yml b/playbooks/edx-east/certs.yml
index 8f9d9b69a52..3c6d9553fec 100644
--- a/playbooks/edx-east/certs.yml
+++ b/playbooks/edx-east/certs.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- certs
diff --git a/playbooks/edx-east/cluster_rabbitmq.yml b/playbooks/edx-east/cluster_rabbitmq.yml
index adaeff78108..e95beb94ad1 100644
--- a/playbooks/edx-east/cluster_rabbitmq.yml
+++ b/playbooks/edx-east/cluster_rabbitmq.yml
@@ -1,5 +1,5 @@
-# ansible-playbook -i ec2.py commoncluster.yml --limit tag_Name_stage-edx-commoncluster -e@/path/to/vars/env-deployment.yml -T 30 --list-hosts
+# ansible-playbook -i ec2.py cluster_rabbitmq.yml --limit tag_Name_stage-edx-commoncluster -e@/path/to/vars/env-deployment.yml -T 30 --list-hosts
- hosts: all
sudo: True
@@ -28,14 +28,9 @@
tasks:
- debug: msg="{{ ansible_ec2_local_ipv4 }}"
with_items: list.results
- - shell: echo "rabbit@ip-{{ item|replace('.', '-') }}"
- when: item != ansible_ec2_local_ipv4
- with_items: hostvars.keys()
- register: list
- command: rabbitmqctl stop_app
- - command: rabbitmqctl join_cluster {{ item.stdout }}
- when: item.stdout is defined
- with_items: list.results
+ - command: rabbitmqctl join_cluster rabbit@ip-{{ hostvars.keys()[0]|replace('.', '-') }}
+ when: hostvars.keys()[0] != ansible_ec2_local_ipv4
- command: rabbitmqctl start_app
post_tasks:
- debug: var="{{ ansible_ec2_instance_id }}"
diff --git a/playbooks/edx-east/common.yml b/playbooks/edx-east/common.yml
index 35051f49927..549ecfeb325 100644
--- a/playbooks/edx-east/common.yml
+++ b/playbooks/edx-east/common.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- common
- role: datadog
diff --git a/playbooks/edx-east/connect_sandbox.yml b/playbooks/edx-east/connect_sandbox.yml
index 0a6c33d180b..1547ffacc8b 100644
--- a/playbooks/edx-east/connect_sandbox.yml
+++ b/playbooks/edx-east/connect_sandbox.yml
@@ -31,7 +31,7 @@
- "EDXAPP_MONGO_HOSTS: {{ EDXAPP_MONGO_HOSTS }}"
- "EDXAPP_MONGO_DB_NAME: {{ EDXAPP_MONGO_DB_NAME }}"
- "EDXAPP_MONGO_USER: {{ EDXAPP_MONGO_USER }}"
- - "EDXAPP_MONGO_PASS: {{ EDXAPP_MONGO_PASS }}"
+ - "EDXAPP_MONGO_PASSWORD: {{ EDXAPP_MONGO_PASSWORD }}"
tags: update_edxapp_mysql_host
- name: call update on edx-platform
diff --git a/playbooks/edx-east/create_dbs.yml b/playbooks/edx-east/create_dbs.yml
index fcb4dda4b61..e82760374ce 100644
--- a/playbooks/edx-east/create_dbs.yml
+++ b/playbooks/edx-east/create_dbs.yml
@@ -40,6 +40,10 @@
sudo: yes
with_items:
- python-mysqldb
+ # When this is run on jenkins the package will already
+ # exist and can't run as the jenkins user because it
+ # does not have sudo privs.
+ when: ansible_ssh_user != 'jenkins'
- name: create mysql databases for the edX stack
mysql_db: >
db={{ item[0] }}{{ item[1].db_name }}
diff --git a/playbooks/edx-east/demo.yml b/playbooks/edx-east/demo.yml
index 76e77aa17f1..fff266f0d91 100644
--- a/playbooks/edx-east/demo.yml
+++ b/playbooks/edx-east/demo.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- demo
- role: datadog
diff --git a/playbooks/edx-east/devpi.yml b/playbooks/edx-east/devpi.yml
index 205c65d404c..a57a741ffd6 100644
--- a/playbooks/edx-east/devpi.yml
+++ b/playbooks/edx-east/devpi.yml
@@ -2,5 +2,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- devpi
diff --git a/playbooks/edx-east/discern.yml b/playbooks/edx-east/discern.yml
index 78e7e4dbf15..710717323e5 100644
--- a/playbooks/edx-east/discern.yml
+++ b/playbooks/edx-east/discern.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- role: nginx
diff --git a/playbooks/edx-east/edx_ansible.yml b/playbooks/edx-east/edx_ansible.yml
index 7ae30643abc..ca890dbd7fe 100644
--- a/playbooks/edx-east/edx_ansible.yml
+++ b/playbooks/edx-east/edx_ansible.yml
@@ -1,6 +1,9 @@
- name: Deploy the edx_ansible role
hosts: all
sudo: True
- gather_facts: False
+ gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- edx_ansible
diff --git a/playbooks/edx-east/edx_continuous_integration.yml b/playbooks/edx-east/edx_continuous_integration.yml
index 5b237bfc29b..8752023d526 100644
--- a/playbooks/edx-east/edx_continuous_integration.yml
+++ b/playbooks/edx-east/edx_continuous_integration.yml
@@ -12,7 +12,6 @@
nginx_sites:
- cms
- lms
- - ora
- xqueue
- xserver
- certs
@@ -31,7 +30,6 @@
- forum
- { role: "xqueue", update_users: True }
- xserver
- - ora
- certs
- edx_ansible
- analytics-api
diff --git a/playbooks/edx-east/edx_provision.yml b/playbooks/edx-east/edx_provision.yml
index a240de0352d..7c1e2bd98b9 100644
--- a/playbooks/edx-east/edx_provision.yml
+++ b/playbooks/edx-east/edx_provision.yml
@@ -1,7 +1,7 @@
- name: Create ec2 instance
hosts: localhost
connection: local
- gather_facts: False
+ gather_facts: True
vars:
keypair: continuous-integration
instance_type: t2.medium
diff --git a/playbooks/edx-east/edxapp.yml b/playbooks/edx-east/edxapp.yml
index f97486f57f7..64df0a84415 100644
--- a/playbooks/edx-east/edxapp.yml
+++ b/playbooks/edx-east/edxapp.yml
@@ -3,6 +3,8 @@
sudo: True
gather_facts: True
vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- role: nginx
@@ -20,6 +22,10 @@
- role: splunkforwarder
when: COMMON_ENABLE_SPLUNKFORWARDER
- role: newrelic
+ NEWRELIC_LOGWATCH:
+ - logwatch-503.j2
+ - logwatch-cms-errors.j2
+ - logwatch-lms-errors.j2
when: COMMON_ENABLE_NEWRELIC
- role: minos
when: COMMON_ENABLE_MINOS
diff --git a/playbooks/edx-east/edxapp_migrate.yml b/playbooks/edx-east/edxapp_migrate.yml
index 1a09770b08a..a5bb63819d8 100644
--- a/playbooks/edx-east/edxapp_migrate.yml
+++ b/playbooks/edx-east/edxapp_migrate.yml
@@ -4,15 +4,27 @@
gather_facts: False
vars:
db_dry_run: "--db-dry-run"
+ syncdb: false
tasks:
# Syncdb with migrate when the migrate user is overridden in extra vars
- - name: syncdb and migrate
+ - name: migrate
shell: >
chdir={{ edxapp_code_dir }}
- python manage.py {{ item }} syncdb --migrate --noinput --settings=aws_migrate {{ db_dry_run }}
+ python manage.py {{ item }} migrate --noinput {{ db_dry_run }} --settings=aws_migrate
environment:
DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
with_items:
- lms
- cms
+ - name: syncdb
+ shell: >
+ chdir={{ edxapp_code_dir }}
+ python manage.py {{ item }} syncdb --noinput --settings=aws_migrate
+ environment:
+ DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+ DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
+ when: syncdb
+ with_items:
+ - lms
+ - cms
diff --git a/playbooks/edx-east/flower.yml b/playbooks/edx-east/flower.yml
index 5da7b27a4f8..c78b69fcbdc 100644
--- a/playbooks/edx-east/flower.yml
+++ b/playbooks/edx-east/flower.yml
@@ -2,5 +2,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- flower
diff --git a/playbooks/edx-east/forum.yml b/playbooks/edx-east/forum.yml
index 5b7d4dd2693..6eb534cc0e3 100644
--- a/playbooks/edx-east/forum.yml
+++ b/playbooks/edx-east/forum.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- role: nginx
diff --git a/playbooks/edx-east/insights.yml b/playbooks/edx-east/insights.yml
new file mode 100644
index 00000000000..8824a8c5c25
--- /dev/null
+++ b/playbooks/edx-east/insights.yml
@@ -0,0 +1,20 @@
+- name: Deploy Insights
+ hosts: all
+ sudo: True
+ gather_facts: True
+ vars:
+ ENABLE_DATADOG: False
+ ENABLE_SPLUNKFORWARDER: False
+ ENABLE_NEWRELIC: True
+ roles:
+ - role: nginx
+ nginx_sites:
+ - insights
+ - aws
+ - insights
+ - role: datadog
+ when: COMMON_ENABLE_DATADOG
+ - role: splunkforwarder
+ when: COMMON_ENABLE_SPLUNKFORWARDER
+ - role: newrelic
+ when: COMMON_ENABLE_NEWRELIC
diff --git a/playbooks/edx-east/jenkins_admin.yml b/playbooks/edx-east/jenkins_admin.yml
index 763f842b85c..ccd177b03ff 100644
--- a/playbooks/edx-east/jenkins_admin.yml
+++ b/playbooks/edx-east/jenkins_admin.yml
@@ -3,5 +3,8 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- jenkins_admin
diff --git a/playbooks/edx-east/jenkins_worker.yml b/playbooks/edx-east/jenkins_worker.yml
index f814baf13ed..2d1a4da23ca 100644
--- a/playbooks/edx-east/jenkins_worker.yml
+++ b/playbooks/edx-east/jenkins_worker.yml
@@ -8,6 +8,8 @@
gather_facts: True
vars:
mongo_enable_journal: False
+ serial_count: 1
+ serial: "{{ serial_count }}"
vars_files:
- roles/edxapp/defaults/main.yml
- roles/ora/defaults/main.yml
@@ -18,4 +20,5 @@
- edxlocal
- mongo
- browsers
+ - browsermob-proxy
- jenkins_worker
diff --git a/playbooks/edx-east/legacy_ora.yml b/playbooks/edx-east/legacy_ora.yml
index 295ebf32974..5a45ff3b363 100644
--- a/playbooks/edx-east/legacy_ora.yml
+++ b/playbooks/edx-east/legacy_ora.yml
@@ -3,7 +3,9 @@
hosts: all
sudo: True
gather_facts: True
- serial: 1
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
vars_files:
- "{{secure_dir}}/vars/{{COMMON_ENVIRONMENT}}/legacy-ora.yml"
roles:
diff --git a/playbooks/edx-east/lifecycle_inventory.py b/playbooks/edx-east/lifecycle_inventory.py
index fa4a39dcf5e..b3d4c9f8694 100755
--- a/playbooks/edx-east/lifecycle_inventory.py
+++ b/playbooks/edx-east/lifecycle_inventory.py
@@ -41,6 +41,18 @@ def __init__(self, profile):
parser = argparse.ArgumentParser()
self.profile = profile
+ def get_e_d_from_tags(self, group):
+
+ environment = "default_environment"
+ deployment = "default_deployment"
+
+ for r in group.tags:
+ if r.key == "environment":
+ environment = r.value
+ elif r.key == "deployment":
+ deployment = r.value
+ return environment,deployment
+
def get_instance_dict(self):
ec2 = boto.connect_ec2(profile_name=self.profile)
reservations = ec2.get_all_instances()
@@ -64,10 +76,12 @@ def run(self):
for instance in group.instances:
private_ip_address = instances[instance.instance_id].private_ip_address
-
- inventory[group.name].append(private_ip_address)
- inventory[group.name + "_" + instance.lifecycle_state].append(private_ip_address)
- inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
+ if private_ip_address:
+ environment,deployment = self.get_e_d_from_tags(group)
+ inventory[environment + "_" + deployment + "_" + instance.lifecycle_state.replace(":","_")].append(private_ip_address)
+ inventory[group.name].append(private_ip_address)
+ inventory[group.name + "_" + instance.lifecycle_state.replace(":","_")].append(private_ip_address)
+ inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
print json.dumps(inventory, sort_keys=True, indent=2)
@@ -77,8 +91,8 @@ def run(self):
parser.add_argument('-p', '--profile', help='The aws profile to use when connecting.')
parser.add_argument('-l', '--list', help='Ansible passes this, we ignore it.', action='store_true', default=True)
args = parser.parse_args()
-
+
LifecycleInventory(args.profile).run()
-
+
diff --git a/playbooks/edx-east/minos.yml b/playbooks/edx-east/minos.yml
index 8339970da61..e5865d9686b 100644
--- a/playbooks/edx-east/minos.yml
+++ b/playbooks/edx-east/minos.yml
@@ -3,6 +3,9 @@
sudo: True
gather_facts: True
vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- common
- - minos
+ - aws
+ - minos
diff --git a/playbooks/edx-east/mongo.yml b/playbooks/edx-east/mongo.yml
index f6afd481759..fe71cedc2e9 100644
--- a/playbooks/edx-east/mongo.yml
+++ b/playbooks/edx-east/mongo.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- mongo
- mongo_mms
diff --git a/playbooks/edx-east/notifier.yml b/playbooks/edx-east/notifier.yml
new file mode 100644
index 00000000000..be18bbf11a5
--- /dev/null
+++ b/playbooks/edx-east/notifier.yml
@@ -0,0 +1,11 @@
+- name: Configure notifier instance
+ hosts: all
+ sudo: True
+ gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
+ roles:
+ - aws
+ - notifier
+
diff --git a/playbooks/edx-east/ora.yml b/playbooks/edx-east/ora.yml
index 3e74011fde7..3e3b392afc6 100644
--- a/playbooks/edx-east/ora.yml
+++ b/playbooks/edx-east/ora.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- role: nginx
nginx_sites:
diff --git a/playbooks/edx-east/retire_host.yml b/playbooks/edx-east/retire_host.yml
index 99c2abca60f..55b4c779d8f 100644
--- a/playbooks/edx-east/retire_host.yml
+++ b/playbooks/edx-east/retire_host.yml
@@ -1,12 +1,15 @@
-# ansible-playbook -i ./lifecycle_inventory.py ./retire_host.yml
-# -e@/vars/env.yml --limit Terminating_Wait
+# ansible-playbook -i ./lifecycle_inventory.py ./retire_host.yml
+# -e@/vars/env.yml --limit Terminating_Wait -e TARGET="Terminating_Wait"
+#
+# Note that the target now must be specified as an argument
+#
#
# This is separate because it's use of handlers
# leads to various race conditions.
#
- name: Stop all services
- hosts: Terminating_Wait
+ hosts: "{{TARGET}}"
sudo: True
gather_facts: False
vars:
@@ -15,41 +18,30 @@
- stop_all_edx_services
- name: Server retirement workflow
- hosts: Terminating_Wait
+ hosts: "{{TARGET}}"
sudo: True
gather_facts: False
tasks:
- - name: Force a log rotation
- command: /usr/sbin/logrotate -f /etc/logrotate.d/{{ item }}
- with_items:
- - "apport"
- - "apt"
- - "aptitude"
- - "dpkg"
- - "hourly"
- - "landscape-client"
- - "newrelic-sysmond"
- - "nginx"
- - "nginx-access"
- - "nginx-error"
- - "ppp"
- - "rsyslog"
- - "ufw"
- - "unattended-upgrades"
- - "upstart"
- - name: Force a log rotation
+ - name: Terminate existing s3 log sync
+ command: /usr/bin/pkill send-logs-to-s3 || true
+ - name: "Ensure send-logs-to-s3 script is in the logrotate file"
+ shell: grep send-logs-to-s3 /etc/logrotate.d/hourly/tracking.log
+ # We only force a rotation of edx logs.
+ # Forced rotation of system logfiles will only
+ # work if there hasn't already been a previous rotation
+ # The logrotate will also call send-logs-to-s3 but hasn't
+ # been updated for all servers yet.
+ - name: Force a log rotation which will call the log sync
command: /usr/sbin/logrotate -f /etc/logrotate.d/hourly/{{ item }}
with_items:
- "tracking.log"
- "edx-services"
- - name: Terminate existing s3 log sync
- command: /usr/bin/pkill send-logs-to-s3 || true
- - name: Send logs to s3
- command: /edx/bin/send-logs-to-s3
-
-
+ # This catches the case where tracking.log is 0b
+ - name: Sync again
+ command: /edx/bin/send-logs-to-s3 -d "{{ COMMON_LOG_DIR }}/tracking/*" -b "{{ COMMON_AWS_SYNC_BUCKET }}/logs/tracking"
+
- name: Run minos verification
- hosts: Terminating_Wait
+ hosts: "{{TARGET}}"
sudo: True
gather_facts: False
tasks:
diff --git a/playbooks/edx-east/snort.yml b/playbooks/edx-east/snort.yml
index 70096bd6dad..b962878b0a2 100644
--- a/playbooks/edx-east/snort.yml
+++ b/playbooks/edx-east/snort.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- snort
- role: datadog
diff --git a/playbooks/edx-east/stop_all_edx_services.yml b/playbooks/edx-east/stop_all_edx_services.yml
index b7228d7693c..9e570e009e0 100644
--- a/playbooks/edx-east/stop_all_edx_services.yml
+++ b/playbooks/edx-east/stop_all_edx_services.yml
@@ -2,5 +2,8 @@
hosts: all
sudo: True
gather_facts: False
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- stop_all_edx_services
diff --git a/playbooks/edx-east/testcourses.yml b/playbooks/edx-east/testcourses.yml
index 6c918926523..a1353dea773 100644
--- a/playbooks/edx-east/testcourses.yml
+++ b/playbooks/edx-east/testcourses.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- testcourses
- role: datadog
diff --git a/playbooks/edx-east/xqueue.yml b/playbooks/edx-east/xqueue.yml
index cb63e8653fc..d48915b2a79 100644
--- a/playbooks/edx-east/xqueue.yml
+++ b/playbooks/edx-east/xqueue.yml
@@ -35,6 +35,8 @@
- role: splunkforwarder
when: COMMON_ENABLE_SPLUNKFORWARDER
- role: newrelic
+ NEWRELIC_LOGWATCH:
+ - logwatch-xqueue-errors.j2
when: COMMON_ENABLE_NEWRELIC
post_tasks:
- debug: var="{{ ansible_ec2_instance_id }}"
diff --git a/playbooks/edx-east/xqwatcher.yml b/playbooks/edx-east/xqwatcher.yml
index 82735f3a0da..6d1e380181f 100644
--- a/playbooks/edx-east/xqwatcher.yml
+++ b/playbooks/edx-east/xqwatcher.yml
@@ -8,6 +8,8 @@
ENABLE_DATADOG: False
ENABLE_SPLUNKFORWARDER: False
ENABLE_NEWRELIC: False
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- xqwatcher
@@ -16,4 +18,4 @@
- role: splunkforwarder
when: COMMON_ENABLE_SPLUNKFORWARDER
- role: newrelic
- when: COMMON_ENABLE_NEWRELIC
\ No newline at end of file
+ when: COMMON_ENABLE_NEWRELIC
diff --git a/playbooks/edx-east/xserver.yml b/playbooks/edx-east/xserver.yml
index 362c1c9f0ee..8f579fdb574 100644
--- a/playbooks/edx-east/xserver.yml
+++ b/playbooks/edx-east/xserver.yml
@@ -2,6 +2,9 @@
hosts: all
sudo: True
gather_facts: True
+ vars:
+ serial_count: 1
+ serial: "{{ serial_count }}"
roles:
- aws
- role: nginx
diff --git a/playbooks/edx-west/cloudformation.yml b/playbooks/edx-west/cloudformation.yml
index 0fe9f50e482..fcee0c7caef 100644
--- a/playbooks/edx-west/cloudformation.yml
+++ b/playbooks/edx-west/cloudformation.yml
@@ -31,14 +31,14 @@
tasks:
- name: edX configuration
cloudformation: >
- stack_name="$name" state=present
- region=$region disable_rollback=false
+ stack_name="{{ name }}" state=present
+ region="{{ region }}" disable_rollback=false
template=../cloudformation_templates/edx-server-multi-instance.json
args:
template_parameters:
- KeyName: $key
+ KeyName: "{{key}}"
InstanceType: m1.small
- GroupTag: $group
+ GroupTag: "{{group}}"
register: stack
- name: show stack outputs
- debug: msg="My stack outputs are ${stack.stack_outputs}"
+ debug: msg="My stack outputs are {{stack.stack_outputs}}"
diff --git a/playbooks/edx_sandbox.yml b/playbooks/edx_sandbox.yml
index bb496a9432c..036c9a8f367 100644
--- a/playbooks/edx_sandbox.yml
+++ b/playbooks/edx_sandbox.yml
@@ -18,6 +18,7 @@
# These should stay false for the public AMI
COMMON_ENABLE_DATADOG: False
COMMON_ENABLE_SPLUNKFORWARDER: False
+ ENABLE_LEGACY_ORA: !!null
roles:
- role: nginx
nginx_sites:
@@ -38,7 +39,8 @@
- elasticsearch
- forum
- { role: "xqueue", update_users: True }
- - ora
+ - role: ora
+ when: ENABLE_LEGACY_ORA
- certs
- edx_ansible
- role: datadog
diff --git a/playbooks/roles/alton/tasks/deploy.yml b/playbooks/roles/alton/tasks/deploy.yml
index cbe06199f22..1714b040449 100644
--- a/playbooks/roles/alton/tasks/deploy.yml
+++ b/playbooks/roles/alton/tasks/deploy.yml
@@ -1,3 +1,10 @@
+- name: setup the alton env
+ template: >
+ src="alton_env.j2" dest="{{ alton_app_dir }}/alton_env"
+ owner="{{ alton_user }}" group="{{ common_web_user }}"
+ mode=0644
+ notify: restart alton
+
- name: configure the boto profiles for alton
template: >
src="boto.j2"
diff --git a/playbooks/roles/alton/tasks/main.yml b/playbooks/roles/alton/tasks/main.yml
index 9715860e7f8..9b71a12b47b 100644
--- a/playbooks/roles/alton/tasks/main.yml
+++ b/playbooks/roles/alton/tasks/main.yml
@@ -34,11 +34,4 @@
- "{{ alton_app_dir }}"
- "{{ alton_venvs_dir }}"
-- name: setup the alton env
- template: >
- src="alton_env.j2" dest="{{ alton_app_dir }}/alton_env"
- owner="{{ alton_user }}" group="{{ common_web_user }}"
- mode=0644
- notify: restart alton
-
- include: deploy.yml tags=deploy
diff --git a/playbooks/roles/alton/templates/alton_env.j2 b/playbooks/roles/alton/templates/alton_env.j2
index 40feb800236..ba2a561f12d 100644
--- a/playbooks/roles/alton/templates/alton_env.j2
+++ b/playbooks/roles/alton/templates/alton_env.j2
@@ -6,3 +6,5 @@ export {{ name }}="{{ value }}"
{% endif %}
{%- endfor %}
+export WILL_BOTO_PROFILES="{{ ALTON_AWS_CREDENTIALS|join(';') }}"
+
diff --git a/playbooks/roles/alton/templates/boto.j2 b/playbooks/roles/alton/templates/boto.j2
index 8a6ccc181b4..28f21ce1b07 100644
--- a/playbooks/roles/alton/templates/boto.j2
+++ b/playbooks/roles/alton/templates/boto.j2
@@ -1,5 +1,5 @@
{% for deployment, creds in ALTON_AWS_CREDENTIALS.iteritems() %}
-[profile {{deployment}}]
+[profile {{ deployment }}]
aws_access_key_id = {{ creds.access_id }}
aws_secret_access_key = {{ creds.secret_key }}
diff --git a/playbooks/roles/analytics-api/defaults/main.yml b/playbooks/roles/analytics-api/defaults/main.yml
index f31157c54f6..b2cfff1fa6b 100644
--- a/playbooks/roles/analytics-api/defaults/main.yml
+++ b/playbooks/roles/analytics-api/defaults/main.yml
@@ -19,46 +19,58 @@ ANALYTICS_API_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }
ANALYTICS_API_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
ANALYTICS_API_NGINX_PORT: "18100"
+ANALYTICS_API_DATABASES:
+ # rw user
+ default:
+ ENGINE: 'django.db.backends.mysql'
+ NAME: 'analytics-api'
+ USER: 'api001'
+ PASSWORD: 'password'
+ HOST: 'localhost'
+ PORT: '3306'
+ # read-only user
+ reports:
+ ENGINE: 'django.db.backends.mysql'
+ NAME: 'reports'
+ USER: 'reports001'
+ PASSWORD: 'password'
+ HOST: 'localhost'
+ PORT: '3306'
+
ANALYTICS_API_VERSION: "master"
# Default dummy user, override this!!
ANALYTICS_API_USERS:
"dummy-api-user": "changeme"
+ANALYTICS_API_SECRET_KEY: 'Your secret key here'
+ANALYTICS_API_TIME_ZONE: 'UTC'
+ANALYTICS_API_LANGUAGE_CODE: 'en-us'
+ANALYTICS_API_EMAIL_HOST: 'localhost'
+ANALYTICS_API_EMAIL_HOST_USER: 'mail_user'
+ANALYTICS_API_EMAIL_HOST_PASSWORD: 'mail_password'
+ANALYTICS_API_EMAIL_PORT: 587
+ANALYTICS_API_AUTH_TOKEN: 'put-your-api-token-here'
+
ANALYTICS_API_CONFIG:
ANALYTICS_DATABASE: 'reports'
- SECRET_KEY: 'Your secret key here'
- TIME_ZONE: 'America/New_York'
- LANGUAGE_CODE: 'en-us'
+ SECRET_KEY: '{{ ANALYTICS_API_SECRET_KEY }}'
+ TIME_ZONE: '{{ ANALYTICS_API_TIME_ZONE }}'
+ LANGUAGE_CODE: '{{ANALYTICS_API_LANGUAGE_CODE }}'
# email config
- EMAIL_HOST: 'smtp.example.com'
- EMAIL_HOST_PASSWORD: ""
- EMAIL_HOST_USER: ""
- EMAIL_PORT: 587
- API_AUTH_TOKEN: 'put-your-api-token-here'
- STATICFILES_DIRS: []
+ EMAIL_HOST: '{{ ANALYTICS_API_EMAIL_HOST }}'
+ EMAIL_HOST_PASSWORD: '{{ ANALYTICS_API_EMAIL_HOST_PASSWORD }}'
+ EMAIL_HOST_USER: '{{ ANALYTICS_API_EMAIL_HOST_USER }}'
+ EMAIL_PORT: $ANALYTICS_API_EMAIL_PORT
+ API_AUTH_TOKEN: '{{ ANALYTICS_API_AUTH_TOKEN }}'
+ STATICFILES_DIRS: ['static']
STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ analytics_api_service_name }}/staticfiles"
# db config
DATABASE_OPTIONS:
connect_timeout: 10
- DATABASES:
- # rw user
- default:
- ENGINE: 'django.db.backends.mysql'
- NAME: 'analytics-api'
- USER: 'api001'
- PASSWORD: 'password'
- HOST: 'localhost'
- PORT: '3306'
- # read-only user
- reports:
- ENGINE: 'django.db.backends.mysql'
- NAME: 'reports'
- USER: 'reports001'
- PASSWORD: 'password'
- HOST: 'localhost'
- PORT: '3306'
+ DATABASES: '{{ ANALYTICS_API_DATABASES }}'
ANALYTICS_API_GUNICORN_WORKERS: "2"
+ANALYTICS_API_GUNICORN_EXTRA: ""
#
# vars are namespace with the module name.
#
diff --git a/playbooks/roles/analytics-api/tasks/main.yml b/playbooks/roles/analytics-api/tasks/main.yml
index 28bc6b47ffd..bb0ce8ab167 100644
--- a/playbooks/roles/analytics-api/tasks/main.yml
+++ b/playbooks/roles/analytics-api/tasks/main.yml
@@ -32,7 +32,7 @@
# ansible-playbook -i 'api.example.com,' ./analyticsapi.yml -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
#
-- fail: msg="You must provide an private key for the analytics repo"
+- fail: msg="You must provide a private key for the analytics repo"
when: not ANALYTICS_API_GIT_IDENTITY
- include: deploy.yml tags=deploy
diff --git a/playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2 b/playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2
index 190cd13e613..e188734102e 100644
--- a/playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2
+++ b/playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2
@@ -15,4 +15,4 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
source {{ analytics_api_app_dir }}/analytics_api_env
-{{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ ANALYTICS_API_GUNICORN_WORKERS }} --timeout={{ analytics_api_gunicorn_timeout }} analyticsdataserver.wsgi:application
+{{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ ANALYTICS_API_GUNICORN_WORKERS }} --timeout={{ analytics_api_gunicorn_timeout }} {{ ANALYTICS_API_GUNICORN_EXTRA }} analyticsdataserver.wsgi:application
diff --git a/playbooks/roles/analytics-server/defaults/main.yml b/playbooks/roles/analytics-server/defaults/main.yml
index 55765c042a7..3f70a808ef7 100644
--- a/playbooks/roles/analytics-server/defaults/main.yml
+++ b/playbooks/roles/analytics-server/defaults/main.yml
@@ -19,6 +19,7 @@ AS_SERVER_PORT: '9000'
AS_ENV_LANG: 'en_US.UTF-8'
AS_LOG_LEVEL: 'INFO'
AS_WORKERS: '2'
+AS_GUNICORN_EXTRA: ""
# add public keys to enable the automator user
# for running manage.py commands
@@ -40,14 +41,14 @@ analytics_auth_config:
DATABASES:
analytics:
<<: *databases_default
- USER: $AS_DB_ANALYTICS_USER
- PASSWORD: $AS_DB_ANALYTICS_PASSWORD
- HOST: $AS_DB_ANALYTICS_HOST
- ANALYTICS_API_KEY: $AS_API_KEY
+ USER: "{{ AS_DB_ANALYTICS_USER }}"
+ PASSWORD: "{{ AS_DB_ANALYTICS_PASSWORD }}"
+ HOST: "{{ AS_DB_ANALYTICS_HOST }}"
+ ANALYTICS_API_KEY: "{{ AS_API_KEY }}"
ANALYTICS_RESULTS_DB:
- MONGO_URI: $AS_DB_RESULTS_URL
- MONGO_DB: $AS_DB_RESULTS_DB
- MONGO_STORED_QUERIES_COLLECTION: $AS_DB_RESULTS_COLLECTION
+ MONGO_URI: "{{ AS_DB_RESULTS_URL }}"
+ MONGO_DB: "{{ AS_DB_RESULTS_DB }}"
+ MONGO_STORED_QUERIES_COLLECTION: "{{ AS_DB_RESULTS_COLLECTION }}"
as_role_name: "analytics-server"
as_user: "analytics-server"
diff --git a/playbooks/roles/analytics-server/tasks/deploy.yml b/playbooks/roles/analytics-server/tasks/deploy.yml
index 7ebc382df97..f4153426f68 100644
--- a/playbooks/roles/analytics-server/tasks/deploy.yml
+++ b/playbooks/roles/analytics-server/tasks/deploy.yml
@@ -28,7 +28,7 @@
accept_hostkey=yes
version={{ as_version }} force=true
environment:
- GIT_SSH: $as_git_ssh
+ GIT_SSH: "{{ as_git_ssh }}"
notify: restart the analytics service
notify: start the analytics service
tags:
diff --git a/playbooks/roles/analytics-server/templates/etc/init/analytics.conf.j2 b/playbooks/roles/analytics-server/templates/etc/init/analytics.conf.j2
index c060f623580..185d74cb625 100644
--- a/playbooks/roles/analytics-server/templates/etc/init/analytics.conf.j2
+++ b/playbooks/roles/analytics-server/templates/etc/init/analytics.conf.j2
@@ -18,4 +18,4 @@ env DJANGO_SETTINGS_MODULE={{ as_django_settings }}
chdir {{ as_code_dir }}
setuid {{ as_web_user }}
-exec {{ as_venv_dir }}/bin/gunicorn -b 0.0.0.0:$PORT -w $WORKERS --pythonpath={{ as_code_dir }}/anserv anserv.wsgi
+exec {{ as_venv_dir }}/bin/gunicorn -b 0.0.0.0:$PORT -w $WORKERS --pythonpath={{ as_code_dir }}/anserv {{ AS_GUNICORN_EXTRA }} anserv.wsgi
diff --git a/playbooks/roles/analytics/defaults/main.yml b/playbooks/roles/analytics/defaults/main.yml
index e17174c1de5..36e0dd3dc62 100644
--- a/playbooks/roles/analytics/defaults/main.yml
+++ b/playbooks/roles/analytics/defaults/main.yml
@@ -19,6 +19,7 @@ ANALYTICS_SERVER_PORT: '9000'
ANALYTICS_ENV_LANG: 'en_US.UTF-8'
ANALYTICS_LOG_LEVEL: 'INFO'
ANALYTICS_WORKERS: '2'
+ANALYTICS_GUNICORN_EXTRA: ""
DATABASES:
default: &databases_default
@@ -33,14 +34,14 @@ analytics_auth_config:
DATABASES:
analytics:
<<: *databases_default
- USER: $ANALYTICS_DB_ANALYTICS_USER
- PASSWORD: $ANALYTICS_DB_ANALYTICS_PASSWORD
- HOST: $ANALYTICS_DB_ANALYTICS_HOST
- ANALYTICS_API_KEY: $ANALYTICS_API_KEY
+ USER: "{{ ANALYTICS_DB_ANALYTICS_USER }}"
+ PASSWORD: "{{ ANALYTICS_DB_ANALYTICS_PASSWORD }}"
+ HOST: "{{ ANALYTICS_DB_ANALYTICS_HOST }}"
+ ANALYTICS_API_KEY: "{{ ANALYTICS_API_KEY }}"
ANALYTICS_RESULTS_DB:
- MONGO_URI: $ANALYTICS_DB_RESULTS_URL
- MONGO_DB: $ANALYTICS_DB_RESULTS_DB
- MONGO_STORED_QUERIES_COLLECTION: $ANALYTICS_DB_RESULTS_COLLECTION
+ MONGO_URI: "{{ ANALYTICS_DB_RESULTS_URL }}"
+ MONGO_DB: "{{ ANALYTICS_DB_RESULTS_DB }}"
+ MONGO_STORED_QUERIES_COLLECTION: "{{ ANALYTICS_DB_RESULTS_COLLECTION }}"
analytics_role_name: "analytics"
analytics_user: "analytics"
diff --git a/playbooks/roles/analytics/tasks/deploy.yml b/playbooks/roles/analytics/tasks/deploy.yml
index c84667d54e5..b6c5945b9db 100644
--- a/playbooks/roles/analytics/tasks/deploy.yml
+++ b/playbooks/roles/analytics/tasks/deploy.yml
@@ -28,7 +28,7 @@
accept_hostkey=yes
version={{ analytics_version }} force=true
environment:
- GIT_SSH: $analytics_git_ssh
+ GIT_SSH: "{{ analytics_git_ssh }}"
notify: restart the analytics service
notify: start the analytics service
tags:
diff --git a/playbooks/roles/analytics/templates/etc/init/analytics.conf.j2 b/playbooks/roles/analytics/templates/etc/init/analytics.conf.j2
index f1e797a240f..8e83f5407bd 100644
--- a/playbooks/roles/analytics/templates/etc/init/analytics.conf.j2
+++ b/playbooks/roles/analytics/templates/etc/init/analytics.conf.j2
@@ -18,4 +18,4 @@ env DJANGO_SETTINGS_MODULE={{ analytics_django_settings }}
chdir {{ analytics_code_dir }}
setuid {{ analytics_web_user }}
-exec {{ analytics_venv_dir }}/bin/gunicorn -b 0.0.0.0:$PORT -w $WORKERS --pythonpath={{ analytics_code_dir }}/anserv anserv.wsgi
+exec {{ analytics_venv_dir }}/bin/gunicorn -b 0.0.0.0:$PORT -w $WORKERS --pythonpath={{ analytics_code_dir }}/anserv {{ ANALYTICS_GUNICORN_EXTRA }} anserv.wsgi
diff --git a/playbooks/roles/ansible-role/tasks/main.yml b/playbooks/roles/ansible-role/tasks/main.yml
index ba2a7f5e37f..61ca73a1e10 100644
--- a/playbooks/roles/ansible-role/tasks/main.yml
+++ b/playbooks/roles/ansible-role/tasks/main.yml
@@ -10,7 +10,7 @@
when: role_exists | success
- name: create role directories
- file: path=roles/{{role_name}}/{{ item }} state=directory
+ file: path=roles/{{ role_name }}/{{ item }} state=directory
with_items:
- tasks
- meta
diff --git a/playbooks/roles/ansible_debug/tasks/main.yml b/playbooks/roles/ansible_debug/tasks/main.yml
index adb4728cc74..a651dfab90f 100644
--- a/playbooks/roles/ansible_debug/tasks/main.yml
+++ b/playbooks/roles/ansible_debug/tasks/main.yml
@@ -6,7 +6,7 @@
- debug
- name: Dump lms auth|env file
- template: src=../../edxapp/templates/lms.{{item}}.json.j2 dest=/tmp/lms.{{item}}.json mode=0600
+ template: src=../../edxapp/templates/lms.{{ item }}.json.j2 dest=/tmp/lms.{{ item }}.json mode=0600
with_items:
- env
- auth
@@ -16,7 +16,7 @@
- debug
- name: Dump lms-preview auth|env file
- template: src=../../edxapp/templates/lms-preview.{{item}}.json.j2 dest=/tmp/lms-preview.{{item}}.json mode=0600
+ template: src=../../edxapp/templates/lms-preview.{{ item }}.json.j2 dest=/tmp/lms-preview.{{ item }}.json mode=0600
with_items:
- env
- auth
@@ -26,7 +26,7 @@
- debug
- name: Dump cms auth|env file
- template: src=../../edxapp/templates/cms.{{item}}.json.j2 dest=/tmp/cms.{{item}}.json mode=0600
+ template: src=../../edxapp/templates/cms.{{ item }}.json.j2 dest=/tmp/cms.{{ item }}.json mode=0600
with_items:
- env
- auth
@@ -44,7 +44,7 @@
- name: fetch remote files
# fetch is fail-safe for remote files that don't exist
# setting mode is not an option
- fetch: src=/tmp/{{item}} dest=/tmp/{{ansible_hostname}}-{{item}} flat=True
+ fetch: src=/tmp/{{ item }} dest=/tmp/{{ ansible_hostname }}-{{item}} flat=True
with_items:
- ansible.all.json
- ansible.all.yml
diff --git a/playbooks/roles/antivirus/defaults/main.yml b/playbooks/roles/antivirus/defaults/main.yml
new file mode 100644
index 00000000000..89a5e5b072f
--- /dev/null
+++ b/playbooks/roles/antivirus/defaults/main.yml
@@ -0,0 +1,38 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role antivirus
+#
+
+#
+# vars are namespace with the module name.
+#
+antivirus_role_name: antivirus
+
+#
+# OS packages
+#
+
+antivirus_debian_pkgs: [clamav]
+antivirus_redhat_pkgs: []
+antivirus_pip_pkgs: []
+
+antivirus_app_dir: /edx/app/antivirus
+antivirus_user: "antivirus"
+
+ANTIVIRUS_BUCKETS: !!null
+ANTIVIRUS_MAILTO: "{{ EDXAPP_TECH_SUPPORT_EMAIL }}"
+ANTIVIRUS_MAILFROM: "{{ EDXAPP_DEFAULT_FROM_EMAIL }}"
+ANTIVIRUS_AWS_KEY: ""
+ANTIVIRUS_AWS_SECRET: ""
+ANTIVIRUS_S3_AWS_KEY: "{{ ANTIVIRUS_AWS_KEY }}"
+ANTIVIRUS_SES_AWS_KEY: "{{ ANTIVIRUS_AWS_KEY }}"
+ANTIVIRUS_S3_AWS_SECRET: "{{ ANTIVIRUS_AWS_SECRET}}"
+ANTIVIRUS_SES_AWS_SECRET: "{{ ANTIVIRUS_AWS_SECRET}}"
diff --git a/playbooks/roles/antivirus/handlers/main.yml b/playbooks/roles/antivirus/handlers/main.yml
new file mode 100644
index 00000000000..faa9e377fd9
--- /dev/null
+++ b/playbooks/roles/antivirus/handlers/main.yml
@@ -0,0 +1,16 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role antivirus
+#
+# Overview:
+#
+#
diff --git a/playbooks/roles/antivirus/meta/main.yml b/playbooks/roles/antivirus/meta/main.yml
new file mode 100644
index 00000000000..32949d70a9d
--- /dev/null
+++ b/playbooks/roles/antivirus/meta/main.yml
@@ -0,0 +1,16 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role antivirus
+#
+dependencies:
+ - role: user
+ user_info: "{{ BASTION_USER_INFO }}"
+ - aws
diff --git a/playbooks/roles/antivirus/tasks/main.yml b/playbooks/roles/antivirus/tasks/main.yml
new file mode 100644
index 00000000000..f20d3d2f966
--- /dev/null
+++ b/playbooks/roles/antivirus/tasks/main.yml
@@ -0,0 +1,63 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role antivirus
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: install antivirus system packages
+ apt: pkg={{ item }} install_recommends=yes state=present
+ with_items: antivirus_debian_pkgs
+
+- name: create antivirus scanner user
+ user: >
+ name="{{ antivirus_user }}"
+ home="{{ antivirus_app_dir }}"
+ createhome=no
+ shell=/bin/false
+
+- name: create antivirus app and data dirs
+ file: >
+ path="{{ item }}"
+ state=directory
+ owner="{{ antivirus_user }}"
+ group="{{ antivirus_user }}"
+ with_items:
+ - "{{ antivirus_app_dir }}"
+ - "{{ antivirus_app_dir }}/data"
+
+- name: install antivirus s3 scanner script
+ template: >
+ src=s3_bucket_virus_scan.sh.j2
+ dest={{ antivirus_app_dir }}/s3_bucket_virus_scan.sh
+ mode=0555
+ owner={{ antivirus_user }}
+ group={{ antivirus_user }}
+
+- name: install antivirus s3 scanner cronjob
+ cron: >
+ name="antivirus-{{ item }}"
+ job="{{ antivirus_app_dir }}/s3_bucket_virus_scan.sh -b '{{ item }}' -m '{{ ANTIVIRUS_MAILTO }}' -f '{{ ANTIVIRUS_MAILFROM }}'"
+ backup=yes
+ cron_file=antivirus-{{ item }}
+ user={{ antivirus_user }}
+ hour="*"
+ minute="0"
+ day="*"
+ with_items: ANTIVIRUS_BUCKETS
diff --git a/playbooks/roles/antivirus/templates/s3_bucket_virus_scan.sh.j2 b/playbooks/roles/antivirus/templates/s3_bucket_virus_scan.sh.j2
new file mode 100644
index 00000000000..72b568ce45e
--- /dev/null
+++ b/playbooks/roles/antivirus/templates/s3_bucket_virus_scan.sh.j2
@@ -0,0 +1,77 @@
+#! /bin/bash
+
+DEBUG="false"
+BUCKETNAME="none"
+MAILTO=""
+MAILFROM=""
+ANTIVIRUS_S3_AWS_KEY="{{ ANTIVIRUS_S3_AWS_KEY }}"
+ANTIVIRUS_SES_AWS_KEY="{{ ANTIVIRUS_SES_AWS_KEY }}"
+ANTIVIRUS_S3_AWS_SECRET="{{ ANTIVIRUS_S3_AWS_SECRET}}"
+ANTIVIRUS_SES_AWS_SECRET="{{ ANTIVIRUS_SES_AWS_SECRET}}"
+AWS_DEFAULT_REGION="{{ aws_region }}"
+
+
+function usage {
+ echo "$0 - $VERSION";
+ echo "Run ClamAV against the contents of an S3 Bucket.";
+ echo "Usage: $0 [options]";
+ echo "options:";
+ echo " -d Debug mode";
+ echo " -h Usage (this screen)";
+ echo " -b ";
+ echo " -m ";
+ echo " -f ";
+ echo " -k ";
+ echo " -s "
+
+}
+
+while getopts "dhb:m:f:k:s:" optionName; do
+ case "$optionName" in
+ d)
+ DEBUG="true"
+ ;;
+ h)
+ usage;
+ exit;
+ ;;
+ [?])
+ usage;
+ exit;
+ ;;
+ b)
+ BUCKETNAME=$OPTARG;
+ ;;
+ m)
+ MAILTO=$OPTARG;
+ ;;
+ f)
+ MAILFROM=$OPTARG;
+ ;;
+ k)
+ AWS_ACCESS_KEY_ID=$OPTARG;
+ ANTIVIRUS_S3_AWS_KEY=$OPTARG;
+ ANTIVIRUS_SES_AWS_KEY=$OPTARG;
+ ;;
+ s)
+ AWS_SECRET_ACCESS_KEY=$OPTARG;
+ ANTIVIRUS_S3_AWS_SECRET=$OPTARG;
+ ANTIVIRUS_SES_AWS_SECRET=$OPTARG;
+ ;;
+ esac
+done
+
+cd {{ antivirus_app_dir }}
+
+export AWS_ACCESS_KEY_ID=$ANTIVIRUS_S3_AWS_KEY
+export AWS_SECRET_ACCESS_KEY=$ANTIVIRUS_S3_AWS_SECRET
+export AWS_DEFAULT_REGION
+
+mkdir -p data/$BUCKETNAME
+aws s3 sync s3://$BUCKETNAME/ data/$BUCKETNAME
+CLAMOUT=$(clamscan -ri data/$BUCKETNAME);
+if [[ $? -ne 0 ]]; then
+ export AWS_ACCESS_KEY_ID=$ANTIVIRUS_SES_AWS_KEY
+ export AWS_SECRET_ACCESS_KEY=$ANTIVIRUS_SES_AWS_SECRET
+ aws ses send-email --to $MAILTO --from $MAILFROM --subject "Virus Scanner malicious file on $BUCKETNAME" --text "$CLAMOUT"
+fi
diff --git a/playbooks/roles/apache/templates/lms.j2 b/playbooks/roles/apache/templates/lms.j2
index e1ccf96f194..2f5c068a5b9 100644
--- a/playbooks/roles/apache/templates/lms.j2
+++ b/playbooks/roles/apache/templates/lms.j2
@@ -1,7 +1,7 @@
WSGIPythonHome {{ edxapp_venv_dir }}
WSGIRestrictEmbedded On
-
+
ServerName https://{{ lms_env_config.SITE_NAME }}
ServerAlias *.{{ lms_env_config.SITE_NAME }}
UseCanonicalName On
diff --git a/playbooks/roles/aws/defaults/main.yml b/playbooks/roles/aws/defaults/main.yml
index c60912f2220..209b4295675 100644
--- a/playbooks/roles/aws/defaults/main.yml
+++ b/playbooks/roles/aws/defaults/main.yml
@@ -23,29 +23,34 @@ AWS_S3_LOGS: false
# This relies on your server being able to send mail
AWS_S3_LOGS_NOTIFY_EMAIL: dummy@example.com
AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
-# Separate buckets for tracking logs and everything else
-# You should be overriding the environment and deployment vars
-# Order of precedence is left to right for exclude and include options
-AWS_S3_LOG_PATHS:
- - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/tracking"
- path: "{{ COMMON_LOG_DIR }}/tracking/*"
- - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/application"
- path: "{{ COMMON_LOG_DIR }}/!(*tracking*)"
- - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/system"
- path: "/var/log/*"
#
# vars are namespace with the module name.
#
aws_role_name: aws
-aws_data_dir: "{{ COMMON_DATA_DIR }}/aws"
-aws_app_dir: "{{ COMMON_APP_DIR }}/aws"
-aws_s3_sync_script: "{{ aws_app_dir }}/send-logs-to-s3"
-aws_s3_logfile: "{{ aws_log_dir }}/s3-log-sync.log"
-aws_log_dir: "{{ COMMON_LOG_DIR }}/aws"
+
+aws_dirs:
+ home:
+ path: "{{ COMMON_APP_DIR }}/{{ aws_role_name }}"
+ owner: "root"
+ group: "root"
+ mode: "0755"
+ logs:
+ path: "{{ COMMON_LOG_DIR }}/{{ aws_role_name }}"
+ owner: "syslog"
+ group: "syslog"
+ mode: "0700"
+ data:
+ path: "{{ COMMON_DATA_DIR }}/{{ aws_role_name }}"
+ owner: "root"
+ group: "root"
+ mode: "0700"
+
+aws_s3_sync_script: "{{ aws_dirs.home.path }}/send-logs-to-s3"
+aws_s3_logfile: "{{ aws_dirs.logs.path }}/s3-log-sync.log"
aws_region: "us-east-1"
# default path to the aws binary
-s3cmd_cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
+aws_s3cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
aws_cmd: "/usr/local/bin/aws"
#
# OS packages
diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml
index 1fba494e85f..ca694d76e15 100644
--- a/playbooks/roles/aws/tasks/main.yml
+++ b/playbooks/roles/aws/tasks/main.yml
@@ -21,26 +21,14 @@
#
#
-- name: create data directories
+- name: create all service directories
file: >
- path={{ item }}
- state=directory
- owner=root
- group=root
- mode=0700
- with_items:
- - "{{ aws_data_dir }}"
- - "{{ aws_log_dir }}"
-
-- name: create app directory
- file: >
- path={{ item }}
- state=directory
- owner=root
- group=root
- mode=0755
- with_items:
- - "{{ aws_app_dir }}"
+ path="{{ item.value.path }}"
+ state="directory"
+ owner="{{ item.value.owner }}"
+ group="{{ item.value.group }}"
+ mode="{{ item.value.mode }}"
+ with_dict: aws_dirs
- name: install system packages
apt: >
@@ -57,18 +45,18 @@
- name: get s3cmd
get_url: >
url={{ aws_s3cmd_url }}
- dest={{ aws_data_dir }}/
+ dest={{ aws_dirs.data.path }}/
- name: untar s3cmd
shell: >
- tar xf {{ aws_data_dir }}/{{ aws_s3cmd_version }}.tar.gz
- creates={{ aws_app_dir }}/{{ aws_s3cmd_version }}/s3cmd
- chdir={{ aws_app_dir }}
+ tar xf {{ aws_dirs.data.path }}/{{ aws_s3cmd_version }}.tar.gz
+ creates={{ aws_dirs.data.path }}/{{ aws_s3cmd_version }}/s3cmd
+ chdir={{ aws_dirs.home.path }}
- name: create symlink for s3cmd
file: >
- src={{ aws_app_dir }}/{{ aws_s3cmd_version }}/s3cmd
- dest={{ COMMON_BIN_DIR }}/s3cmd
+ src={{ aws_dirs.home.path }}/{{ aws_s3cmd_version }}/s3cmd
+ dest={{ aws_s3cmd }}
state=link
- name: create s3 log sync script
@@ -84,7 +72,7 @@
dest={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
when: AWS_S3_LOGS
-- name: run s3 log sync script on supervisor shutdown
+- name: force logrotate on supervisor stop
template: >
src=etc/init/sync-on-stop.conf.j2
dest=/etc/init/sync-on-stop.conf
@@ -99,4 +87,5 @@
user: root
minute: 0
job: "{{ aws_s3_sync_script }} > /dev/null 2>&1"
+ state: absent
when: AWS_S3_LOGS
diff --git a/playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2 b/playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2
index db9b50d9bec..783417c846f 100644
--- a/playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2
+++ b/playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2
@@ -1,5 +1,6 @@
start on stopped supervisor
-description "sync s3 logs on supervisor shutdown"
+description "sync tracking logs on supervisor shutdown"
script
- /bin/bash {{ aws_s3_sync_script }}
+ /usr/sbin/logrotate -f /etc/logrotate.d/hourly/tracking.log
+ /usr/sbin/logrotate -f /etc/logrotate.d/hourly/edx-services
end script
diff --git a/playbooks/roles/aws/templates/send-logs-to-s3.j2 b/playbooks/roles/aws/templates/send-logs-to-s3.j2
index c3018b5466e..6df79f237f6 100644
--- a/playbooks/roles/aws/templates/send-logs-to-s3.j2
+++ b/playbooks/roles/aws/templates/send-logs-to-s3.j2
@@ -4,13 +4,23 @@
#
# This script can be called from logrotate
# to sync logs to s3
+#
if (( $EUID != 0 )); then
echo "Please run as the root user"
exit 1
fi
-exec > >(tee "{{ aws_s3_logfile }}")
+#
+# Ensure the log processors can read without
+# running as root
+if [ ! -f "{{ aws_s3_logfile }}" ]; then
+ sudo -u syslog touch "{{ aws_s3_logfile }}"
+else
+ chown syslog.syslog "{{ aws_s3_logfile }}"
+fi
+
+exec > >(tee -a "{{ aws_s3_logfile }}")
exec 2>&1
# s3cmd sync requires a valid home
@@ -31,10 +41,12 @@ usage() {
-v add verbosity (set -x)
-n echo what will be done
-h this
+ -d directory to sync
+ -b bucket path to sync to
EO
}
-while getopts "vhn" opt; do
+while getopts "vhnb:d:" opt; do
case $opt in
v)
set -x
@@ -48,9 +60,21 @@ while getopts "vhn" opt; do
noop="echo Would have run: "
shift
;;
+ d)
+ directory=$OPTARG
+ ;;
+ b)
+ bucket_path=$OPTARG
+ ;;
esac
done
+if [[ -z $bucket_path || -z $directory ]]; then
+ echo "ERROR: You must provide a directory and a bucket to sync!"
+ usage
+ exit 1
+fi
+
# grab the first security group for the instance
# which will be used as a directory name in the s3
# bucket
@@ -90,9 +114,7 @@ instance_id=$(ec2metadata --instance-id)
ip=$(ec2metadata --local-ipv4)
availability_zone=$(ec2metadata --availability-zone)
# region isn't available via the metadata service
-region=${availability_zone:0:${{lb}}#availability_zone{{rb}} - 1}
+region=${availability_zone:0:${{ lb }}#availability_zone{{ rb }} - 1}
s3_path="${2}/$sec_grp/"
-{% for item in AWS_S3_LOG_PATHS -%}
-$noop {{ s3cmd_cmd }} sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/"
-{% endfor %}
+$noop {{ aws_s3cmd }} --multipart-chunk-size-mb 5120 --disable-multipart sync $directory "s3://${bucket_path}/${sec_grp}/${instance_id}-${ip}/"
diff --git a/playbooks/roles/browsermob-proxy/files/browsermob-proxy b/playbooks/roles/browsermob-proxy/files/browsermob-proxy
index 9e9fdef16ff..e2344acadcb 100644
--- a/playbooks/roles/browsermob-proxy/files/browsermob-proxy
+++ b/playbooks/roles/browsermob-proxy/files/browsermob-proxy
@@ -1,2 +1,2 @@
#!/bin/sh
-/etc/browsermob-proxy/bin/browsermob-proxy
+/etc/browsermob-proxy/bin/browsermob-proxy $*
diff --git a/playbooks/roles/browsers/defaults/main.yml b/playbooks/roles/browsers/defaults/main.yml
index 8cc4d8100e3..1d05333a569 100644
--- a/playbooks/roles/browsers/defaults/main.yml
+++ b/playbooks/roles/browsers/defaults/main.yml
@@ -13,7 +13,7 @@ browser_deb_pkgs:
# which often causes spurious acceptance test failures.
browser_s3_deb_pkgs:
- { name: "google-chrome-stable_30.0.1599.114-1_amd64.deb", url: "https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_30.0.1599.114-1_amd64.deb" }
- - { name: "firefox_25.0+build3-0ubuntu0.12.04.1_amd64.deb", url: "https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox_25.0%2Bbuild3-0ubuntu0.12.04.1_amd64.deb" }
+ - { name: "firefox_28.0+build2-0ubuntu0.12.04.1_amd64.deb", url: "https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox_28.0%2Bbuild2-0ubuntu0.12.04.1_amd64.deb" }
# Chrome and ChromeDriver
chromedriver_version: 2.6
diff --git a/playbooks/roles/certs/defaults/main.yml b/playbooks/roles/certs/defaults/main.yml
index 8ae5b28e42d..a8c317b51a3 100644
--- a/playbooks/roles/certs/defaults/main.yml
+++ b/playbooks/roles/certs/defaults/main.yml
@@ -71,25 +71,25 @@ certs_env_config:
# CERTS_DATA is legacy, not used
CERT_DATA: {}
QUEUE_NAME: "certificates"
- QUEUE_URL: $CERTS_QUEUE_URL
- CERT_BUCKET: $CERTS_BUCKET
+ QUEUE_URL: "{{ CERTS_QUEUE_URL }}"
+ CERT_BUCKET: "{{ CERTS_BUCKET }}"
# gnupg signing key
- CERT_KEY_ID: $CERTS_KEY_ID
+ CERT_KEY_ID: "{{ CERTS_KEY_ID }}"
LOGGING_ENV: ""
- CERT_GPG_DIR: $certs_gpg_dir
- CERT_URL: $CERTS_URL
- CERT_DOWNLOAD_URL: $CERTS_DOWNLOAD_URL
- CERT_WEB_ROOT: $CERTS_WEB_ROOT
- COPY_TO_WEB_ROOT: $CERTS_COPY_TO_WEB_ROOT
- S3_UPLOAD: $CERTS_S3_UPLOAD
- CERT_VERIFY_URL: $CERTS_VERIFY_URL
- TEMPLATE_DATA_DIR: $CERTS_TEMPLATE_DATA_DIR
+ CERT_GPG_DIR: "{{ certs_gpg_dir }}"
+ CERT_URL: "{{ CERTS_URL }}"
+ CERT_DOWNLOAD_URL: "{{ CERTS_DOWNLOAD_URL }}"
+ CERT_WEB_ROOT: "{{ CERTS_WEB_ROOT }}"
+ COPY_TO_WEB_ROOT: "{{ CERTS_COPY_TO_WEB_ROOT }}"
+ S3_UPLOAD: "{{ CERTS_S3_UPLOAD }}"
+ CERT_VERIFY_URL: "{{ CERTS_VERIFY_URL }}"
+ TEMPLATE_DATA_DIR: "{{ CERTS_TEMPLATE_DATA_DIR }}"
certs_auth_config:
- QUEUE_USER: $CERTS_QUEUE_USER
- QUEUE_PASS: $CERTS_QUEUE_PASS
- QUEUE_AUTH_USER: $CERTS_XQUEUE_AUTH_USER
- QUEUE_AUTH_PASS: $CERTS_XQUEUE_AUTH_PASS
- CERT_KEY_ID: $CERTS_KEY_ID
- CERT_AWS_ID: $CERTS_AWS_ID
- CERT_AWS_KEY: $CERTS_AWS_KEY
+ QUEUE_USER: "{{ CERTS_QUEUE_USER }}"
+ QUEUE_PASS: "{{ CERTS_QUEUE_PASS }}"
+ QUEUE_AUTH_USER: "{{ CERTS_XQUEUE_AUTH_USER }}"
+ QUEUE_AUTH_PASS: "{{ CERTS_XQUEUE_AUTH_PASS }}"
+ CERT_KEY_ID: "{{ CERTS_KEY_ID }}"
+ CERT_AWS_ID: "{{ CERTS_AWS_ID }}"
+ CERT_AWS_KEY: "{{ CERTS_AWS_KEY }}"
diff --git a/playbooks/roles/common/defaults/main.yml b/playbooks/roles/common/defaults/main.yml
index 72980edf6f6..8c8cbaed74b 100644
--- a/playbooks/roles/common/defaults/main.yml
+++ b/playbooks/roles/common/defaults/main.yml
@@ -7,7 +7,12 @@
COMMON_ENABLE_BASIC_AUTH: False
COMMON_HTPASSWD_USER: edx
COMMON_HTPASSWD_PASS: edx
-
+# Turn on syncing logs on rotation for edx
+# application and tracking logs, must also
+# have the AWS role installed
+COMMON_AWS_SYNC: False
+COMMON_AWS_SYNC_BUCKET: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}"
+COMMON_AWS_S3_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-s3"
COMMON_BASE_DIR: /edx
COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"
COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app"
@@ -24,6 +29,7 @@ COMMON_ENVIRONMENT: 'default_env'
COMMON_DEPLOYMENT: 'default_deployment'
COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
COMMON_NPM_MIRROR_URL: 'http://registry.npmjs.org'
+COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search="
# do not include http/https
COMMON_GIT_MIRROR: 'github.com'
# override this var to set a different hostname
@@ -38,6 +44,7 @@ COMMON_CUSTOM_DHCLIENT_CONFIG: false
COMMON_MOTD_TEMPLATE: "motd.tail.j2"
COMMON_SSH_PASSWORD_AUTH: "no"
+COMMON_SECURITY_UPDATES: no
# These are three maintenance accounts across all databases
# the read only user is is granted select privs on all dbs
# the admin user is granted create user privs on all dbs
@@ -69,6 +76,7 @@ common_debian_pkgs:
- mosh
- rsyslog
- screen
+ - tmux
- tree
- git
- unzip
@@ -102,6 +110,10 @@ disable_edx_services: False
# so different start scripts are generated in dev mode.
devstack: False
+# Some cluster apps need special settings when in vagrant
+# due to eth0 always being the same IP address
+vagrant_cluster: False
+
common_debian_variants:
- Ubuntu
- Debian
diff --git a/playbooks/roles/common/meta/main.yml b/playbooks/roles/common/meta/main.yml
index 6a517a3c36f..e43fb06d68a 100644
--- a/playbooks/roles/common/meta/main.yml
+++ b/playbooks/roles/common/meta/main.yml
@@ -2,5 +2,5 @@
dependencies:
- role: user
user_info: "{{ COMMON_USER_INFO }}"
-
-
+ - role: security
+ when: COMMON_SECURITY_UPDATES
diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml
index 4483ba1f1b4..0f94382ec6d 100644
--- a/playbooks/roles/common/tasks/main.yml
+++ b/playbooks/roles/common/tasks/main.yml
@@ -1,4 +1,8 @@
---
+- name: Update CA Certificates
+ shell: >
+ /usr/sbin/update-ca-certificates
+
- name: Add user www-data
# This is the default user for nginx
user: >
diff --git a/playbooks/roles/common/templates/etc/dhcp/dhclient.conf.j2 b/playbooks/roles/common/templates/etc/dhcp/dhclient.conf.j2
index 953c2c78076..2fd4f15d062 100644
--- a/playbooks/roles/common/templates/etc/dhcp/dhclient.conf.j2
+++ b/playbooks/roles/common/templates/etc/dhcp/dhclient.conf.j2
@@ -57,6 +57,6 @@ request subnet-mask, broadcast-address, time-offset, routers,
#}
interface "eth0" {
- prepend domain-search {% for search in COMMON_DHCLIENT_DNS_SEARCH -%}"{{search}}"{%- if not loop.last -%},{%- endif -%}
+ prepend domain-search {% for search in COMMON_DHCLIENT_DNS_SEARCH -%}"{{ search }}"{%- if not loop.last -%},{%- endif -%}
{%- endfor -%};
}
diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2
index 165b4c14bc7..0623eed858d 100644
--- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2
+++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2
@@ -11,4 +11,9 @@
postrotate
/usr/bin/killall -HUP rsyslogd
endscript
+ lastaction
+ {% if COMMON_AWS_SYNC -%}
+ {{ COMMON_AWS_S3_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking/*" -b "{{ COMMON_AWS_SYNC_BUCKET }}/logs/tracking"
+ {% endif -%}
+ endscript
}
diff --git a/playbooks/roles/datadog/defaults/main.yml b/playbooks/roles/datadog/defaults/main.yml
index 8fcc334a267..39f1a3220b4 100644
--- a/playbooks/roles/datadog/defaults/main.yml
+++ b/playbooks/roles/datadog/defaults/main.yml
@@ -1,7 +1,9 @@
---
DATADOG_API_KEY: "SPECIFY_KEY_HERE"
-datadog_apt_key: "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x226AE980C7A7DA52"
+datadog_agent_version: '1:5.0.4-516'
+
+datadog_apt_key: "0x226AE980C7A7DA52"
datadog_debian_pkgs:
- apparmor-utils
- build-essential
diff --git a/playbooks/roles/datadog/tasks/main.yml b/playbooks/roles/datadog/tasks/main.yml
index 85d2540f04d..d52a2031b10 100644
--- a/playbooks/roles/datadog/tasks/main.yml
+++ b/playbooks/roles/datadog/tasks/main.yml
@@ -22,17 +22,22 @@
- datadog
- name: add apt key
- apt_key: id=C7A7DA52 url={{datadog_apt_key}} state=present
+ apt_key: id=C7A7DA52 url={{ COMMON_UBUNTU_APT_KEYSERVER }}{{ datadog_apt_key }} state=present
+ tags:
+ - datadog
+
+- name: remove unstable apt repository
+ apt_repository_1.8: repo='deb http://apt.datadoghq.com/ unstable main' validate_certs=no state=absent
tags:
- datadog
- name: install apt repository
- apt_repository_1.8: repo='deb http://apt.datadoghq.com/ unstable main' update_cache=yes validate_certs=no
+ apt_repository_1.8: repo='deb http://apt.datadoghq.com/ stable main' update_cache=yes validate_certs=no
tags:
- datadog
- name: install datadog agent
- apt: pkg="datadog-agent"
+ apt: pkg="datadog-agent={{ datadog_agent_version }}"
tags:
- datadog
diff --git a/playbooks/roles/demo/defaults/main.yml b/playbooks/roles/demo/defaults/main.yml
index 65314bda297..8910f34b697 100644
--- a/playbooks/roles/demo/defaults/main.yml
+++ b/playbooks/roles/demo/defaults/main.yml
@@ -30,6 +30,6 @@ demo_test_users:
password: edx
demo_edxapp_user: 'edxapp'
-demo_edxapp_venv_bin: '{{COMMON_APP_DIR}}/{{demo_edxapp_user}}/venvs/{{demo_edxapp_user}}/bin'
-demo_edxapp_course_data_dir: '{{COMMON_DATA_DIR}}/{{demo_edxapp_user}}/data'
-demo_edxapp_code_dir: '{{COMMON_APP_DIR}}/{{demo_edxapp_user}}/edx-platform'
+demo_edxapp_venv_bin: '{{ COMMON_APP_DIR }}/{{ demo_edxapp_user }}/venvs/{{demo_edxapp_user}}/bin'
+demo_edxapp_course_data_dir: '{{ COMMON_DATA_DIR }}/{{ demo_edxapp_user }}/data'
+demo_edxapp_code_dir: '{{ COMMON_APP_DIR }}/{{ demo_edxapp_user }}/edx-platform'
diff --git a/playbooks/roles/discern/defaults/main.yml b/playbooks/roles/discern/defaults/main.yml
index cf423c6d93d..6032cb5054e 100644
--- a/playbooks/roles/discern/defaults/main.yml
+++ b/playbooks/roles/discern/defaults/main.yml
@@ -11,7 +11,7 @@ DISCERN_MYSQL_PASSWORD: 'password'
DISCERN_MYSQL_HOST: 'localhost'
DISCERN_MYSQL_PORT: '3306'
DISCERN_LANG: "en_US.UTF-8"
-
+DISCERN_GUNICORN_EXTRA: ""
discern_app_dir: "{{ COMMON_APP_DIR }}/discern"
discern_code_dir: "{{ discern_app_dir }}/discern"
@@ -53,23 +53,23 @@ discern_env_config:
discern_auth_config:
- AWS_ACCESS_KEY_ID: $DISCERN_AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: $DISCERN_SECRET_ACCESS_KEY
- BROKER_URL: $DISCERN_BROKER_URL
+ AWS_ACCESS_KEY_ID: "{{ DISCERN_AWS_ACCESS_KEY_ID }}"
+ AWS_SECRET_ACCESS_KEY: "{{ DISCERN_SECRET_ACCESS_KEY }}"
+ BROKER_URL: "{{ DISCERN_BROKER_URL }}"
CACHES:
default:
BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
- LOCATION: $DISCERN_MEMCACHE
- CELERY_RESULT_BACKEND: $DISCERN_RESULT_BACKEND
+ LOCATION: "{{ DISCERN_MEMCACHE }}"
+ CELERY_RESULT_BACKEND: "{{ DISCERN_RESULT_BACKEND }}"
DATABASES:
default:
ENGINE: django.db.backends.mysql
- HOST: $DISCERN_MYSQL_HOST
- NAME: $DISCERN_MYSQL_DB_NAME
- PASSWORD: $DISCERN_MYSQL_PASSWORD
- PORT: $DISCERN_MYSQL_PORT
- USER: $DISCERN_MYSQL_USER
- GOOGLE_ANALYTICS_PROPERTY_ID: $DISCERN_GOOGLE_ANALYTICS_PROPERTY_ID
+ HOST: "{{ DISCERN_MYSQL_HOST }}"
+ NAME: "{{ DISCERN_MYSQL_DB_NAME }}"
+ PASSWORD: "{{ DISCERN_MYSQL_PASSWORD }}"
+ PORT: "{{ DISCERN_MYSQL_PORT }}"
+ USER: "{{ DISCERN_MYSQL_USER }}"
+ GOOGLE_ANALYTICS_PROPERTY_ID: "{{ DISCERN_GOOGLE_ANALYTICS_PROPERTY_ID }}"
discern_debian_pkgs:
diff --git a/playbooks/roles/discern/tasks/deploy.yml b/playbooks/roles/discern/tasks/deploy.yml
index 230e5e1397d..a8f1476e89b 100644
--- a/playbooks/roles/discern/tasks/deploy.yml
+++ b/playbooks/roles/discern/tasks/deploy.yml
@@ -51,7 +51,7 @@
#Numpy has to be a pre-requirement in order for scipy to build
- name : install python pre-requirements for discern and ease
pip: >
- requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
+ requirements={{ item }} virtualenv={{ discern_venv_dir }} state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ discern_user }}"
notify:
@@ -62,7 +62,7 @@
- name : install python requirements for discern and ease
pip: >
- requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
+ requirements={{ item }} virtualenv={{ discern_venv_dir }} state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ discern_user }}"
notify:
@@ -84,8 +84,8 @@
tar zxf {{ discern_nltk_tmp_file }}
rm -f {{ discern_nltk_tmp_file }}
touch {{ discern_nltk_download_url|basename }}-installed
- creates={{ discern_data_dir }}/{{ discern_nltk_download_url|basename }}-installed
- chdir={{ discern_data_dir }}
+ creates={{ discern_data_dir }}/{{ discern_nltk_download_url|basename }}-installed
+ chdir={{ discern_data_dir }}
sudo_user: "{{ discern_user }}"
notify:
- restart discern
@@ -95,8 +95,8 @@
#support virtualenvs as of this comment
- name: django syncdb migrate and collectstatic for discern
shell: >
- {{ discern_venv_dir }}/bin/python {{discern_code_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_code_dir}}
- chdir={{ discern_code_dir }}
+ {{ discern_venv_dir }}/bin/python {{ discern_code_dir }}/manage.py {{ item }} --noinput --settings={{discern_settings}} --pythonpath={{discern_code_dir}}
+ chdir={{ discern_code_dir }}
sudo_user: "{{ discern_user }}"
notify:
- restart discern
@@ -107,8 +107,8 @@
#Have this separate from the other three because it doesn't take the noinput flag
- name: django update_index for discern
shell: >
- {{ discern_venv_dir}}/bin/python {{discern_code_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_code_dir}}
- chdir={{ discern_code_dir }}
+ {{ discern_venv_dir}}/bin/python {{ discern_code_dir }}/manage.py update_index --settings={{ discern_settings }} --pythonpath={{discern_code_dir}}
+ chdir={{ discern_code_dir }}
sudo_user: "{{ discern_user }}"
notify:
- restart discern
diff --git a/playbooks/roles/discern/templates/celery.conf.j2 b/playbooks/roles/discern/templates/celery.conf.j2
index fb9d6312a74..db247a39412 100644
--- a/playbooks/roles/discern/templates/celery.conf.j2
+++ b/playbooks/roles/discern/templates/celery.conf.j2
@@ -9,9 +9,9 @@ stop on runlevel [!2345]
respawn
respawn limit 3 30
-env DJANGO_SETTINGS_MODULE={{discern_settings}}
+env DJANGO_SETTINGS_MODULE={{ discern_settings }}
chdir {{ discern_code_dir }}
-setuid {{discern_user}}
+setuid {{ discern_user }}
exec {{ discern_venv_dir }}/bin/python {{ discern_code_dir }}/manage.py celeryd --loglevel=info --settings={{ discern_settings }} --pythonpath={{ discern_code_dir }} -B --autoscale={{ ansible_processor_cores * 2 }},1
diff --git a/playbooks/roles/discern/templates/discern.conf.j2 b/playbooks/roles/discern/templates/discern.conf.j2
index 8b64ce24d57..555f8712ab4 100644
--- a/playbooks/roles/discern/templates/discern.conf.j2
+++ b/playbooks/roles/discern/templates/discern.conf.j2
@@ -1,9 +1,9 @@
[program:discern]
{% if ansible_processor|length > 0 %}
-command={{ discern_venv_bin }}/gunicorn --preload -b {{ discern_gunicorn_host }}:{{ discern_gunicorn_port }} -w {{ ansible_processor|length * discern_worker_mult }} --timeout=30 --pythonpath={{ discern_code_dir }} discern.wsgi
+command={{ discern_venv_bin }}/gunicorn --preload -b {{ discern_gunicorn_host }}:{{ discern_gunicorn_port }} -w {{ ansible_processor|length * discern_worker_mult }} --timeout=30 --pythonpath={{ discern_code_dir }} {{ DISCERN_GUNICORN_EXTRA }} discern.wsgi
{% else %}
-command={{ discern_venv_bin }}/gunicorn --preload -b {{ discern_gunicorn_host }}:{{ discern_gunicorn_port }} -w {{ discern_worker_mult }} --timeout=30 --pythonpath={{ discern_code_dir }} discern.wsgi
+command={{ discern_venv_bin }}/gunicorn --preload -b {{ discern_gunicorn_host }}:{{ discern_gunicorn_port }} -w {{ discern_worker_mult }} --timeout=30 --pythonpath={{ discern_code_dir }} {{ DISCERN_GUNICORN_EXTRA }} discern.wsgi
{% endif %}
user={{ common_web_user }}
directory={{ discern_code_dir }}
diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2
index d950464ac14..893cfaa8578 100644
--- a/playbooks/roles/edx_ansible/templates/update.j2
+++ b/playbooks/roles/edx_ansible/templates/update.j2
@@ -12,7 +12,7 @@ IFS=","
-v add verbosity to edx_ansible run
-h this
- - must be one of edx-platform, xqueue, cs_comments_service, xserver, ease, edx-ora, configuration, read-only-certificate-code edx-analytics-data-api
+ - must be one of edx-platform, xqueue, cs_comments_service, xserver, ease, edx-ora, configuration, read-only-certificate-code, edx-analytics-data-api
- can be a commit or tag
EO
@@ -51,6 +51,7 @@ repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuratio
repos_to_cmd["read-only-certificate-code"]="$edx_ansible_cmd certs.yml -e 'certs_version=$2'"
repos_to_cmd["edx-analytics-data-api"]="$edx_ansible_cmd analyticsapi.yml -e 'ANALYTICS_API_VERSION=$2'"
repos_to_cmd["edx-ora2"]="$edx_ansible_cmd ora2.yml -e 'ora2_version=$2'"
+repos_to_cmd["insights"]="$edx_ansible_cmd insights.yml -e 'INSIGHTS_VERSION=$2'"
if [[ -z $1 || -z $2 ]]; then
diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml
index 6a60f144268..ec73b21ec50 100644
--- a/playbooks/roles/edxapp/defaults/main.yml
+++ b/playbooks/roles/edxapp/defaults/main.yml
@@ -27,6 +27,9 @@ EDXAPP_LMS_BASE: ""
EDXAPP_PREVIEW_LMS_BASE: ""
EDXAPP_CMS_BASE: ""
+EDXAPP_LMS_GUNICORN_EXTRA: ""
+EDXAPP_CMS_GUNICORN_EXTRA: ""
+
# Set this to the maximum number
# of requests for gunicorn for the lms and cms
# gunicorn --max-requests
@@ -63,6 +66,9 @@ EDXAPP_MYSQL_REPLICA_PORT: "{{ EDXAPP_MYSQL_PORT }}"
EDXAPP_MYSQL_HOST: 'localhost'
EDXAPP_MYSQL_PORT: '3306'
+EDXAPP_LMS_ENV: 'lms.envs.aws'
+EDXAPP_CMS_ENV: 'cms.envs.aws'
+
EDXAPP_EMAIL_BACKEND: 'django.core.mail.backends.smtp.EmailBackend'
EDXAPP_EMAIL_HOST: 'localhost'
EDXAPP_EMAIL_PORT: 25
@@ -105,7 +111,7 @@ EDXAPP_CAS_ATTRIBUTE_PACKAGE: ""
EDXAPP_ENABLE_AUTO_AUTH: false
# Settings for enabling and configuring third party authorization
EDXAPP_ENABLE_THIRD_PARTY_AUTH: false
-EDXAPP_THIRD_PARTY_AUTH: "None"
+EDXAPP_THIRD_PARTY_AUTH: {}
EDXAPP_MODULESTORE_MAPPINGS:
'preview\.': 'draft-preferred'
@@ -114,12 +120,13 @@ EDXAPP_FEATURES:
AUTH_USE_OPENID_PROVIDER: true
CERTIFICATES_ENABLED: true
ENABLE_DISCUSSION_SERVICE: true
- ENABLE_INSTRUCTOR_ANALYTICS: true
+ ENABLE_INSTRUCTOR_ANALYTICS: false
SUBDOMAIN_BRANDING: false
SUBDOMAIN_COURSE_LISTINGS: false
PREVIEW_LMS_BASE: "{{ EDXAPP_PREVIEW_LMS_BASE }}"
ENABLE_S3_GRADE_DOWNLOADS: true
USE_CUSTOM_THEME: $edxapp_use_custom_theme
+ ENABLE_MKTG_SITE: $EDXAPP_ENABLE_MKTG_SITE
AUTOMATIC_AUTH_FOR_TESTING: $EDXAPP_ENABLE_AUTO_AUTH
ENABLE_THIRD_PARTY_AUTH: $EDXAPP_ENABLE_THIRD_PARTY_AUTH
@@ -147,6 +154,11 @@ EDXAPP_LMS_PREVIEW_NGINX_PORT: 18020
EDXAPP_CMS_NGINX_PORT: 18010
EDXAPP_CMS_SSL_NGINX_PORT: 48010
+# NGINX Rate limiting related vars
+EDXAPP_ENABLE_RATE_LIMITING: false
+EDXAPP_COURSE_REQUEST_RATE: '5r/s'
+EDXAPP_COURSE_REQUEST_BURST_RATE: 10
+
EDXAPP_LANG: 'en_US.UTF-8'
EDXAPP_LANGUAGE_CODE : 'en'
EDXAPP_TIME_ZONE: 'America/New_York'
@@ -162,9 +174,16 @@ EDXAPP_UNIVERSITY_EMAIL: 'university@example.com'
EDXAPP_PRESS_EMAIL: 'press@example.com'
EDXAPP_PLATFORM_TWITTER_ACCOUNT: '@YourPlatformTwitterAccount'
EDXAPP_PLATFORM_FACEBOOK_ACCOUNT: 'http://www.facebook.com/YourPlatformFacebookAccount'
+EDXAPP_PLATFORM_TWITTER_URL: "https://twitter.com/YourPlatformTwitterAccount"
+EDXAPP_PLATFORM_MEETUP_URL: "http://www.meetup.com/YourMeetup"
+EDXAPP_PLATFORM_LINKEDIN_URL: "http://www.linkedin.com/company/YourPlatform"
+EDXAPP_PLATFORM_GOOGLE_PLUS_URL: "https://plus.google.com/YourGooglePlusAccount/"
EDXAPP_ENV_EXTRA: {}
EDXAPP_AUTH_EXTRA: {}
+EDXAPP_LMS_AUTH_EXTRA: "{{ EDXAPP_AUTH_EXTRA }}"
+EDXAPP_CMS_AUTH_EXTRA: "{{ EDXAPP_AUTH_EXTRA }}"
+EDXAPP_ENABLE_MKTG_SITE: false
EDXAPP_MKTG_URL_LINK_MAP: {}
EDXAPP_MKTG_URLS: {}
# Set this sets the url for static files
@@ -241,6 +260,7 @@ EDXAPP_PEARSON_TEST_PASSWORD: ""
EDXAPP_SEGMENT_IO_LMS: false
EDXAPP_SEGMENT_IO_LMS_KEY: ""
EDXAPP_OPTIMIZELY_PROJECT_ID: "None"
+EDXAPP_TRACKING_SEGMENTIO_WEBHOOK_SECRET: ""
# For the CMS
EDXAPP_SEGMENT_IO_KEY: ""
EDXAPP_SEGMENT_IO: false
@@ -266,22 +286,22 @@ EDXAPP_XML_FROM_GIT: false
EDXAPP_XML_S3_BUCKET: !!null
EDXAPP_XML_S3_KEY: !!null
-EDXAPP_NEWRELIC_LMS_APPNAME: "{{COMMON_ENVIRONMENT}}-{{COMMON_DEPLOYMENT}}-edxapp-lms"
-EDXAPP_NEWRELIC_CMS_APPNAME: "{{COMMON_ENVIRONMENT}}-{{COMMON_DEPLOYMENT}}-edxapp-cms"
+EDXAPP_NEWRELIC_LMS_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-edxapp-lms"
+EDXAPP_NEWRELIC_CMS_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-edxapp-cms"
EDXAPP_AWS_STORAGE_BUCKET_NAME: 'edxuploads'
-EDXAPP_ORA2_FILE_PREFIX: '{{COMMON_ENVIRONMENT}}-{{COMMON_DEPLOYMENT}}/ora2'
+EDXAPP_ORA2_FILE_PREFIX: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/ora2'
EDXAPP_FILE_UPLOAD_STORAGE_BUCKET_NAME: '{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}'
EDXAPP_FILE_UPLOAD_STORAGE_PREFIX: 'submissions_attachments'
EDXAPP_CODE_JAIL_LIMITS:
# Limit the memory of the jailed process to something high but not
- # infinite (128MiB in bytes)
- VMEM: 134217728
+ # infinite (512MiB in bytes)
+ VMEM: 536870912
# Time in seconds that the jailed process has to run.
- REALTIME: 1
+ REALTIME: 3
# Needs to be non-zero so that jailed code can use it as their temp directory.(1MiB in bytes)
FSIZE: 1048576
@@ -295,6 +315,44 @@ EDXAPP_SUBDOMAIN_BRANDING: {}
EDXAPP_WORKERS: !!null
EDXAPP_ANALYTICS_DATA_TOKEN: ""
EDXAPP_ANALYTICS_DATA_URL: ""
+# Dashboard URL, assumes that the insights role is installed locally
+EDXAPP_ANALYTICS_DASHBOARD_URL: "http://localhost:18110/courses"
+
+EDXAPP_REGISTRATION_EXTRA_FIELDS:
+ level_of_education: "optional"
+ gender: "optional"
+ year_of_birth: "optional"
+ mailing_address: "optional"
+ goals: "optional"
+ honor_code: "required"
+ city: "hidden"
+ country: "hidden"
+
+EDXAPP_CELERY_WORKERS:
+ - queue: low
+ service_variant: cms
+ concurrency: 3
+ - queue: default
+ service_variant: cms
+ concurrency: 4
+ - queue: high
+ service_variant: cms
+ concurrency: 1
+ - queue: low
+ service_variant: lms
+ concurrency: 1
+ - queue: default
+ service_variant: lms
+ concurrency: 3
+ - queue: high
+ service_variant: lms
+ concurrency: 4
+ - queue: high_mem
+ service_variant: lms
+ concurrency: 2
+
+EDXAPP_DEFAULT_CACHE_VERSION: "1"
+
#-------- Everything below this line is internal to the role ------------
#Use YAML references (& and *) and hash merge <<: to factor out shared settings
@@ -326,28 +384,7 @@ edxapp_git_ssh: "/tmp/edxapp_git_ssh.sh"
# TODO: This can be removed once VPC-122 is resolved
edxapp_legacy_course_data_dir: "{{ edxapp_app_dir }}/data"
-edxapp_workers:
- - queue: low
- service_variant: cms
- concurrency: 3
- - queue: default
- service_variant: cms
- concurrency: 4
- - queue: high
- service_variant: cms
- concurrency: 1
- - queue: low
- service_variant: lms
- concurrency: 1
- - queue: default
- service_variant: lms
- concurrency: 3
- - queue: high
- service_variant: lms
- concurrency: 4
- - queue: high_mem
- service_variant: lms
- concurrency: 2
+edxapp_workers: "{{ EDXAPP_CELERY_WORKERS }}"
# setup for python codejail
edxapp_sandbox_venv_dir: '{{ edxapp_venvs_dir }}/edxapp-sandbox'
@@ -384,30 +421,30 @@ edxapp_all_req_files:
# for lists and dictionaries
edxapp_environment:
- LANG: $EDXAPP_LANG
- NO_PREREQ_INSTALL: $EDXAPP_NO_PREREQ_INSTALL
+ LANG: "{{ EDXAPP_LANG }}"
+ NO_PREREQ_INSTALL: "{{ EDXAPP_NO_PREREQ_INSTALL }}"
SKIP_WS_MIGRATIONS: 1
- RBENV_ROOT: $edxapp_rbenv_root
- GEM_HOME: $edxapp_gem_root
- GEM_PATH: $edxapp_gem_root
- PATH: $edxapp_deploy_path
+ RBENV_ROOT: "{{ edxapp_rbenv_root }}"
+ GEM_HOME: "{{ edxapp_gem_root }}"
+ GEM_PATH: "{{ edxapp_gem_root }}"
+ PATH: "{{ edxapp_deploy_path }}"
edxapp_generic_auth_config: &edxapp_generic_auth
- ANALYTICS_DATA_TOKEN: $EDXAPP_ANALYTICS_DATA_TOKEN
- AWS_ACCESS_KEY_ID: $EDXAPP_AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: $EDXAPP_AWS_SECRET_ACCESS_KEY
- SECRET_KEY: $EDXAPP_EDXAPP_SECRET_KEY
+ ANALYTICS_DATA_TOKEN: "{{ EDXAPP_ANALYTICS_DATA_TOKEN }}"
+ AWS_ACCESS_KEY_ID: "{{ EDXAPP_AWS_ACCESS_KEY_ID }}"
+ AWS_SECRET_ACCESS_KEY: "{{ EDXAPP_AWS_SECRET_ACCESS_KEY }}"
+ SECRET_KEY: "{{ EDXAPP_EDXAPP_SECRET_KEY }}"
XQUEUE_INTERFACE:
- basic_auth: $EDXAPP_XQUEUE_BASIC_AUTH
- django_auth: $EDXAPP_XQUEUE_DJANGO_AUTH
- url: $EDXAPP_XQUEUE_URL
+ basic_auth: "{{ EDXAPP_XQUEUE_BASIC_AUTH }}"
+ django_auth: "{{ EDXAPP_XQUEUE_DJANGO_AUTH }}"
+ url: "{{ EDXAPP_XQUEUE_URL }}"
DOC_STORE_CONFIG: &edxapp_generic_default_docstore
- db: $EDXAPP_MONGO_DB_NAME
- host: $EDXAPP_MONGO_HOSTS
- password: $EDXAPP_MONGO_PASSWORD
+ db: "{{ EDXAPP_MONGO_DB_NAME }}"
+ host: "{{ EDXAPP_MONGO_HOSTS }}"
+ password: "{{ EDXAPP_MONGO_PASSWORD }}"
port: $EDXAPP_MONGO_PORT
- user: $EDXAPP_MONGO_USER
+ user: "{{ EDXAPP_MONGO_USER }}"
collection: 'modulestore'
CONTENTSTORE:
ENGINE: 'xmodule.contentstore.mongo.MongoContentStore'
@@ -416,18 +453,18 @@ edxapp_generic_auth_config: &edxapp_generic_auth
# backward compatibility
#
OPTIONS:
- db: $EDXAPP_MONGO_DB_NAME
- host: $EDXAPP_MONGO_HOSTS
- password: $EDXAPP_MONGO_PASSWORD
+ db: "{{ EDXAPP_MONGO_DB_NAME }}"
+ host: "{{ EDXAPP_MONGO_HOSTS }}"
+ password: "{{ EDXAPP_MONGO_PASSWORD }}"
port: $EDXAPP_MONGO_PORT
- user: $EDXAPP_MONGO_USER
- ADDITIONAL_OPTIONS: $EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS
+ user: "{{ EDXAPP_MONGO_USER }}"
+ ADDITIONAL_OPTIONS: "{{ EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS }}"
DOC_STORE_CONFIG: *edxapp_generic_default_docstore
MODULESTORE:
default:
ENGINE: 'xmodule.modulestore.mixed.MixedModuleStore'
OPTIONS:
- mappings: $EDXAPP_XML_MAPPINGS
+ mappings: "{{ EDXAPP_XML_MAPPINGS }}"
stores:
- &edxapp_generic_draft_modulestore
NAME: 'draft'
@@ -435,13 +472,13 @@ edxapp_generic_auth_config: &edxapp_generic_auth
DOC_STORE_CONFIG: *edxapp_generic_default_docstore
OPTIONS:
default_class: 'xmodule.hidden_module.HiddenDescriptor'
- fs_root: $edxapp_course_data_dir
+ fs_root: "{{ edxapp_course_data_dir }}"
render_template: 'edxmako.shortcuts.render_to_string'
- &edxapp_generic_xml_modulestore
NAME: 'xml'
ENGINE: 'xmodule.modulestore.xml.XMLModuleStore'
OPTIONS:
- data_dir: $edxapp_course_data_dir
+ data_dir: "{{ edxapp_course_data_dir }}"
default_class: 'xmodule.hidden_module.HiddenDescriptor'
- &edxapp_generic_split_modulestore
NAME: 'split'
@@ -449,86 +486,96 @@ edxapp_generic_auth_config: &edxapp_generic_auth
DOC_STORE_CONFIG: *edxapp_generic_default_docstore
OPTIONS:
default_class: 'xmodule.hidden_module.HiddenDescriptor'
- fs_root: $edxapp_course_data_dir
+ fs_root: "{{ edxapp_course_data_dir }}"
render_template: 'edxmako.shortcuts.render_to_string'
DATABASES:
read_replica:
ENGINE: 'django.db.backends.mysql'
- NAME: $EDXAPP_MYSQL_REPLICA_DB_NAME
- USER: $EDXAPP_MYSQL_REPLICA_USER
- PASSWORD: $EDXAPP_MYSQL_REPLICA_PASSWORD
- HOST: $EDXAPP_MYSQL_REPLICA_HOST
- PORT: $EDXAPP_MYSQL_REPLICA_PORT
+ NAME: "{{ EDXAPP_MYSQL_REPLICA_DB_NAME }}"
+ USER: "{{ EDXAPP_MYSQL_REPLICA_USER }}"
+ PASSWORD: "{{ EDXAPP_MYSQL_REPLICA_PASSWORD }}"
+ HOST: "{{ EDXAPP_MYSQL_REPLICA_HOST }}"
+ PORT: "{{ EDXAPP_MYSQL_REPLICA_PORT }}"
default:
ENGINE: 'django.db.backends.mysql'
- NAME: $EDXAPP_MYSQL_DB_NAME
- USER: $EDXAPP_MYSQL_USER
- PASSWORD: $EDXAPP_MYSQL_PASSWORD
- HOST: $EDXAPP_MYSQL_HOST
- PORT: $EDXAPP_MYSQL_PORT
+ NAME: "{{ EDXAPP_MYSQL_DB_NAME }}"
+ USER: "{{ EDXAPP_MYSQL_USER }}"
+ PASSWORD: "{{ EDXAPP_MYSQL_PASSWORD }}"
+ HOST: "{{ EDXAPP_MYSQL_HOST }}"
+ PORT: "{{ EDXAPP_MYSQL_PORT }}"
OPEN_ENDED_GRADING_INTERFACE:
- url: $EDXAPP_OEE_URL
- password: $EDXAPP_OEE_PASSWORD
+ url: "{{ EDXAPP_OEE_URL }}"
+ password: "{{ EDXAPP_OEE_PASSWORD }}"
peer_grading: 'peer_grading'
staff_grading: 'staff_grading'
grading_controller: 'grading_controller'
- username: $EDXAPP_OEE_USER
- ANALYTICS_API_KEY: $EDXAPP_ANALYTICS_API_KEY
- EMAIL_HOST_USER: $EDXAPP_EMAIL_HOST_USER
- EMAIL_HOST_PASSWORD: $EDXAPP_EMAIL_HOST_PASSWORD
- ZENDESK_USER: $EDXAPP_ZENDESK_USER
- ZENDESK_API_KEY: $EDXAPP_ZENDESK_API_KEY
- CELERY_BROKER_USER: $EDXAPP_CELERY_USER
- CELERY_BROKER_PASSWORD: $EDXAPP_CELERY_PASSWORD
- GOOGLE_ANALYTICS_ACCOUNT: $EDXAPP_GOOGLE_ANALYTICS_ACCOUNT
- THIRD_PARTY_AUTH: $EDXAPP_THIRD_PARTY_AUTH
+ username: "{{ EDXAPP_OEE_USER }}"
+ ANALYTICS_API_KEY: "{{ EDXAPP_ANALYTICS_API_KEY }}"
+ EMAIL_HOST_USER: "{{ EDXAPP_EMAIL_HOST_USER }}"
+ EMAIL_HOST_PASSWORD: "{{ EDXAPP_EMAIL_HOST_PASSWORD }}"
+ ZENDESK_USER: "{{ EDXAPP_ZENDESK_USER }}"
+ ZENDESK_API_KEY: "{{ EDXAPP_ZENDESK_API_KEY }}"
+ CELERY_BROKER_USER: "{{ EDXAPP_CELERY_USER }}"
+ CELERY_BROKER_PASSWORD: "{{ EDXAPP_CELERY_PASSWORD }}"
+ GOOGLE_ANALYTICS_ACCOUNT: "{{ EDXAPP_GOOGLE_ANALYTICS_ACCOUNT }}"
+ THIRD_PARTY_AUTH: "{{ EDXAPP_THIRD_PARTY_AUTH }}"
AWS_STORAGE_BUCKET_NAME: "{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}"
+ DJFS: $EDXAPP_DJFS
+
+generic_cache_config: &default_generic_cache
+ BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
+ KEY_FUNCTION: 'util.memcache.safe_key'
+ KEY_PREFIX: 'default'
+ LOCATION: "{{ EDXAPP_MEMCACHE }}"
generic_env_config: &edxapp_generic_env
- ANALYTICS_DATA_URL: $EDXAPP_ANALYTICS_DATA_URL
- CELERY_BROKER_VHOST: $EDXAPP_CELERY_BROKER_VHOST
- PAYMENT_SUPPORT_EMAIL: $EDXAPP_PAYMENT_SUPPORT_EMAIL
- ZENDESK_URL: $EDXAPP_ZENDESK_URL
- COURSES_WITH_UNSAFE_CODE: $EDXAPP_COURSES_WITH_UNSAFE_CODE
+ OAUTH_OIDC_ISSUER: "https://{{ EDXAPP_LMS_BASE }}/oauth2"
+ XBLOCK_FS_STORAGE_BUCKET: "{{ EDXAPP_XBLOCK_FS_STORAGE_BUCKET }}"
+ XBLOCK_FS_STORAGE_PREFIX: "{{ EDXAPP_XBLOCK_FS_STORAGE_PREFIX }}"
+ ANALYTICS_DATA_URL: "{{ EDXAPP_ANALYTICS_DATA_URL }}"
+ ANALYTICS_DASHBOARD_URL: '{{ EDXAPP_ANALYTICS_DASHBOARD_URL }}'
+ CELERY_BROKER_VHOST: "{{ EDXAPP_CELERY_BROKER_VHOST }}"
+ PAYMENT_SUPPORT_EMAIL: "{{ EDXAPP_PAYMENT_SUPPORT_EMAIL }}"
+ ZENDESK_URL: "{{ EDXAPP_ZENDESK_URL }}"
+ COURSES_WITH_UNSAFE_CODE: "{{ EDXAPP_COURSES_WITH_UNSAFE_CODE }}"
BULK_EMAIL_EMAILS_PER_TASK: $EDXAPP_BULK_EMAIL_EMAILS_PER_TASK
- MICROSITE_ROOT_DIR: $EDXAPP_MICROSITE_ROOT_DIR
+ MICROSITE_ROOT_DIR: "{{ EDXAPP_MICROSITE_ROOT_DIR }}"
MICROSITE_CONFIGURATION: $EDXAPP_MICROSITE_CONFIGURATION
GRADES_DOWNLOAD:
- STORAGE_TYPE: $EDXAPP_GRADE_STORAGE_TYPE
- BUCKET: $EDXAPP_GRADE_BUCKET
- ROOT_PATH: $EDXAPP_GRADE_ROOT_PATH
- STATIC_URL_BASE: $EDXAPP_STATIC_URL_BASE
- STATIC_ROOT_BASE: $edxapp_staticfile_dir
- LMS_BASE: $EDXAPP_LMS_BASE
- CMS_BASE: $EDXAPP_CMS_BASE
- BOOK_URL: $EDXAPP_BOOK_URL
- PLATFORM_NAME: $EDXAPP_PLATFORM_NAME
+ STORAGE_TYPE: "{{ EDXAPP_GRADE_STORAGE_TYPE }}"
+ BUCKET: "{{ EDXAPP_GRADE_BUCKET }}"
+ ROOT_PATH: "{{ EDXAPP_GRADE_ROOT_PATH }}"
+ STATIC_URL_BASE: "{{ EDXAPP_STATIC_URL_BASE }}"
+ STATIC_ROOT_BASE: "{{ edxapp_staticfile_dir }}"
+ LMS_BASE: "{{ EDXAPP_LMS_BASE }}"
+ CMS_BASE: "{{ EDXAPP_CMS_BASE }}"
+ BOOK_URL: "{{ EDXAPP_BOOK_URL }}"
+ PLATFORM_NAME: "{{ EDXAPP_PLATFORM_NAME }}"
CERT_QUEUE: 'certificates'
- LOCAL_LOGLEVEL: $EDXAPP_LOG_LEVEL
+ LOCAL_LOGLEVEL: "{{ EDXAPP_LOG_LEVEL }}"
# default email backed set to local SMTP
- EMAIL_BACKEND: $EDXAPP_EMAIL_BACKEND
- EMAIL_HOST: $EDXAPP_EMAIL_HOST
+ EMAIL_BACKEND: "{{ EDXAPP_EMAIL_BACKEND }}"
+ EMAIL_HOST: "{{ EDXAPP_EMAIL_HOST }}"
EMAIL_PORT: $EDXAPP_EMAIL_PORT
EMAIL_USE_TLS: $EDXAPP_EMAIL_USE_TLS
- FEATURES: $EDXAPP_FEATURES
+ FEATURES: "{{ EDXAPP_FEATURES }}"
WIKI_ENABLED: true
- SYSLOG_SERVER: $EDXAPP_SYSLOG_SERVER
+ SYSLOG_SERVER: "{{ EDXAPP_SYSLOG_SERVER }}"
LOG_DIR: "{{ COMMON_DATA_DIR }}/logs/edx"
- MEDIA_URL: $EDXAPP_MEDIA_URL
- ANALYTICS_SERVER_URL: $EDXAPP_ANALYTICS_SERVER_URL
- FEEDBACK_SUBMISSION_EMAIL: $EDXAPP_FEEDBACK_SUBMISSION_EMAIL
- TIME_ZONE: $EDXAPP_TIME_ZONE
- LANGUAGE_CODE : $EDXAPP_LANGUAGE_CODE
- MKTG_URL_LINK_MAP: $EDXAPP_MKTG_URL_LINK_MAP
- MKTG_URLS: $EDXAPP_MKTG_URLS
+ MEDIA_URL: "{{ EDXAPP_MEDIA_URL }}"
+ ANALYTICS_SERVER_URL: "{{ EDXAPP_ANALYTICS_SERVER_URL }}"
+ FEEDBACK_SUBMISSION_EMAIL: "{{ EDXAPP_FEEDBACK_SUBMISSION_EMAIL }}"
+ TIME_ZONE: "{{ EDXAPP_TIME_ZONE }}"
+ LANGUAGE_CODE: "{{ EDXAPP_LANGUAGE_CODE }}"
+ MKTG_URL_LINK_MAP: "{{ EDXAPP_MKTG_URL_LINK_MAP }}"
+ MKTG_URLS: "{{ EDXAPP_MKTG_URLS }}"
# repo root for courses
- GITHUB_REPO_ROOT: $edxapp_course_data_dir
+ GITHUB_REPO_ROOT: "{{ edxapp_course_data_dir }}"
CACHES:
- default: &default_generic_cache
- BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
- KEY_FUNCTION: 'util.memcache.safe_key'
+ default:
+ <<: *default_generic_cache
KEY_PREFIX: 'default'
- LOCATION: $EDXAPP_MEMCACHE
+ VERSION: "{{ EDXAPP_DEFAULT_CACHE_VERSION }}"
general:
<<: *default_generic_cache
KEY_PREFIX: 'general'
@@ -544,67 +591,72 @@ generic_env_config: &edxapp_generic_env
KEY_PREFIX: 'celery'
TIMEOUT: "7200"
CELERY_BROKER_TRANSPORT: 'amqp'
- CELERY_BROKER_HOSTNAME: $EDXAPP_RABBIT_HOSTNAME
- COMMENTS_SERVICE_URL: $EDXAPP_COMMENTS_SERVICE_URL
- LOGGING_ENV: $EDXAPP_LOGGING_ENV
- SESSION_COOKIE_DOMAIN: $EDXAPP_SESSION_COOKIE_DOMAIN
- SESSION_COOKIE_NAME: $EDXAPP_SESSION_COOKIE_NAME
- COMMENTS_SERVICE_KEY: $EDXAPP_COMMENTS_SERVICE_KEY
+ CELERY_BROKER_HOSTNAME: "{{ EDXAPP_RABBIT_HOSTNAME }}"
+ COMMENTS_SERVICE_URL: "{{ EDXAPP_COMMENTS_SERVICE_URL }}"
+ LOGGING_ENV: "{{ EDXAPP_LOGGING_ENV }}"
+ SESSION_COOKIE_DOMAIN: "{{ EDXAPP_SESSION_COOKIE_DOMAIN }}"
+ SESSION_COOKIE_NAME: "{{ EDXAPP_SESSION_COOKIE_NAME }}"
+ COMMENTS_SERVICE_KEY: "{{ EDXAPP_COMMENTS_SERVICE_KEY }}"
SEGMENT_IO_LMS: $EDXAPP_SEGMENT_IO_LMS
SEGMENT_IO: $EDXAPP_SEGMENT_IO
- THEME_NAME: $edxapp_theme_name
- TECH_SUPPORT_EMAIL: $EDXAPP_TECH_SUPPORT_EMAIL
- CONTACT_EMAIL: $EDXAPP_CONTACT_EMAIL
- BUGS_EMAIL: $EDXAPP_BUGS_EMAIL
- DEFAULT_FROM_EMAIL: $EDXAPP_DEFAULT_FROM_EMAIL
- DEFAULT_FEEDBACK_EMAIL: $EDXAPP_DEFAULT_FEEDBACK_EMAIL
- SERVER_EMAIL: $EDXAPP_DEFAULT_SERVER_EMAIL
- BULK_EMAIL_DEFAULT_FROM_EMAIL: $EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL
- CAS_SERVER_URL: $EDXAPP_CAS_SERVER_URL
- CAS_EXTRA_LOGIN_PARAMS: $EDXAPP_CAS_EXTRA_LOGIN_PARAMS
- CAS_ATTRIBUTE_CALLBACK: $EDXAPP_CAS_ATTRIBUTE_CALLBACK
+ THEME_NAME: "{{ edxapp_theme_name }}"
+ TECH_SUPPORT_EMAIL: "{{ EDXAPP_TECH_SUPPORT_EMAIL }}"
+ CONTACT_EMAIL: "{{ EDXAPP_CONTACT_EMAIL }}"
+ BUGS_EMAIL: "{{ EDXAPP_BUGS_EMAIL }}"
+ DEFAULT_FROM_EMAIL: "{{ EDXAPP_DEFAULT_FROM_EMAIL }}"
+ DEFAULT_FEEDBACK_EMAIL: "{{ EDXAPP_DEFAULT_FEEDBACK_EMAIL }}"
+ SERVER_EMAIL: "{{ EDXAPP_DEFAULT_SERVER_EMAIL }}"
+ BULK_EMAIL_DEFAULT_FROM_EMAIL: "{{ EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL }}"
+ CAS_SERVER_URL: "{{ EDXAPP_CAS_SERVER_URL }}"
+ CAS_EXTRA_LOGIN_PARAMS: "{{ EDXAPP_CAS_EXTRA_LOGIN_PARAMS }}"
+ CAS_ATTRIBUTE_CALLBACK: "{{ EDXAPP_CAS_ATTRIBUTE_CALLBACK }}"
HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS: "{{ EDXAPP_MODULESTORE_MAPPINGS }}"
- UNIVERSITY_EMAIL: $EDXAPP_UNIVERSITY_EMAIL
- PRESS_EMAIL: $EDXAPP_PRESS_EMAIL
- PLATFORM_TWITTER_ACCOUNT: $EDXAPP_PLATFORM_TWITTER_ACCOUNT
- PLATFORM_FACEBOOK_ACCOUNT: $EDXAPP_PLATFORM_FACEBOOK_ACCOUNT
- ORA2_FILE_PREFIX: $EDXAPP_ORA2_FILE_PREFIX
- FILE_UPLOAD_STORAGE_BUCKET_NAME: $EDXAPP_FILE_UPLOAD_STORAGE_BUCKET_NAME
- FILE_UPLOAD_STORAGE_PREFIX: $EDXAPP_FILE_UPLOAD_STORAGE_PREFIX
- VIRTUAL_UNIVERSITIES: $EDXAPP_VIRTUAL_UNIVERSITIES
- SUBDOMAIN_BRANDING: $EDXAPP_SUBDOMAIN_BRANDING
-
+ UNIVERSITY_EMAIL: "{{ EDXAPP_UNIVERSITY_EMAIL }}"
+ PRESS_EMAIL: "{{ EDXAPP_PRESS_EMAIL }}"
+ PLATFORM_TWITTER_ACCOUNT: "{{ EDXAPP_PLATFORM_TWITTER_ACCOUNT }}"
+ PLATFORM_FACEBOOK_ACCOUNT: "{{ EDXAPP_PLATFORM_FACEBOOK_ACCOUNT }}"
+ PLATFORM_TWITTER_URL: "{{ EDXAPP_PLATFORM_TWITTER_URL }}"
+ PLATFORM_MEETUP_URL: "{{ EDXAPP_PLATFORM_MEETUP_URL }}"
+ PLATFORM_LINKEDIN_URL: "{{ EDXAPP_PLATFORM_LINKEDIN_URL }}"
+ PLATFORM_GOOGLE_PLUS_URL: "{{ EDXAPP_PLATFORM_GOOGLE_PLUS_URL }}"
+ ORA2_FILE_PREFIX: "{{ EDXAPP_ORA2_FILE_PREFIX }}"
+ FILE_UPLOAD_STORAGE_BUCKET_NAME: "{{ EDXAPP_FILE_UPLOAD_STORAGE_BUCKET_NAME }}"
+ FILE_UPLOAD_STORAGE_PREFIX: "{{ EDXAPP_FILE_UPLOAD_STORAGE_PREFIX }}"
+ VIRTUAL_UNIVERSITIES: "{{ EDXAPP_VIRTUAL_UNIVERSITIES }}"
+ SUBDOMAIN_BRANDING: "{{ EDXAPP_SUBDOMAIN_BRANDING }}"
+ REGISTRATION_EXTRA_FIELDS: "{{ EDXAPP_REGISTRATION_EXTRA_FIELDS }}"
lms_auth_config:
<<: *edxapp_generic_auth
- PEARSON_TEST_PASSWORD: $EDXAPP_PEARSON_TEST_PASSWORD
- SEGMENT_IO_LMS_KEY: $EDXAPP_SEGMENT_IO_LMS_KEY
- OPTIMIZELY_PROJECT_ID: $EDXAPP_OPTIMIZELY_PROJECT_ID
- EDX_API_KEY: $EDXAPP_EDX_API_KEY
- VERIFY_STUDENT: $EDXAPP_VERIFY_STUDENT
- GOOGLE_ANALYTICS_LINKEDIN: $EDXAPP_GOOGLE_ANALYTICS_LINKEDIN
- CC_PROCESSOR_NAME: $EDXAPP_CC_PROCESSOR_NAME
- CC_PROCESSOR: $EDXAPP_CC_PROCESSOR
+ PEARSON_TEST_PASSWORD: "{{ EDXAPP_PEARSON_TEST_PASSWORD }}"
+ SEGMENT_IO_LMS_KEY: "{{ EDXAPP_SEGMENT_IO_LMS_KEY }}"
+ OPTIMIZELY_PROJECT_ID: "{{ EDXAPP_OPTIMIZELY_PROJECT_ID }}"
+ EDX_API_KEY: "{{ EDXAPP_EDX_API_KEY }}"
+ VERIFY_STUDENT: "{{ EDXAPP_VERIFY_STUDENT }}"
+ GOOGLE_ANALYTICS_LINKEDIN: "{{ EDXAPP_GOOGLE_ANALYTICS_LINKEDIN }}"
+ CC_PROCESSOR_NAME: "{{ EDXAPP_CC_PROCESSOR_NAME }}"
+ CC_PROCESSOR: "{{ EDXAPP_CC_PROCESSOR }}"
+ TRACKING_SEGMENTIO_WEBHOOK_SECRET: "{{ EDXAPP_TRACKING_SEGMENTIO_WEBHOOK_SECRET }}"
lms_env_config:
<<: *edxapp_generic_env
PAID_COURSE_REGISTRATION_CURRENCY: $EDXAPP_PAID_COURSE_REGISTRATION_CURRENCY
- SITE_NAME: $EDXAPP_LMS_SITE_NAME
- VIDEO_CDN_URL: $EDXAPP_VIDEO_CDN_URLS
+ SITE_NAME: "{{ EDXAPP_LMS_SITE_NAME }}"
+ VIDEO_CDN_URL: "{{ EDXAPP_VIDEO_CDN_URLS }}"
CODE_JAIL:
# from https://github.com/edx/codejail/blob/master/codejail/django_integration.py#L24, '' should be same as None
python_bin: '{% if EDXAPP_PYTHON_SANDBOX %}{{ edxapp_sandbox_venv_dir }}/bin/python{% endif %}'
- limits: $EDXAPP_CODE_JAIL_LIMITS
+ limits: "{{ EDXAPP_CODE_JAIL_LIMITS }}"
user: '{{ edxapp_sandbox_user }}'
cms_auth_config:
<<: *edxapp_generic_auth
- SEGMENT_IO_KEY: $EDXAPP_SEGMENT_IO_KEY
+ SEGMENT_IO_KEY: "{{ EDXAPP_SEGMENT_IO_KEY }}"
MODULESTORE:
default:
ENGINE: 'xmodule.modulestore.mixed.MixedModuleStore'
OPTIONS:
# See commented section below. LMS-11258
-# mappings: $EDXAPP_XML_MAPPINGS
+# mappings: "{{ EDXAPP_XML_MAPPINGS }}"
mappings: {}
stores:
- *edxapp_generic_draft_modulestore
@@ -613,7 +665,7 @@ cms_auth_config:
- *edxapp_generic_split_modulestore
cms_env_config:
<<: *edxapp_generic_env
- SITE_NAME: $EDXAPP_CMS_SITE_NAME
+ SITE_NAME: "{{ EDXAPP_CMS_SITE_NAME }}"
# install dir for the edx-platform repo
edxapp_code_dir: "{{ edxapp_app_dir }}/edx-platform"
@@ -635,9 +687,6 @@ service_variants_enabled:
- lms
- cms
-edxapp_lms_env: 'lms.envs.aws'
-edxapp_cms_env: 'cms.envs.aws'
-
#Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores
worker_core_mult:
lms: 4
diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml
index f6881ac5a42..77eaf4e5e9d 100644
--- a/playbooks/roles/edxapp/tasks/deploy.yml
+++ b/playbooks/roles/edxapp/tasks/deploy.yml
@@ -27,11 +27,11 @@
when: EDXAPP_USE_GIT_IDENTITY
# Do A Checkout
-- name: checkout edx-platform repo into {{edxapp_code_dir}}
+- name: checkout edx-platform repo into {{ edxapp_code_dir }}
git: >
- dest={{edxapp_code_dir}}
- repo={{edx_platform_repo}}
- version={{edx_platform_version}}
+ dest={{ edxapp_code_dir }}
+ repo={{ edx_platform_repo }}
+ version={{ edx_platform_version }}
accept_hostkey=yes
sudo_user: "{{ edxapp_user }}"
environment:
@@ -42,7 +42,7 @@
- "restart edxapp_workers"
- name: git clean after checking out edx-platform
- shell: cd {{edxapp_code_dir}} && git clean -xdf
+ shell: cd {{ edxapp_code_dir }} && git clean -xdf
sudo_user: "{{ edxapp_user }}"
notify:
- "restart edxapp"
@@ -50,9 +50,9 @@
- name: checkout theme
git: >
- dest={{ edxapp_app_dir }}/themes/{{edxapp_theme_name}}
- repo={{edxapp_theme_source_repo}}
- version={{edxapp_theme_version}}
+ dest={{ edxapp_app_dir }}/themes/{{ edxapp_theme_name }}
+ repo={{ edxapp_theme_source_repo }}
+ version={{ edxapp_theme_version }}
accept_hostkey=yes
when: edxapp_theme_name != ''
sudo_user: "{{ edxapp_user }}"
@@ -91,8 +91,8 @@
- name: gem install bundler
shell: >
gem install bundle
- chdir={{ edxapp_code_dir }}
- executable=/bin/bash
+ chdir={{ edxapp_code_dir }}
+ executable=/bin/bash
environment: "{{ edxapp_environment }}"
sudo_user: "{{ edxapp_user }}"
notify:
@@ -102,8 +102,8 @@
- name: bundle install
shell: >
bundle install --binstubs
- chdir={{ edxapp_code_dir }}
- executable=/bin/bash
+ chdir={{ edxapp_code_dir }}
+ executable=/bin/bash
sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}"
notify:
@@ -144,8 +144,8 @@
# Install the python pre requirements into {{ edxapp_venv_dir }}
- name : install python pre-requirements
pip: >
- requirements="{{pre_requirements_file}}"
- virtualenv="{{edxapp_venv_dir}}"
+ requirements="{{ pre_requirements_file }}"
+ virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ edxapp_user }}"
@@ -173,8 +173,8 @@
# Install the python post requirements into {{ edxapp_venv_dir }}
- name : install python post-requirements
pip: >
- requirements="{{post_requirements_file}}"
- virtualenv="{{edxapp_venv_dir}}"
+ requirements="{{ post_requirements_file }}"
+ virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ edxapp_user }}"
@@ -187,8 +187,8 @@
# Install the python paver requirements into {{ edxapp_venv_dir }}
- name : install python paver-requirements
pip: >
- requirements="{{paver_requirements_file}}"
- virtualenv="{{edxapp_venv_dir}}"
+ requirements="{{ paver_requirements_file }}"
+ virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ edxapp_user }}"
@@ -257,7 +257,7 @@
- name: install CAS attribute module
pip: >
name="{{ EDXAPP_CAS_ATTRIBUTE_PACKAGE }}"
- virtualenv="{{edxapp_venv_dir}}"
+ virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w --use-mirrors"
sudo_user: "{{ edxapp_user }}"
@@ -294,8 +294,8 @@
- name: code sandbox | Install base sandbox requirements and create sandbox virtualenv
pip: >
- requirements="{{sandbox_base_requirements}}"
- virtualenv="{{edxapp_sandbox_venv_dir}}"
+ requirements="{{ sandbox_base_requirements }}"
+ virtualenv="{{ edxapp_sandbox_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w --use-mirrors"
sudo_user: "{{ edxapp_sandbox_user }}"
diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml
index 53530096e67..8f0b1b7442e 100644
--- a/playbooks/roles/edxapp/tasks/main.yml
+++ b/playbooks/roles/edxapp/tasks/main.yml
@@ -26,6 +26,7 @@
- "{{ edxapp_theme_dir }}"
- "{{ edxapp_staticfile_dir }}"
- "{{ edxapp_course_static_dir }}"
+ - "{{ edxapp_course_data_dir }}"
# This is a symlink that has to exist because
# we currently can't override the DATA_DIR var
@@ -38,7 +39,6 @@
state=link
owner="{{ edxapp_user }}"
group="{{ common_web_group }}"
-
- name: create edxapp log dir
file: >
@@ -70,6 +70,12 @@
- "restart edxapp"
- "restart edxapp_workers"
+- name: set up edxapp .npmrc
+ template:
+ src=.npmrc.j2 dest={{ edxapp_app_dir }}/.npmrc
+ owner={{ edxapp_user }} group={{ common_web_group }}
+ mode=0600
+
- name: create log directories for service variants
notify:
- "restart edxapp"
@@ -81,7 +87,7 @@
with_items: service_variants_enabled
# Set up the python sandbox execution environment
-- include: python_sandbox_env.yml
+- include: python_sandbox_env.yml tags=deploy
when: EDXAPP_PYTHON_SANDBOX
- include: deploy.yml tags=deploy
diff --git a/playbooks/roles/edxapp/tasks/python_sandbox_env.yml b/playbooks/roles/edxapp/tasks/python_sandbox_env.yml
index 9d144ad709f..e582e579bd5 100644
--- a/playbooks/roles/edxapp/tasks/python_sandbox_env.yml
+++ b/playbooks/roles/edxapp/tasks/python_sandbox_env.yml
@@ -1,3 +1,13 @@
+# Set the alternatives this way for blas and lapack to work correctly for the
+# MITx 6.341x course.
+# TODO: Switch to using alternatives module in 1.6
+- name: code sandbox | Use libblas for 3gf
+ command: update-alternatives --set libblas.so.3gf /usr/lib/libblas/libblas.so.3gf
+
+# TODO: Switch to using alternatives module in 1.6
+- name: code sandbox | Use liblapac for 3gf
+ command: update-alternatives --set liblapack.so.3gf /usr/lib/lapack/liblapack.so.3gf
+
- name: code sandbox | Create edxapp sandbox user
user: name={{ edxapp_sandbox_user }} shell=/bin/false home={{ edxapp_sandbox_venv_dir }}
notify:
diff --git a/playbooks/roles/edxapp/tasks/xml.yml b/playbooks/roles/edxapp/tasks/xml.yml
index 988bf867798..bb15266ee4f 100644
--- a/playbooks/roles/edxapp/tasks/xml.yml
+++ b/playbooks/roles/edxapp/tasks/xml.yml
@@ -1,3 +1,11 @@
+- name: make the course data updatable by the edxapp user
+ file:
+ path="{{ edxapp_course_data_dir }}"
+ state=directory
+ recurse=yes
+ owner="{{ edxapp_user }}"
+ group="{{ edxapp_user }}"
+
- name: clone the xml course repo
git: >
repo="{{ item.repo_url }}"
diff --git a/playbooks/roles/edxapp/templates/.npmrc.j2 b/playbooks/roles/edxapp/templates/.npmrc.j2
new file mode 100644
index 00000000000..3a82b3419dd
--- /dev/null
+++ b/playbooks/roles/edxapp/templates/.npmrc.j2
@@ -0,0 +1 @@
+registry={{ COMMON_NPM_MIRROR_URL }}
diff --git a/playbooks/roles/edxapp/templates/95-sandbox-sudoer.j2 b/playbooks/roles/edxapp/templates/95-sandbox-sudoer.j2
index 71f624335a0..b291add83a0 100644
--- a/playbooks/roles/edxapp/templates/95-sandbox-sudoer.j2
+++ b/playbooks/roles/edxapp/templates/95-sandbox-sudoer.j2
@@ -1,11 +1,11 @@
{% if devstack %}
{{ edxapp_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:{{ edxapp_sandbox_venv_dir }}/bin/python
-{{ edxapp_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:/bin/rm /tmp/codejail-*/tmp
+{{ edxapp_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:/usr/bin/find /tmp/codejail-*/tmp -mindepth 1 -maxdepth 1 -exec rm -rf {} ;
{{ edxapp_user }} ALL=(ALL) NOPASSWD:/bin/kill
{{ edxapp_user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill
{% else %}
{{ common_web_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:{{ edxapp_sandbox_venv_dir }}/bin/python
-{{ common_web_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:/bin/rm /tmp/codejail-*/tmp
+{{ common_web_user }} ALL=({{ edxapp_sandbox_user }}) SETENV:NOPASSWD:/usr/bin/find /tmp/codejail-*/tmp -mindepth 1 -maxdepth 1 -exec rm -rf {} ;
{{ common_web_user }} ALL=(ALL) NOPASSWD:/bin/kill
{{ common_web_user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill
{% endif %}
diff --git a/playbooks/roles/edxapp/templates/cms.auth.json.j2 b/playbooks/roles/edxapp/templates/cms.auth.json.j2
index 07bb9f00e09..b6cb39bb5cd 100644
--- a/playbooks/roles/edxapp/templates/cms.auth.json.j2
+++ b/playbooks/roles/edxapp/templates/cms.auth.json.j2
@@ -1,4 +1,4 @@
-{% do cms_auth_config.update(EDXAPP_AUTH_EXTRA) %}
+{% do cms_auth_config.update(EDXAPP_CMS_AUTH_EXTRA) %}
{% for key, value in cms_auth_config.iteritems() %}
{% if value == 'None' %}
{% do cms_auth_config.update({key: None }) %}
diff --git a/playbooks/roles/edxapp/templates/cms.conf.j2 b/playbooks/roles/edxapp/templates/cms.conf.j2
index b9657559622..cb888fb8c17 100644
--- a/playbooks/roles/edxapp/templates/cms.conf.j2
+++ b/playbooks/roles/edxapp/templates/cms.conf.j2
@@ -16,15 +16,15 @@ command={{ executable }} {{ max_req }} --preload -b {{ edxapp_cms_gunicorn_host
{% else -%}
{# This is for backwards compatibility, set workers explicitely using EDXAPP_WORKERS #}
{% if ansible_processor|length > 0 -%}
-command={{ executable }} {{ max_req }} --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
+command={{ executable }} {{ max_req }} --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} {{ EDXAPP_CMS_GUNICORN_EXTRA }} cms.wsgi
{% else -%}
-command={{ executable }} {{ max_req }} --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
+command={{ executable }} {{ max_req }} --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} {{ EDXAPP_CMS_GUNICORN_EXTRA }} cms.wsgi
{% endif -%}
{% endif -%}
user={{ common_web_user }}
directory={{ edxapp_code_dir }}
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_CMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PORT={{edxapp_cms_gunicorn_port}},ADDRESS={{edxapp_cms_gunicorn_host}},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ edxapp_cms_env }},SERVICE_VARIANT="cms"
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_CMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PORT={{ edxapp_cms_gunicorn_port }},ADDRESS={{ edxapp_cms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_CMS_ENV }},SERVICE_VARIANT="cms"
stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
killasgroup=true
diff --git a/playbooks/roles/edxapp/templates/code.sandbox.j2 b/playbooks/roles/edxapp/templates/code.sandbox.j2
index 354d8726b6b..6074886121a 100644
--- a/playbooks/roles/edxapp/templates/code.sandbox.j2
+++ b/playbooks/roles/edxapp/templates/code.sandbox.j2
@@ -9,7 +9,7 @@
/tmp/codejail-*/** wrix,
#
- # Whitelist particiclar shared objects from the system
+ # Whitelist particular shared objects from the system
# python installation
#
/usr/lib/python2.7/lib-dynload/_json.so mr,
@@ -21,6 +21,22 @@
/usr/lib/python2.7/lib-dynload/_elementtree.so mr,
/usr/lib/python2.7/lib-dynload/pyexpat.so mr,
+ # Matplot lib needs a place for temp caches
+ {{ edxapp_sandbox_venv_dir }}/.config/ wrix,
+ {{ edxapp_sandbox_venv_dir }}/.cache/ wrix,
+ {{ edxapp_sandbox_venv_dir }}/.config/** wrix,
+ {{ edxapp_sandbox_venv_dir }}/.cache/** wrix,
+
+ # Matplotlib related libraries
+ /usr/lib/python2.7/lib-dynload/termios.so mr,
+ /usr/lib/python2.7/lib-dynload/parser.so mr,
+
+ # Matplot lib needs fonts to make graphs
+ /usr/share/fonts/ r,
+ /usr/share/fonts/** r,
+ /usr/local/share/fonts/ r,
+ /usr/local/share/fonts/** r,
+
#
# Allow access to selections from /proc
#
diff --git a/playbooks/roles/edxapp/templates/lms.auth.json.j2 b/playbooks/roles/edxapp/templates/lms.auth.json.j2
index 94b4946f055..4538e7ff4a3 100644
--- a/playbooks/roles/edxapp/templates/lms.auth.json.j2
+++ b/playbooks/roles/edxapp/templates/lms.auth.json.j2
@@ -1,4 +1,4 @@
-{% do lms_auth_config.update(EDXAPP_AUTH_EXTRA) %}
+{% do lms_auth_config.update(EDXAPP_LMS_AUTH_EXTRA) %}
{% for key, value in lms_auth_config.iteritems() %}
{% if value == 'None' %}
{% do lms_auth_config.update({key: None }) %}
diff --git a/playbooks/roles/edxapp/templates/lms.conf.j2 b/playbooks/roles/edxapp/templates/lms.conf.j2
index fe563509a4b..554bac72b2c 100644
--- a/playbooks/roles/edxapp/templates/lms.conf.j2
+++ b/playbooks/roles/edxapp/templates/lms.conf.j2
@@ -17,15 +17,15 @@ command={{ executable }} {{ max_req }} --preload -b {{ edxapp_lms_gunicorn_host
{% else -%}
{# This is for backwards compatibility, set workers explicitely using EDXAPP_WORKERS #}
{% if ansible_processor|length > 0 -%}
-command={{ executable }} {{ max_req }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
+command={{ executable }} {{ max_req }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} {{ EDXAPP_LMS_GUNICORN_EXTRA }} lms.wsgi
{% else -%}
-command={{ executable }} {{ max_req }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
+command={{ executable }} {{ max_req }} --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} {{ EDXAPP_LMS_GUNICORN_EXTRA }} lms.wsgi
{% endif %}
{% endif %}
user={{ common_web_user }}
directory={{ edxapp_code_dir }}
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_LMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%} PORT={{edxapp_lms_gunicorn_port}},ADDRESS={{edxapp_lms_gunicorn_host}},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }},SERVICE_VARIANT="lms",PATH="{{ edxapp_deploy_path }}"
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_LMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%} PORT={{ edxapp_lms_gunicorn_port }},ADDRESS={{ edxapp_lms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_LMS_ENV }},SERVICE_VARIANT="lms",PATH="{{ edxapp_deploy_path }}"
stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
killasgroup=true
diff --git a/playbooks/roles/edxlocal/tasks/main.yml b/playbooks/roles/edxlocal/tasks/main.yml
index a6e9d183c9f..a83c5e09d5e 100644
--- a/playbooks/roles/edxlocal/tasks/main.yml
+++ b/playbooks/roles/edxlocal/tasks/main.yml
@@ -17,7 +17,7 @@
mysql_user: >
name={{ EDXAPP_MYSQL_USER }}
password={{ EDXAPP_MYSQL_PASSWORD }}
- priv='{{EDXAPP_MYSQL_DB_NAME}}.*:ALL'
+ priv='{{ EDXAPP_MYSQL_DB_NAME }}.*:ALL'
when: EDXAPP_MYSQL_USER is defined
- name: create a database for edxapp
@@ -31,7 +31,7 @@
mysql_user: >
name={{ XQUEUE_MYSQL_USER }}
password={{ XQUEUE_MYSQL_PASSWORD }}
- priv='{{XQUEUE_MYSQL_DB_NAME}}.*:ALL'
+ priv='{{ XQUEUE_MYSQL_DB_NAME }}.*:ALL'
when: XQUEUE_MYSQL_USER is defined
- name: create a database for xqueue
@@ -45,7 +45,7 @@
mysql_user: >
name={{ ORA_MYSQL_USER }}
password={{ ORA_MYSQL_PASSWORD }}
- priv='{{ORA_MYSQL_DB_NAME}}.*:ALL'
+ priv='{{ ORA_MYSQL_DB_NAME }}.*:ALL'
when: ORA_MYSQL_USER is defined
- name: create a database for ora
diff --git a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2 b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2
index d129ffca73c..39aadaddec2 100644
--- a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2
+++ b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2
@@ -3,11 +3,11 @@
# Path to directory where to store index data allocated for this node.
#
-path.data: {{elasticsearch_data_dir}}
+path.data: {{ elasticsearch_data_dir }}
# Path to log files:
#
-path.logs: {{elasticsearch_log_dir}}
+path.logs: {{ elasticsearch_log_dir }}
# ElasticSearch performs poorly when JVM starts swapping: you should ensure that
# it _never_ swaps.
@@ -43,3 +43,8 @@ script.disable_dynamic: true
discovery.zen.ping.unicast.hosts: ['{{hosts|join("\',\'") }}']
{% endif -%}
+
+{% if vagrant_cluster|bool %}
+network:
+ host: {{ ansible_ssh_host }}
+{% endif %}
diff --git a/playbooks/roles/flower/defaults/main.yml b/playbooks/roles/flower/defaults/main.yml
index 62ebb32b128..d757638826d 100644
--- a/playbooks/roles/flower/defaults/main.yml
+++ b/playbooks/roles/flower/defaults/main.yml
@@ -24,4 +24,4 @@ flower_deploy_path: "{{ flower_venv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/b
flower_broker: "amqp://{{ FLOWER_BROKER_USERNAME }}:{{ FLOWER_BROKER_PASSWORD }}@{{ FLOWER_BROKER_HOST }}:{{ FLOWER_BROKER_PORT }}"
flower_environment:
- PATH: $flower_deploy_path
+ PATH: "{{ flower_deploy_path }}"
diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml
index e977526cd54..51d14900b50 100644
--- a/playbooks/roles/forum/defaults/main.yml
+++ b/playbooks/roles/forum/defaults/main.yml
@@ -18,7 +18,7 @@ FORUM_MONGO_HOSTS:
FORUM_MONGO_TAGS: !!null
FORUM_MONGO_PORT: "27017"
FORUM_MONGO_DATABASE: "cs_comments_service"
-FORUM_MONGO_URL: "mongodb://{{ FORUM_MONGO_USER }}:{{ FORUM_MONGO_PASSWORD }}@{%- for host in FORUM_MONGO_HOSTS -%}{{host}}:{{ FORUM_MONGO_PORT }}{%- if not loop.last -%},{%- endif -%}{%- endfor -%}/{{ FORUM_MONGO_DATABASE }}{%- if FORUM_MONGO_TAGS -%}?tags={{ FORUM_MONGO_TAGS }}{%- endif -%}"
+FORUM_MONGO_URL: "mongodb://{{ FORUM_MONGO_USER }}:{{ FORUM_MONGO_PASSWORD }}@{%- for host in FORUM_MONGO_HOSTS -%}{{ host }}:{{ FORUM_MONGO_PORT }}{%- if not loop.last -%},{%- endif -%}{%- endfor -%}/{{ FORUM_MONGO_DATABASE }}{%- if FORUM_MONGO_TAGS -%}?tags={{ FORUM_MONGO_TAGS }}{%- endif -%}"
FORUM_SINATRA_ENV: "development"
FORUM_RACK_ENV: "development"
FORUM_NGINX_PORT: "18080"
@@ -28,8 +28,8 @@ FORUM_ELASTICSEARCH_PORT: "9200"
FORUM_ELASTICSEARCH_URL: "http://{{ FORUM_ELASTICSEARCH_HOST }}:{{ FORUM_ELASTICSEARCH_PORT }}"
# This needs to be a string, set to 'false' to disable
-FORUM_NEW_RELIC_ENABLE: 'true'
-FORUM_NEW_RELIC_LICENSE_KEY: "new-relic-license-key"
+FORUM_NEW_RELIC_ENABLE: '{{ COMMON_ENABLE_NEWRELIC }}'
+FORUM_NEW_RELIC_LICENSE_KEY: '{{ NEWRELIC_LICENSE_KEY | default("") }}'
FORUM_NEW_RELIC_APP_NAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-forum"
FORUM_WORKER_PROCESSES: "4"
@@ -72,4 +72,3 @@ forum_version: "master"
#
forum_services:
- {service: "elasticsearch", host: "{{ FORUM_ELASTICSEARCH_HOST }}", port: "{{ FORUM_ELASTICSEARCH_PORT }}"}
-
diff --git a/playbooks/roles/forum/templates/cs_comments_service.conf.j2 b/playbooks/roles/forum/templates/cs_comments_service.conf.j2
index 9022fdc623e..523261bf436 100644
--- a/playbooks/roles/forum/templates/cs_comments_service.conf.j2
+++ b/playbooks/roles/forum/templates/cs_comments_service.conf.j2
@@ -10,7 +10,7 @@ env PID=/var/tmp/comments_service.pid
chdir {{ forum_code_dir }}
script
- . {{forum_app_dir}}/forum_env
- {{forum_app_dir}}/.rbenv/shims/ruby app.rb
+ . {{ forum_app_dir }}/forum_env
+ {{ forum_app_dir }}/.rbenv/shims/ruby app.rb
end script
diff --git a/playbooks/roles/haproxy/defaults/main.yml b/playbooks/roles/haproxy/defaults/main.yml
index b42adf89015..243c3f7e83d 100644
--- a/playbooks/roles/haproxy/defaults/main.yml
+++ b/playbooks/roles/haproxy/defaults/main.yml
@@ -49,11 +49,30 @@ haproxy_default_config: |
# desired applications
haproxy_applications:
- |
- listen rabbitmq 127.0.0.1:5672
+ listen rabbitmq 127.0.0.1:35672
mode tcp
balance roundrobin
option tcplog
option tcpka
- server rabbit01 172.23.128.10:5672 check inter 5000 rise 2 fall 3
- server rabbit02 172.23.129.10:5672 backup check inter 5000 rise 2 fall 3
- server rabbit03 172.23.130.10:5672 backup check inter 5000 rise 2 fall 3
+ server rabbit01 192.168.33.100:5672 check inter 5000 rise 2 fall 3
+ server rabbit02 192.168.33.110:5672 check inter 5000 rise 2 fall 3
+ server rabbit03 192.168.33.120:5672 check inter 5000 rise 2 fall 3
+
+ listen mariadb 127.0.0.1:13306
+ mode tcp
+ balance roundrobin
+ option tcplog
+ option tcpka
+ option mysql-check user haproxy
+ server galera1 192.168.33.100:3306 check weight 1
+ server galera2 192.168.33.110:3306 check weight 1
+ server galera3 192.168.33.120:3306 check weight 1
+
+ listen elasticsearch 127.0.0.1:19200
+ mode tcp
+ balance roundrobin
+ option tcplog
+ option tcpka
+ server galera1 192.168.33.100:9200 check weight 1
+ server galera2 192.168.33.110:9200 check weight 1
+ server galera3 192.168.33.120:9200 check weight 1
diff --git a/playbooks/roles/haproxy/meta/main.yml b/playbooks/roles/haproxy/meta/main.yml
index 0fbe1d40ef9..ae1075a8c1d 100644
--- a/playbooks/roles/haproxy/meta/main.yml
+++ b/playbooks/roles/haproxy/meta/main.yml
@@ -18,3 +18,6 @@
# my_role_var0: "foo"
# my_role_var1: "bar"
# }
+
+dependencies:
+ - common
diff --git a/playbooks/roles/haproxy/templates/haproxy.cfg.j2 b/playbooks/roles/haproxy/templates/haproxy.cfg.j2
index c2bd1c6fe2b..3ca147d2b3a 100644
--- a/playbooks/roles/haproxy/templates/haproxy.cfg.j2
+++ b/playbooks/roles/haproxy/templates/haproxy.cfg.j2
@@ -1,8 +1,8 @@
# this config needs haproxy-1.1.28 or haproxy-1.2.1
global
- log 127.0.0.1 local0
- log 127.0.0.1 local1 notice
+ log /dev/log local0 info
+ log /dev/log local0 notice
#log loghost local0 info
maxconn 4096
#chroot /usr/share/haproxy
diff --git a/playbooks/roles/insights/defaults/main.yml b/playbooks/roles/insights/defaults/main.yml
new file mode 100644
index 00000000000..2521f2c86a0
--- /dev/null
+++ b/playbooks/roles/insights/defaults/main.yml
@@ -0,0 +1,154 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Defaults for role insights
+#
+
+INSIGHTS_GIT_IDENTITY: !!null
+
+INSIGHTS_MEMCACHE: [ 'localhost:11211' ]
+INSIGHTS_FEEDBACK_EMAIL: 'dashboard@example.com'
+INSIGHTS_MKTG_BASE: 'http://example.com'
+INSIGHTS_PRIVACY_POLICY_URL: '{{ INSIGHTS_MKTG_BASE }}/privacy-policy'
+INSIGHTS_TERMS_OF_SERVICE_URL: '{{ INSIGHTS_MKTG_BASE }}/terms-service'
+INSIGHTS_SUPPORT_URL: ''
+INSIGHTS_OAUTH2_SECRET: 'secret'
+INSIGHTS_OATH2_KEY: 'key'
+INSIGHTS_OAUTH2_URL_ROOT: 'url_root'
+INSIGHTS_SECRET_KEY: 'YOUR_SECRET_KEY_HERE'
+INSIGHTS_OAUTH2_KEY: 'YOUR_OAUTH2_KEY'
+# This will not work on single instance sandboxes
+INSIGHTS_DOC_BASE: 'http://localhost/en/latest'
+INSIGHTS_LMS_BASE: 'http://localhost:18000'
+ANALYTICS_API_ENDPOINT: 'http://localhost:18010'
+INSIGHTS_DATA_API_AUTH_TOKEN: 'YOUR_DATA_API_AUTH_TOKEN'
+INSIGHTS_PLATFORM_NAME: 'edX'
+INSIGHTS_APPLICATION_NAME: 'Insights'
+INSIGHTS_SEGMENT_IO_KEY: 'YOUR_KEY'
+# should match the timezone of your map reduce pipeline
+INSIGHTS_TIME_ZONE: 'UTC'
+INSIGHTS_LANGUAGE_CODE: 'en-us'
+# email config
+INSIGHTS_EMAIL_HOST: 'smtp.example.com'
+INSIGHTS_EMAIL_HOST_PASSWORD: "mail_password"
+INSIGHTS_EMAIL_HOST_USER: "mail_user"
+INSIGHTS_EMAIL_PORT: 587
+INSIGHTS_ENABLE_AUTO_AUTH: false
+
+INSIGHTS_DATABASES:
+ # rw user
+ default:
+ ENGINE: 'django.db.backends.mysql'
+ NAME: 'dashboard'
+ USER: 'rosencrantz'
+ PASSWORD: 'secret'
+ HOST: 'localhost'
+ PORT: '3306'
+
+#
+# This block of config is dropped into /edx/etc/insights.yml
+# and is read in by analytics_dashboard/settings/production.py
+INSIGHTS_CONFIG:
+ SUPPORT_URL: '{{ INSIGHTS_SUPPORT_URL }}'
+ DOCUMENTATION_LOAD_ERROR_URL: '{{ INSIGHTS_DOC_BASE }}/Reference.html#error-conditions'
+ SEGMENT_IO_KEY: '{{ INSIGHTS_SEGMENT_IO_KEY }}'
+ FEEDBACK_EMAIL: '{{ INSIGHTS_FEEDBACK_EMAIL }}'
+ PRIVACY_POLICY_URL: '{{ INSIGHTS_PRIVACY_POLICY_URL }}'
+ TERMS_OF_SERVICE_URL: '{{ INSIGHTS_TERMS_OF_SERVICE_URL }}'
+ HELP_URL: '{{ INSIGHTS_DOC_BASE }}'
+ SECRET_KEY: '{{ INSIGHTS_SECRET_KEY }}'
+ DATA_API_URL: '{{ ANALYTICS_API_ENDPOINT }}'
+ DATA_API_AUTH_TOKEN: '{{ INSIGHTS_DATA_API_AUTH_TOKEN }}'
+ SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
+ SOCIAL_AUTH_EDX_OIDC_KEY: '{{ INSIGHTS_OAUTH2_KEY }}'
+ SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ INSIGHTS_OAUTH2_SECRET }}'
+ SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ INSIGHTS_OAUTH2_URL_ROOT }}'
+ # This value should be the same as SOCIAL_AUTH_EDX_OIDC_SECRET
+ SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ INSIGHTS_OAUTH2_SECRET }}'
+ ENABLE_AUTO_AUTH: $INSIGHTS_ENABLE_AUTO_AUTH
+ PLATFORM_NAME: '{{ INSIGHTS_PLATFORM_NAME }}'
+ APPLICATION_NAME: '{{ INSIGHTS_APPLICATION_NAME }}'
+ CACHES:
+ default: &default_generic_cache
+ BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
+ KEY_PREFIX: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-insights'
+ LOCATION: "{{ INSIGHTS_MEMCACHE }}"
+ TIME_ZONE: '{{ INSIGHTS_TIME_ZONE }}'
+ LANGUAGE_CODE: '{{ INSIGHTS_LANGUAGE_CODE }}'
+ # email config
+ EMAIL_HOST: '{{ INSIGHTS_EMAIL_HOST }}'
+ EMAIL_HOST_PASSWORD: '{{ INSIGHTS_EMAIL_HOST_PASSWORD }}'
+ EMAIL_HOST_USER: '{{ INSIGHTS_EMAIL_HOST_USER }}'
+ EMAIL_PORT: $INSIGHTS_EMAIL_PORT
+ # static file config
+ STATICFILES_DIRS: ["{{ insights_python_path }}/static"]
+ STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ insights_service_name }}/staticfiles"
+ # db config
+ DATABASE_OPTIONS:
+ connect_timeout: 10
+ DATABASES: "{{ INSIGHTS_DATABASES }}"
+
+
+INSIGHTS_VERSION: "master"
+INSIGHTS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-analytics-api"
+INSIGHTS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+INSIGHTS_NGINX_PORT: "18110"
+INSIGHTS_GUNICORN_WORKERS: "2"
+INSIGHTS_GUNICORN_EXTRA: ""
+#
+# vars are namespace with the module name.
+#
+insights_environment:
+ DJANGO_SETTINGS_MODULE: "analytics_dashboard.settings.production"
+ ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR }}/{{ insights_service_name }}.yaml"
+
+insights_role_name: "insights"
+insights_service_name: "{{ insights_role_name }}"
+insights_user: "{{ insights_role_name }}"
+insights_app_dir: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
+insights_home: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
+insights_venv_base: "{{ insights_home }}/venvs"
+insights_venv_dir: "{{ insights_venv_base }}/{{ insights_service_name }}"
+insights_venv_bin: "{{ insights_venv_dir }}/bin"
+insights_code_dir: "{{ insights_app_dir }}/edx-analytics-dashboard"
+insights_python_path: "{{ insights_code_dir }}/analytics_dashboard"
+insights_conf_dir: "{{ insights_home }}"
+insights_log_dir: "{{ COMMON_LOG_DIR }}/{{ insights_service_name }}"
+
+insights_nodeenv_dir: "{{ insights_home }}/nodeenvs/{{ insights_service_name }}"
+insights_nodeenv_bin: "{{ insights_nodeenv_dir }}/bin"
+insights_node_modules_dir: "{{ insights_code_dir }}/node_modules"
+insights_node_bin: "{{ insights_node_modules_dir }}/.bin"
+
+insights_gunicorn_host: "127.0.0.1"
+insights_gunicorn_port: "8110"
+insights_gunicorn_timeout: "300"
+insights_wsgi: "analytics_dashboard.wsgi:application"
+
+insights_django_settings: "analytics_dashboard.settings.production"
+insights_source_repo: "git@{{ COMMON_GIT_MIRROR }}:/edx/edx-analytics-dashboard"
+insights_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ insights_git_identity_file }}"
+insights_git_identity_file: "{{ insights_home }}/git-identity"
+insights_manage: "{{ insights_code_dir }}/analytics_dashboard/manage.py"
+
+insights_requirements_base: "{{ insights_code_dir }}/requirements"
+insights_requirements:
+ - production.txt
+ - optional.txt
+
+#
+# OS packages
+#
+insights_debian_pkgs:
+ - 'libmysqlclient-dev'
+ - 'build-essential'
+
+insights_redhat_pkgs:
+ - 'community-mysql-devel'
diff --git a/playbooks/roles/insights/handlers/main.yml b/playbooks/roles/insights/handlers/main.yml
new file mode 100644
index 00000000000..40f47ebe17c
--- /dev/null
+++ b/playbooks/roles/insights/handlers/main.yml
@@ -0,0 +1,24 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role insights
+#
+# Overview:
+#
+#
+
+- name: "restart insights"
+ supervisorctl_local: >
+ name={{ insights_service_name }}
+ supervisorctl_path={{ supervisor_ctl }}
+ config={{ supervisor_cfg }}
+ state=restarted
+ when: not disable_edx_services
diff --git a/playbooks/roles/insights/meta/main.yml b/playbooks/roles/insights/meta/main.yml
new file mode 100644
index 00000000000..db98ba999c0
--- /dev/null
+++ b/playbooks/roles/insights/meta/main.yml
@@ -0,0 +1,17 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role insights
+#
+dependencies:
+ - role: edx_service
+ edx_role_name: "{{ insights_role_name }}"
+ edx_service_name: "{{ insights_service_name }}"
+ - supervisor
diff --git a/playbooks/roles/insights/tasks/deploy.yml b/playbooks/roles/insights/tasks/deploy.yml
new file mode 100644
index 00000000000..d5d5a5d3ff8
--- /dev/null
+++ b/playbooks/roles/insights/tasks/deploy.yml
@@ -0,0 +1,130 @@
+---
+
+- name: install read-only ssh key
+ copy: >
+ content="{{ INSIGHTS_GIT_IDENTITY }}" dest={{ insights_git_identity_file }}
+ owner={{ insights_user }} group={{ insights_user }} mode=0600
+
+- name: setup the insights env file
+ template: >
+ src="edx/app/insights/insights_env.j2"
+ dest="{{ insights_app_dir }}/insights_env"
+ owner={{ insights_user }}
+ group={{ insights_user }}
+ mode=0644
+
+- name: checkout code
+ git: >
+ dest={{ insights_code_dir }} repo={{ insights_source_repo }} version={{ INSIGHTS_VERSION }}
+ accept_hostkey=yes
+ ssh_opts="{{ insights_git_ssh_opts }}"
+ register: insights_code_checkout
+ notify: restart insights
+ sudo_user: "{{ insights_user }}"
+
+- name: write out app config file
+ template: >
+ src=edx/app/insights/insights.yaml.j2
+ dest={{ COMMON_CFG_DIR }}/{{ insights_service_name }}.yaml
+ mode=0644 owner={{ insights_user }} group={{ insights_user }}
+ notify: restart insights
+
+- name: install application requirements
+ pip: >
+ requirements="{{ insights_requirements_base }}/{{ item }}"
+ virtualenv="{{ insights_venv_dir }}" state=present
+ extra_args="--exists-action w"
+ sudo_user: "{{ insights_user }}"
+ notify: restart insights
+ with_items: insights_requirements
+
+- name: create nodeenv
+ shell: >
+ creates={{ insights_nodeenv_dir }}
+ {{ insights_venv_bin }}/nodeenv {{ insights_nodeenv_dir }}
+ sudo_user: "{{ insights_user }}"
+
+- name: install node dependencies
+ npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
+ sudo_user: "{{ insights_user }}"
+
+- name: install bower dependencies
+ shell: >
+ chdir={{ insights_code_dir }}
+ . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
+ sudo_user: "{{ insights_user }}"
+
+- name: syncdb and migrate
+ shell: >
+ chdir={{ insights_code_dir }}
+ DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+ DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+ {{ insights_venv_bin }}/python {{ insights_manage }} syncdb --migrate --noinput
+ sudo_user: "{{ insights_user }}"
+ environment: "{{ insights_environment }}"
+ when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run r.js optimizer
+ shell: >
+ chdir={{ insights_code_dir }}
+ . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
+ sudo_user: "{{ insights_user }}"
+
+- name: run collectstatic
+ shell: >
+ chdir={{ insights_code_dir }}
+ {{ insights_venv_bin }}/python {{ insights_manage }} {{ item }}
+ sudo_user: "{{ insights_user }}"
+ environment: "{{ insights_environment }}"
+ with_items:
+ - "collectstatic --noinput"
+ - "compress"
+
+- name: write out the supervisior wrapper
+ template: >
+ src=edx/app/insights/insights.sh.j2
+ dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
+ mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+ notify: restart insights
+
+- name: write supervisord config
+ template: >
+ src=edx/app/supervisor/conf.d.available/insights.conf.j2
+ dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
+ owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+ notify: restart insights
+
+- name: enable supervisor script
+ file: >
+ src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
+ dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
+ state=link
+ force=yes
+ notify: restart insights
+ when: not disable_edx_services
+
+- name: update supervisor configuration
+ shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+ when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+ file: >
+ src="{{ insights_venv_bin }}/{{ item }}"
+ dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_role_name }}"
+ state=link
+ with_items:
+ - python
+ - pip
+ - django-admin.py
+
+- name: create manage.py symlink
+ file: >
+ src="{{ insights_manage }}"
+ dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_role_name }}"
+ state=link
+
+- name: remove read-only ssh key for the content repo
+ file: path={{ insights_git_identity_file }} state=absent
+
+- include: tag_ec2.yml tags=deploy
+ when: COMMON_TAG_EC2_INSTANCE
diff --git a/playbooks/roles/insights/tasks/main.yml b/playbooks/roles/insights/tasks/main.yml
new file mode 100644
index 00000000000..b9bf71a0e46
--- /dev/null
+++ b/playbooks/roles/insights/tasks/main.yml
@@ -0,0 +1,27 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role insights
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- fail: msg="You must provide a private key for the Insights repo"
+ when: not INSIGHTS_GIT_IDENTITY
+
+- include: deploy.yml tags=deploy
diff --git a/playbooks/roles/insights/tasks/tag_ec2.yml b/playbooks/roles/insights/tasks/tag_ec2.yml
new file mode 100644
index 00000000000..df84fd5ccbf
--- /dev/null
+++ b/playbooks/roles/insights/tasks/tag_ec2.yml
@@ -0,0 +1,11 @@
+---
+
+- name: get instance information
+ action: ec2_facts
+
+- name: tag instance
+ ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+ args:
+ tags:
+ "version:insights" : "{{ insights_source_repo }} {{ insights_code_checkout.after |truncate(7,True,'')}}"
+ when: insights_code_checkout.after is defined
diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2
new file mode 100644
index 00000000000..01fd0619343
--- /dev/null
+++ b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = insights_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ INSIGHTS_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+source {{ insights_app_dir }}/insights_env
+
+{{ executable }} --pythonpath={{ insights_python_path }} -b {{ insights_gunicorn_host }}:{{ insights_gunicorn_port }} -w {{ INSIGHTS_GUNICORN_WORKERS }} --timeout={{ insights_gunicorn_timeout }} {{ INSIGHTS_GUNICORN_EXTRA }} {{ insights_wsgi }}
diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2
new file mode 100644
index 00000000000..7532cbc29b1
--- /dev/null
+++ b/playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2
@@ -0,0 +1,4 @@
+---
+# {{ ansible_managed }}
+
+{{ INSIGHTS_CONFIG | to_nice_yaml }}
diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights_env.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights_env.j2
new file mode 100644
index 00000000000..896a3693149
--- /dev/null
+++ b/playbooks/roles/insights/templates/edx/app/insights/insights_env.j2
@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+
+{% for name,value in insights_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}
diff --git a/playbooks/roles/insights/templates/edx/app/supervisor/conf.d.available/insights.conf.j2 b/playbooks/roles/insights/templates/edx/app/supervisor/conf.d.available/insights.conf.j2
new file mode 100644
index 00000000000..4ccf8cfedf3
--- /dev/null
+++ b/playbooks/roles/insights/templates/edx/app/supervisor/conf.d.available/insights.conf.j2
@@ -0,0 +1,11 @@
+# {{ ansible_managed }}
+
+[program:{{ insights_service_name }}]
+
+command={{ insights_app_dir }}/insights.sh
+user={{ common_web_user }}
+directory={{ insights_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true
diff --git a/playbooks/roles/jenkins_admin/defaults/main.yml b/playbooks/roles/jenkins_admin/defaults/main.yml
index 9ba98e61bc3..95321c2aa4d 100644
--- a/playbooks/roles/jenkins_admin/defaults/main.yml
+++ b/playbooks/roles/jenkins_admin/defaults/main.yml
@@ -119,7 +119,7 @@ jenkins_admin_plugins:
- { name: "jquery", version: "1.7.2-1" }
- { name: "dashboard-view", version: "2.9.4" }
- { name: "build-pipeline-plugin", version: "1.4.3" }
- - { name: "s3", version: "0.5" }
+ - { name: "s3", version: "0.6" }
- { name: "tmpcleaner", version: "1.1" }
- { name: "jobConfigHistory", version: "2.8" }
- { name: "build-timeout", version: "1.14" }
diff --git a/playbooks/roles/jenkins_admin/meta/main.yml b/playbooks/roles/jenkins_admin/meta/main.yml
index a20cb27a12f..138fff60852 100644
--- a/playbooks/roles/jenkins_admin/meta/main.yml
+++ b/playbooks/roles/jenkins_admin/meta/main.yml
@@ -9,12 +9,12 @@
#
##
# Role includes for role jenkins_admin
-#
+#
# Example:
#
# dependencies:
# - {
-# role: my_role
+# role: my_role
# my_role_var0: "foo"
# my_role_var1: "bar"
# }
@@ -22,7 +22,7 @@ dependencies:
- common
- aws
- role: jenkins_master
- jenkins_plugins: $jenkins_admin_plugins
+ jenkins_plugins: "{{ jenkins_admin_plugins }}"
- role: supervisor
supervisor_app_dir: "{{ jenkins_supervisor_app_dir }}"
supervisor_data_dir: "{{ jenkins_supervisor_data_dir }}"
diff --git a/playbooks/roles/jenkins_admin/tasks/nat_monitor.yml b/playbooks/roles/jenkins_admin/tasks/nat_monitor.yml
index 968e2703dfd..f1c539add40 100644
--- a/playbooks/roles/jenkins_admin/tasks/nat_monitor.yml
+++ b/playbooks/roles/jenkins_admin/tasks/nat_monitor.yml
@@ -25,6 +25,7 @@
group="{{ jenkins_group }}"
mode="755"
sudo_user: "{{ jenkins_user }}"
+ notify: restart nat monitor
- name: create a supervisor config
template:
diff --git a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/aws_config.j2 b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/aws_config.j2
index f3b34b18649..2ebf8796e63 100644
--- a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/aws_config.j2
+++ b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/aws_config.j2
@@ -1,5 +1,5 @@
{% for deployment, creds in JENKINS_ADMIN_AWS_CREDENTIALS.iteritems() %}
-[profile {{deployment}}]
+[profile {{ deployment }}]
aws_access_key_id = {{ creds.access_id }}
aws_secret_access_key = {{ creds.secret_key }}
diff --git a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/boto.j2 b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/boto.j2
index f3b34b18649..2ebf8796e63 100644
--- a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/boto.j2
+++ b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/boto.j2
@@ -1,5 +1,5 @@
{% for deployment, creds in JENKINS_ADMIN_AWS_CREDENTIALS.iteritems() %}
-[profile {{deployment}}]
+[profile {{ deployment }}]
aws_access_key_id = {{ creds.access_id }}
aws_secret_access_key = {{ creds.secret_key }}
diff --git a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/jobs/backup-jenkins/config.xml.j2 b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/jobs/backup-jenkins/config.xml.j2
index 3357f9fb004..5369df37198 100644
--- a/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/jobs/backup-jenkins/config.xml.j2
+++ b/playbooks/roles/jenkins_admin/templates/edx/var/jenkins/jobs/backup-jenkins/config.xml.j2
@@ -46,7 +46,7 @@ rm -rf $BUILD_ID
{{ JENKINS_ADMIN_S3_PROFILE.name }}
- edx-jenkins-backups/{{JENKINS_ADMIN_NAME}}
+ edx-jenkins-backups/{{ JENKINS_ADMIN_NAME }}
${BUILD_ID}.tar.gz
STANDARD
US_EAST_1
diff --git a/playbooks/roles/jenkins_master/defaults/main.yml b/playbooks/roles/jenkins_master/defaults/main.yml
index 05b10fdb066..5cef7b6a6f3 100644
--- a/playbooks/roles/jenkins_master/defaults/main.yml
+++ b/playbooks/roles/jenkins_master/defaults/main.yml
@@ -12,6 +12,7 @@ jenkins_plugins:
- { name: "build-name-setter", version: "1.3" }
- { name: "build-pipeline-plugin", version: "1.4" }
- { name: "build-timeout", version: "1.11" }
+ - { name: "cloudbees-folder", version: "4.6.1" }
- { name: "cobertura", version: "1.9.2" }
- { name: "copyartifact", version: "1.28" }
- { name: "copy-to-slave", version: "1.4.3" }
@@ -35,7 +36,7 @@ jenkins_plugins:
- { name: "postbuild-task", version: "1.8" }
- { name: "PrioritySorter", version: "2.8" }
- { name: "sauce-ondemand", version: "1.61" }
- - { name: "s3", version: "0.5" }
+ - { name: "s3", version: "0.6" }
- { name: "ssh-agent", version: "1.3" }
- { name: "ssh-credentials", version: "1.5.1" }
- { name: "ssh-slaves", version: "1.4" }
diff --git a/playbooks/roles/jenkins_master/tasks/main.yml b/playbooks/roles/jenkins_master/tasks/main.yml
index c15eb42f2da..08f57308099 100644
--- a/playbooks/roles/jenkins_master/tasks/main.yml
+++ b/playbooks/roles/jenkins_master/tasks/main.yml
@@ -58,7 +58,7 @@
# Using this instead of the user module because the user module
# fails if the directory exists.
- name: set home directory for jenkins user
- shell: usermod -d {{jenkins_home}} {{jenkins_user}}
+ shell: usermod -d {{ jenkins_home }} {{ jenkins_user }}
- name: make plugins directory
file:
diff --git a/playbooks/roles/jenkins_worker/tasks/packer.yml b/playbooks/roles/jenkins_worker/tasks/packer.yml
index 8f2298a16b2..ce700422e0f 100644
--- a/playbooks/roles/jenkins_worker/tasks/packer.yml
+++ b/playbooks/roles/jenkins_worker/tasks/packer.yml
@@ -1,6 +1,6 @@
---
- name: Download packer
- get_url: url={{ packer_url }} dest=/var/tmp/packer.zip
+ shell: "curl -L {{ packer_url }} -o /var/tmp/packer.zip"
- name: Unzip packer
unarchive: src=/var/tmp/packer.zip dest=/usr/local/bin copy=no
diff --git a/playbooks/roles/kibana/templates/config.js.j2 b/playbooks/roles/kibana/templates/config.js.j2
index 7968b3dee71..31885f69e99 100644
--- a/playbooks/roles/kibana/templates/config.js.j2
+++ b/playbooks/roles/kibana/templates/config.js.j2
@@ -21,11 +21,11 @@ function (Settings) {
//elasticsearch: "http://"+window.location.hostname+":9200",
{% if NGINX_ENABLE_SSL %}
- elasticsearch: "https://{{ KIBANA_SERVER_NAME }}/e",
+ elasticsearch: "https://{{ KIBANA_SERVER_NAME }}:{{ KIBANA_SSL_NGINX_PORT }}/e",
{% else %}
- elasticsearch: "http://{{ KIBANA_SERVER_NAME }}/e",
+ elasticsearch: "http://{{ KIBANA_SERVER_NAME }}:{{ KIBANA_NGINX_PORT }}/e",
{% endif %}
diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml
index 54801cca74e..bdd49f7b3bb 100644
--- a/playbooks/roles/launch_ec2/tasks/main.yml
+++ b/playbooks/roles/launch_ec2/tasks/main.yml
@@ -21,7 +21,7 @@
module: ec2
state: 'absent'
region: "{{ region }}"
- instance_ids: ${tag_lookup.instance_ids}
+ instance_ids: "{{tag_lookup.instance_ids}}"
when: terminate_instance == true and tag_lookup.instance_ids|length == 1
- name: deregister instance from an an elb if it was in one
@@ -45,7 +45,7 @@
assign_public_ip: yes
wait: true
region: "{{ region }}"
- instance_tags: "{{instance_tags}}"
+ instance_tags: "{{ instance_tags }}"
volumes:
- device_name: /dev/sda1
volume_size: "{{ root_ebs_size }}"
diff --git a/playbooks/roles/legacy_ora/tasks/main.yml b/playbooks/roles/legacy_ora/tasks/main.yml
index e46c8145bd4..9bb523f52e0 100644
--- a/playbooks/roles/legacy_ora/tasks/main.yml
+++ b/playbooks/roles/legacy_ora/tasks/main.yml
@@ -18,15 +18,15 @@
- name: create ora application config
copy:
- src={{secure_dir}}/files/{{COMMON_ENVIRONMENT}}/legacy_ora/ora.env.json
- dest={{ora_app_dir}}/env.json
+ src={{ secure_dir }}/files/{{ COMMON_ENVIRONMENT }}/legacy_ora/ora.env.json
+ dest={{ ora_app_dir }}/env.json
sudo_user: "{{ ora_user }}"
register: env_state
- name: create ora auth file
copy:
- src={{secure_dir}}/files/{{COMMON_ENVIRONMENT}}/legacy_ora/ora.auth.json
- dest={{ora_app_dir}}/auth.json
+ src={{ secure_dir }}/files/{{ COMMON_ENVIRONMENT }}/legacy_ora/ora.auth.json
+ dest={{ ora_app_dir }}/auth.json
sudo_user: "{{ ora_user }}"
register: auth_state
diff --git a/playbooks/roles/logstash/tasks/main.yml b/playbooks/roles/logstash/tasks/main.yml
index 8f2d3bce99b..24c931e1fcd 100644
--- a/playbooks/roles/logstash/tasks/main.yml
+++ b/playbooks/roles/logstash/tasks/main.yml
@@ -32,7 +32,7 @@
get_url: url={{ logstash_url }} dest={{ logstash_app_dir }}/share/{{ logstash_file }}
- name: ensure symlink with no version exists at {{ logstash_app_dir }}/share/logstash.jar
- file: src={{ logstash_app_dir }}/share/${logstash_file} dest={{ logstash_app_dir }}/share/logstash.jar state=link
+ file: src={{ logstash_app_dir }}/share/{{ logstash_file }} dest={{ logstash_app_dir }}/share/logstash.jar state=link
- name: start logstash
action: service name=logstash state=started enabled=yes
diff --git a/playbooks/roles/mariadb/defaults/main.yml b/playbooks/roles/mariadb/defaults/main.yml
new file mode 100644
index 00000000000..08f144229bc
--- /dev/null
+++ b/playbooks/roles/mariadb/defaults/main.yml
@@ -0,0 +1,118 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role mariadb
+#
+MARIADB_APT_KEY_ID: '0xcbcb082a1bb943db'
+# Note: version is determined by repo
+MARIADB_REPO: "deb http://mirrors.syringanetworks.net/mariadb/repo/10.0/ubuntu precise main"
+
+MARIADB_CREATE_DBS: yes
+MARIADB_CLUSTERED: no
+MARIADB_CLUSTER_USER_ADMIN: "mariadb_clu_root"
+MARIADB_CLUSTER_PASSWORD_ADMIN: "password"
+MARIADB_HOST_PRIV: '%'
+
+MARIADB_HAPROXY_USER: 'haproxy'
+MARIADB_HAPROXY_HOSTS:
+ - '192.168.33.100'
+ - '192.168.33.110'
+ - '192.168.33.120'
+
+MARIADB_LISTEN_ALL: false
+
+MARIADB_DATABASES:
+ - "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}"
+ - "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}"
+ - "{{ ORA_MYSQL_DB_NAME|default('ora') }}"
+
+MARIADB_ANALYTICS_DATABASES:
+ - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}"
+ - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}"
+
+MARIADB_USERS:
+ - name: "{{ EDXAPP_MYSQL_USER|default('edxapp001') }}"
+ pass: "{{ EDXAPP_MYSQL_PASSWORD|default('password') }}"
+ priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ XQUEUE_MYSQL_USER|default('xqueue001') }}"
+ pass: "{{ XQUEUE_MYSQL_PASSWORD|default('password') }}"
+ priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ ORA_MYSQL_USER|default('ora001') }}"
+ pass: "{{ ORA_MYSQL_PASSWORD|default('password') }}"
+ priv: "{{ ORA_MYSQL_DB_NAME|default('ora') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+ pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+ priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+ pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+ priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+ pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+ priv: "{{ ORA_MYSQL_DB_NAME|default('ora') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_READ_ONLY_USER|default('read_only') }}"
+ pass: "{{ COMMON_MYSQL_READ_ONLY_PASS|default('password') }}"
+ priv: "*.*:SELECT"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_ADMIN_USER|default('admin') }}"
+ pass: "{{ COMMON_MYSQL_ADMIN_PASS|default('password') }}"
+ priv: "*.*:CREATE USER"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+MARIADB_ANALYTICS_USERS:
+ - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['USER']|default('api001') }}"
+ pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['PASSWORD']|default('password') }}"
+ priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['USER']|default('reports001') }}"
+ pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['PASSWORD']|default('password') }}"
+ priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+ pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+ priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+ - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+ pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+ priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}.*:ALL"
+ host: "{{ MARIADB_HOST_PRIV }}"
+
+#
+# OS packages
+#
+mariadb_debian_pkgs:
+ - python-software-properties
+ - python-mysqldb
+
+mariadb_redhat_pkgs: []
+
+mariadb_apt_repository:
+
+mariadb_solo_packages:
+ - mariadb-server
+
+mariadb_cluster_packages:
+ - mariadb-galera-server
+ - galera
diff --git a/playbooks/roles/mariadb/meta/main.yml b/playbooks/roles/mariadb/meta/main.yml
new file mode 100644
index 00000000000..24e2fcd6d19
--- /dev/null
+++ b/playbooks/roles/mariadb/meta/main.yml
@@ -0,0 +1,22 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role mariadb
+#
+# Example:
+#
+# dependencies:
+# - {
+# role: my_role
+# my_role_var0: "foo"
+# my_role_var1: "bar"
+# }
+dependencies:
+ - common
diff --git a/playbooks/roles/mariadb/tasks/cluster.yml b/playbooks/roles/mariadb/tasks/cluster.yml
new file mode 100644
index 00000000000..185d8b823b7
--- /dev/null
+++ b/playbooks/roles/mariadb/tasks/cluster.yml
@@ -0,0 +1,58 @@
+- name: copy galera cluster config
+ template: >
+ src="etc/mysql/conf.d/galera.cnf.j2"
+ dest="/etc/mysql/conf.d/galera.cnf"
+ owner="root"
+ group="root"
+ mode=0600
+
+- name: check if we have already bootstrapped the cluster
+ stat: path=/etc/mysql/ansible_cluster_started
+ register: mariadb_bootstrap
+
+- name: stop mysql for cluster bootstrap
+ service: name=mysql state=stopped
+ when: not mariadb_bootstrap.stat.exists
+
+- name: setup bootstrap on primary
+ lineinfile: >
+ dest="/etc/mysql/conf.d/galera.cnf"
+ regexp="^wsrep_cluster_address=gcomm://{{ hostvars.keys()|sort|join(',') }}$"
+ line="wsrep_cluster_address=gcomm://"
+ when: ansible_hostname == hostvars[hostvars.keys()[0]].ansible_hostname and not mariadb_bootstrap.stat.exists
+
+- name: fetch debian.cnf file so start-stop will work properly
+ fetch: >
+ src=/etc/mysql/debian.cnf
+ dest=/tmp/debian.cnf
+ fail_on_missing=yes
+ flat=yes
+ when: ansible_hostname == hostvars[hostvars.keys()[0]].ansible_hostname and not mariadb_bootstrap.stat.exists
+ register: mariadb_new_debian_cnf
+
+- name: copy fetched file to other cluster members
+ copy: src=/tmp/debian.cnf dest=/etc/mysql/debian.cnf
+ when: mariadb_new_debian_cnf is defined
+
+- name: start everything
+ service: name=mysql state=started
+ when: not mariadb_bootstrap.stat.exists
+
+- name: reset galera cluster config since we are bootstrapped
+ template: >
+ src="etc/mysql/conf.d/galera.cnf.j2"
+ dest="/etc/mysql/conf.d/galera.cnf"
+ owner="root"
+ group="root"
+ mode=0600
+ when: not mariadb_bootstrap.stat.exists
+
+- name: touch bootstrap file to confirm we are fully up
+ file: path="/etc/mysql/ansible_cluster_started" state=touch
+
+# This is needed for mysql-check in haproxy or other mysql monitor
+# scripts to prevent haproxy checks exceeding `max_connect_errors`.
+- name: create haproxy monitor user
+ command: >
+ mysql -e "INSERT INTO mysql.user (Host,User) values ('{{ item }}','{{ MARIADB_HAPROXY_USER }}'); FLUSH PRIVILEGES;"
+ with_items: MARIADB_HAPROXY_HOSTS
diff --git a/playbooks/roles/mariadb/tasks/main.yml b/playbooks/roles/mariadb/tasks/main.yml
new file mode 100644
index 00000000000..64f7f0bcb99
--- /dev/null
+++ b/playbooks/roles/mariadb/tasks/main.yml
@@ -0,0 +1,93 @@
+---
+#
+# edX Configuration
+#
+# github: https://github.com/edx/configuration
+# wiki: https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role mariadb
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: Install pre-req debian packages
+ apt: name={{ item }} state=present
+ with_items: mariadb_debian_pkgs
+
+- name: Add mongo key
+ apt_key: url="{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ MARIADB_APT_KEY_ID }}"
+
+- name: add the mariadb repo to the sources list
+ apt_repository: >
+ repo='{{ MARIADB_REPO }}'
+ state=present
+
+- name: install mariadb solo packages
+ apt: name={{ item }} update_cache=yes
+ with_items: mariadb_solo_packages
+ when: not MARIADB_CLUSTERED|bool
+
+- name: install mariadb cluster packages
+ apt: name={{ item }} update_cache=yes
+ with_items: mariadb_cluster_packages
+ when: MARIADB_CLUSTERED|bool
+
+- name: remove bind-address
+ lineinfile: >
+ dest=/etc/mysql/my.cnf
+ regexp="^bind-address\s+=\s+127\.0\.0\.1$"
+ state=absent
+ when: MARIADB_LISTEN_ALL|bool or MARIADB_CLUSTERED|bool
+
+- include: cluster.yml
+ when: MARIADB_CLUSTERED|bool
+
+- name: start everything
+ service: name=mysql state=started
+
+- name: create all databases
+ mysql_db: >
+ db={{ item }}
+ state=present
+ encoding=utf8
+ with_items: MARIADB_DATABASES
+ when: MARIADB_CREATE_DBS|bool
+
+- name: create all analytics dbs
+ mysql_db: >
+ db={{ item }}
+ state=present
+ encoding=utf8
+ with_items: MARIADB_ANALYTICS_DATABASES
+ when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined
+
+- name: create all users/privs
+ mysql_user: >
+ name="{{ item.name }}"
+ password="{{ item.pass }}"
+ priv="{{ item.priv }}"
+ host="{{ item.host }}"
+ append_privs=yes
+ with_items: MARIADB_USERS
+ when: MARIADB_CREATE_DBS|bool
+
+- name: create all analytics users/privs
+ mysql_user: >
+ name="{{ item.name }}"
+ password="{{ item.pass }}"
+ priv="{{ item.priv }}"
+ host="{{ item.host }}"
+ append_privs=yes
+ with_items: MARIADB_ANALYTICS_USERS
+ when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined
diff --git a/playbooks/roles/mariadb/templates/etc/mysql/conf.d/galera.cnf.j2 b/playbooks/roles/mariadb/templates/etc/mysql/conf.d/galera.cnf.j2
new file mode 100644
index 00000000000..9a7c8b09e10
--- /dev/null
+++ b/playbooks/roles/mariadb/templates/etc/mysql/conf.d/galera.cnf.j2
@@ -0,0 +1,17 @@
+{%- set hosts= [] -%}
+{%- for host in hostvars.keys()|sort -%}
+ {% do hosts.append(host) %}
+{%- endfor %}
+[mysqld]
+binlog_format=ROW
+innodb_autoinc_lock_mode=2
+innodb_doublewrite=1
+query_cache_size=0
+
+wsrep_provider=/usr/lib/galera/libgalera_smm.so
+wsrep_cluster_address=gcomm://{{ hosts|join(',') }}?pc.wait_prim=no
+wsrep_sst_auth={{ MARIADB_CLUSTER_USER_ADMIN }}:{{ MARIADB_CLUSTER_PASSWORD_ADMIN }}
+
+{% if vagrant_cluster|bool %}
+wsrep_node_address={{ ansible_ssh_host }}
+{% endif %}
diff --git a/playbooks/roles/minos/tasks/main.yml b/playbooks/roles/minos/tasks/main.yml
index 0a346023b84..1a83debe622 100644
--- a/playbooks/roles/minos/tasks/main.yml
+++ b/playbooks/roles/minos/tasks/main.yml
@@ -61,6 +61,8 @@
- "ProccessQuienscenceVoterCelery"
- "ProccessQuienscenceVoterGunicorn"
- "TrackingLogVoter"
+ - "ZippedTrackingLogVoter"
+ - "RolledTrackingLogVoter"
# Optional auth for git
- name: create ssh script for git (not authenticated)
diff --git a/playbooks/roles/minos/templates/edx/etc/minos/conf.d/RolledTrackingLogVoter.yml.j2 b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/RolledTrackingLogVoter.yml.j2
new file mode 100644
index 00000000000..e3f7e169bed
--- /dev/null
+++ b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/RolledTrackingLogVoter.yml.j2
@@ -0,0 +1,3 @@
+RolledTrackingLogVoter:
+ config:
+ tracking_directory: '{{ COMMON_LOG_DIR }}/tracking'
\ No newline at end of file
diff --git a/playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2 b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2
index 98aca96c82b..ada33dd324d 100644
--- a/playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2
+++ b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2
@@ -2,5 +2,5 @@ TrackingLogVoter:
config:
aws_profile: !!null
local_directory: '{{ COMMON_LOG_DIR }}/tracking'
- s3_bucket: 'edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}'
+ s3_bucket: '{{ COMMON_AWS_SYNC_BUCKET }}'
bucket_path_prefix: 'logs/tracking'
diff --git a/playbooks/roles/minos/templates/edx/etc/minos/conf.d/ZippedTrackingLogVoter.yml.j2 b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/ZippedTrackingLogVoter.yml.j2
new file mode 100644
index 00000000000..aa774a5f5fe
--- /dev/null
+++ b/playbooks/roles/minos/templates/edx/etc/minos/conf.d/ZippedTrackingLogVoter.yml.j2
@@ -0,0 +1,3 @@
+ZippedTrackingLogVoter:
+ config:
+ tracking_directory: '{{ COMMON_LOG_DIR }}/tracking'
\ No newline at end of file
diff --git a/playbooks/roles/mongo/defaults/main.yml b/playbooks/roles/mongo/defaults/main.yml
index 0264f1b6212..8a7b4550f48 100644
--- a/playbooks/roles/mongo/defaults/main.yml
+++ b/playbooks/roles/mongo/defaults/main.yml
@@ -1,5 +1,5 @@
mongo_logappend: true
-mongo_version: 2.4.7
+mongo_version: 2.6.4
mongo_port: "27017"
mongo_extra_conf: ''
mongo_key_file: '/etc/mongodb_key'
@@ -31,8 +31,8 @@ mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
# Have to use this conditional instead of ignore errors
-# because the mongo_user module fails and doesn't ginore errors.
-mongo_create_users: !!null
+# because the mongo_user module fails and doesn't ignore errors.
+mongo_create_users: true
# If the system is running out of an Amazon Web Services
# cloudformation stack, this group name can used to pull out
@@ -42,3 +42,21 @@ mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
# In environments that do not require durability (devstack / Jenkins)
# you can disable the journal to reduce disk usage
mongo_enable_journal: True
+
+# We can do regular backups of MongoDB to S3.
+MONGO_S3_BACKUP: false
+# backup cron time:
+MONGO_S3_BACKUP_HOUR: "*/12"
+MONGO_S3_BACKUP_DAY: "*"
+# override with a secondary node that will perform backups
+MONGO_S3_BACKUP_NODE: "undefined"
+# back up data into a specific S3 bucket
+MONGO_S3_BACKUP_BUCKET: "undefined"
+# temporary directory mongodump will use to store data
+MONGO_S3_BACKUP_TEMPDIR: "{{ mongo_data_dir }}"
+MONGO_S3_NOTIFY_EMAIL: "dummy@example.com"
+mongo_s3_logfile: "{{ COMMON_LOG_DIR }}/mongo/s3-mongo-backup.log"
+MONGO_S3_S3CMD_CONFIG: "{{ COMMON_DATA_DIR }}/mongo-s3-backup.s3cfg"
+MONGO_S3_BACKUP_AWS_ACCESS_KEY: !!null
+MONGO_S3_BACKUP_AWS_SECRET_KEY: !!null
+
diff --git a/playbooks/roles/mongo/handlers/main.yml b/playbooks/roles/mongo/handlers/main.yml
index 108f006d984..83a9ecdaf94 100644
--- a/playbooks/roles/mongo/handlers/main.yml
+++ b/playbooks/roles/mongo/handlers/main.yml
@@ -1,4 +1,4 @@
---
- name: restart mongo
- service: name=mongodb state=restarted
+ service: name=mongod state=restarted
diff --git a/playbooks/roles/mongo/tasks/main.yml b/playbooks/roles/mongo/tasks/main.yml
index 827e7cf86ba..b0751a2857e 100644
--- a/playbooks/roles/mongo/tasks/main.yml
+++ b/playbooks/roles/mongo/tasks/main.yml
@@ -1,5 +1,14 @@
---
+- name: check to see that MongoDB 2.4 isn't installed
+ stat: path=/etc/init.d/mongodb
+ register: mongodb_needs_upgrade
+
+- name: verify 2.4 not installed
+ fail: msg="MongoDB 2.4 is currently installed. If on a stand alone host (devstack), apt-get remove mongodb-10gen and re-run ansible. if on a cluster, read http://docs.mongodb.org/manual/release-notes/2.6-upgrade/#upgrade-considerations and upgrade to 2.6."
+ when: mongodb_needs_upgrade.stat.exists
+
+
- name: install python pymongo for mongo_user ansible module
pip: >
name=pymongo state=present
@@ -8,7 +17,7 @@
- name: add the mongodb signing key
apt_key: >
id=7F0CEB10
- url={{MONGODB_APT_KEY}}
+ url={{ MONGODB_APT_KEY }}
state=present
- name: add the mongodb repo to the sources list
@@ -19,7 +28,7 @@
- name: install mongo server and recommends
apt: >
- pkg=mongodb-10gen={{ mongo_version }}
+ pkg=mongodb-org={{ mongo_version }}
state=present install_recommends=yes
force=yes update_cache=yes
@@ -33,8 +42,8 @@
- "{{ mongo_dbpath }}"
- "{{ mongo_log_dir }}"
-- name: stop mongo service
- service: name=mongodb state=stopped
+- name: stop mongod service
+ service: name=mongod state=stopped
- name: move mongodb to {{ mongo_data_dir }}
command: mv /var/lib/mongodb {{ mongo_data_dir}}/. creates={{ mongo_data_dir }}/mongodb
@@ -50,11 +59,11 @@
when: MONGO_CLUSTERED
- name: copy configuration template
- template: src=mongodb.conf.j2 dest=/etc/mongodb.conf backup=yes
+ template: src=mongodb.conf.j2 dest=/etc/mongod.conf backup=yes
notify: restart mongo
- name: start mongo service
- service: name=mongodb state=started
+ service: name=mongod state=started
- name: wait for mongo server to start
wait_for: port=27017 delay=2
@@ -77,3 +86,39 @@
state=present
with_items: MONGO_USERS
when: mongo_create_users
+
+- name: install s3cmd
+ pip: >
+ name="s3cmd"
+ state=present
+ extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+ when: MONGO_S3_BACKUP
+
+- name: configure s3cmd
+ template: >
+ dest="{{ MONGO_S3_S3CMD_CONFIG }}"
+ src=mongo-s3-backup-s3cfg.j2
+ owner=root
+ group=root
+ mode=0600
+ when: MONGO_S3_BACKUP
+
+- name: install backup-mongo-to-s3 script
+ template: >
+ src=backup-mongo-to-s3.j2
+ dest=/edx/bin/backup-mongo-to-s3.sh
+ owner=root
+ group=root
+ mode=0700
+ when: MONGO_S3_BACKUP
+
+- name: schedule backup-mongo-to-3s crontab
+ cron:
+ name="backup-mongo-to-s3"
+ job="/edx/bin/backup-mongo-to-s3.sh"
+ backup=yes
+ cron_file=backup-mongo-to-s3
+ user=root
+ hour="{{ MONGO_S3_BACKUP_HOUR }}"
+ minute="0"
+ day="{{ MONGO_S3_BACKUP_DAY }}"
diff --git a/playbooks/roles/mongo/templates/backup-mongo-to-s3.j2 b/playbooks/roles/mongo/templates/backup-mongo-to-s3.j2
new file mode 100644
index 00000000000..c44d19aa8cc
--- /dev/null
+++ b/playbooks/roles/mongo/templates/backup-mongo-to-s3.j2
@@ -0,0 +1,68 @@
+{% set lb = '{' %}
+{% set rb = '}' %}
+#!/bin/bash
+#
+
+exec > >(tee "{{ mongo_s3_logfile }}")
+exec 2>&1
+
+shopt -s extglob
+
+usage() {
+
+ cat<
+ dest={{ newrelic_logwatch_repo_dir }}
+ repo={{ newrelic_logwatch_repo }} version={{ newrelic_logwatch_version }}
+ accept_hostkey=yes
+
+- name: bundle install
+ shell: >
+ chdir={{ newrelic_logwatch_dir }}
+ creates=/var/lib/gems/1.8/gems/newrelic_plugin-1.0.2/
+ bundle install
+ notify: restart newrelic-logwatch-agent
+
+- name: create agent configuration
+ template: >
+ src=opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2
+ dest={{ newrelic_logwatch_dir }}/config/newrelic_plugin.yml.copy
+ notify: restart newrelic-logwatch-agent
+
+- template:
+ owner: root
+ src: etc/init/newrelic-logwatch-agent.conf.j2
+ dest: /etc/init/newrelic-logwatch-agent.conf
+ notify: restart newrelic-logwatch-agent
diff --git a/playbooks/roles/newrelic/templates/etc/init/newrelic-logwatch-agent.conf.j2 b/playbooks/roles/newrelic/templates/etc/init/newrelic-logwatch-agent.conf.j2
new file mode 100644
index 00000000000..4c342fb29dc
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/etc/init/newrelic-logwatch-agent.conf.j2
@@ -0,0 +1,11 @@
+description "newrelic logwatch plugin"
+start on runlevel [2345]
+stop on runlevel [016]
+respawn
+chdir {{ newrelic_logwatch_dir }}
+pre-start script
+ ami_id=$(ec2metadata --ami-id)
+ hostname=$(hostname)
+ sed "s/HOSTNAME/${ami_id}-${hostname}/" {{ newrelic_logwatch_dir }}/config/newrelic_plugin.yml.copy > {{ newrelic_logwatch_dir }}/config/newrelic_plugin.yml
+end script
+exec ruby newrelic_logwatcher_agent.rb
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-503.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-503.j2
new file mode 100644
index 00000000000..92b6822e85c
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-503.j2
@@ -0,0 +1,15 @@
+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME + '-nginx-503' %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') + '-nginx-503' + '-HOSTNAME' %}
+{% endif %}
+
+ {{ server_name }}:
+ # Full path to the the log file
+ log_path: {{ nginx_log_dir|default('/edx/var/log/nginx') }}/error.log
+ # Returns the number of matches for this term. Use Linux Regex formatting.
+ term: "limiting requests"
+ # Provide any options to pass to grep when running.
+ # For example, to count non-matching lines, enter 'v'.
+ # Use the abbreviated format ('v' and not 'invert-match').
+ grep_options:
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-certs-errors.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-certs-errors.j2
new file mode 100644
index 00000000000..6fdb9ac73e1
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-certs-errors.j2
@@ -0,0 +1,15 @@
+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME + '-certs-errors' %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') + '-certs-errors' + '-HOSTNAME'%}
+{% endif %}
+
+ {{ server_name }}:
+ # Full path to the the log file
+ log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/certs/edx.log
+ # Returns the number of matches for this term. Use Linux Regex formatting.
+ term: " ERROR "
+ # Provide any options to pass to grep when running.
+ # For example, to count non-matching lines, enter 'v'.
+ # Use the abbreviated format ('v' and not 'invert-match').
+ grep_options:
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-cms-errors.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-cms-errors.j2
new file mode 100644
index 00000000000..54271393804
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-cms-errors.j2
@@ -0,0 +1,15 @@
+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME + '-cms-errors' %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') + '-cms-errors' + '-HOSTNAME' %}
+{% endif %}
+
+ {{ server_name }}:
+ # Full path to the the log file
+ log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/cms/edx.log
+ # Returns the number of matches for this term. Use Linux Regex formatting.
+ term: " ERROR "
+ # Provide any options to pass to grep when running.
+ # For example, to count non-matching lines, enter 'v'.
+ # Use the abbreviated format ('v' and not 'invert-match').
+ grep_options:
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-lms-errors.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-lms-errors.j2
new file mode 100644
index 00000000000..5c01535a70f
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-lms-errors.j2
@@ -0,0 +1,15 @@
+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME + '-lms-errors' %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') + '-lms-errors' + '-HOSTNAME' %}
+{% endif %}
+
+ {{ server_name }}:
+ # Full path to the the log file
+ log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/lms/edx.log
+ # Returns the number of matches for this term. Use Linux Regex formatting.
+ term: " ERROR "
+ # Provide any options to pass to grep when running.
+ # For example, to count non-matching lines, enter 'v'.
+ # Use the abbreviated format ('v' and not 'invert-match').
+ grep_options:
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-xqueue-errors.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-xqueue-errors.j2
new file mode 100644
index 00000000000..8e5165fd928
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-xqueue-errors.j2
@@ -0,0 +1,15 @@
+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME + '-xqueue-errors' %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') + '-xqueue-errors' + '-HOSTNAME' %}
+{% endif %}
+
+ {{ server_name }}:
+ # Full path to the the log file
+ log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/xqueue/edx.log
+ # Returns the number of matches for this term. Use Linux Regex formatting.
+ term: " ERROR "
+ # Provide any options to pass to grep when running.
+ # For example, to count non-matching lines, enter 'v'.
+ # Use the abbreviated format ('v' and not 'invert-match').
+ grep_options:
diff --git a/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2 b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2
new file mode 100644
index 00000000000..4f456403dd6
--- /dev/null
+++ b/playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2
@@ -0,0 +1,33 @@
+#
+#
+# This is a sample newrelic_plugin.yml file. Please move this file
+# to the following location if it is not already there:
+#
+# ./config/newrelic_plugin.yml
+#
+# Where the current directory is the directory where your main program resides and is your current
+# directory when you run the main program.
+#
+# Please make sure to update the license_key information with the license key for your New Relic
+# account.
+#
+#
+newrelic:
+ #
+ # Update with your New Relic account license key:
+ #
+ license_key: '{{ NEWRELIC_LICENSE_KEY }}'
+ #
+ # Set to '1' for verbose output, remove for normal output.
+ # All output goes to stdout/stderr.
+ #
+ # verbose: 1
+#
+# Agent Configuration:
+#
+agents:
+{% for agent in NEWRELIC_LOGWATCH %}
+
+{% include agent %}
+
+{% endfor %}
diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml
index 512d4a89a2d..b70ff7290f8 100644
--- a/playbooks/roles/nginx/defaults/main.yml
+++ b/playbooks/roles/nginx/defaults/main.yml
@@ -23,6 +23,7 @@ NGINX_ENABLE_SSL: False
NGINX_SSL_CERTIFICATE: 'ssl-cert-snakeoil.pem'
NGINX_SSL_KEY: 'ssl-cert-snakeoil.key'
+NGINX_LOG_FORMAT_NAME: 'p_combined'
# When set to False, nginx will pass X-Forwarded-For, X-Forwarded-Port,
# and X-Forwarded-Proto headers through to the backend unmodified.
# This is desired when nginx is deployed behind another load balancer
@@ -32,8 +33,24 @@ NGINX_SSL_KEY: 'ssl-cert-snakeoil.key'
# headers to reflect the properties of the incoming request.
NGINX_SET_X_FORWARDED_HEADERS: False
+NGINX_SERVER_ERROR_IMG: 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/11/Pendleton_Sinking_Ship.jpg/640px-Pendleton_Sinking_Ship.jpg'
+NGINX_SERVER_HTML_FILES:
+ - file: rate-limit.html
+ title: 'Rate limit exceeded'
+ msg: 'If think you have encountered this message in error please let us know at {{ EDXAPP_TECH_SUPPORT_EMAIL|default("technical@example.com") }}'
+ img: "{{ NGINX_SERVER_ERROR_IMG }}"
+ heading: 'Uh oh, we are having some server issues..'
+ - file: server-error.html
+ title: 'Server error'
+ msg: 'We have been notified of the error, if it persists please let us know at {{ EDXAPP_TECH_SUPPORT_EMAIL|default("technical@example.com") }}'
+ img: "{{ NGINX_SERVER_ERROR_IMG }}"
+ heading: 'Uh oh, we are having some server issues..'
+
+
+
nginx_app_dir: "{{ COMMON_APP_DIR }}/nginx"
nginx_data_dir: "{{ COMMON_DATA_DIR }}/nginx"
+nginx_server_static_dir: "{{ nginx_data_dir }}/server-static"
nginx_conf_dir: "{{ nginx_app_dir }}/conf.d"
nginx_log_dir: "{{ COMMON_LOG_DIR }}/nginx"
nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available"
@@ -63,6 +80,8 @@ nginx_cms_gunicorn_hosts:
- 127.0.0.1
nginx_analytics_api_gunicorn_hosts:
- 127.0.0.1
+nginx_insights_gunicorn_hosts:
+ - 127.0.0.1
nginx_cfg:
# - link - turn on
diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml
index 61f6e68be3a..5f38fdaab04 100644
--- a/playbooks/roles/nginx/tasks/main.yml
+++ b/playbooks/roles/nginx/tasks/main.yml
@@ -24,6 +24,7 @@
with_items:
- "{{ nginx_data_dir }}"
- "{{ nginx_log_dir }}"
+ - "{{ nginx_server_static_dir }}"
notify: restart nginx
- name: Install nginx packages
@@ -113,6 +114,16 @@
notify: reload nginx
with_dict: nginx_redirects
+ # These are static pages that can be used
+ # for nginx rate limiting, 500 errors, etc.
+
+- name: Create NGINX server templates
+ template: >
+ src=edx/var/nginx/server-static/server-template.j2
+ dest={{ nginx_server_static_dir }}/{{ item.file }}
+ owner=root group={{ common_web_user }} mode=0640
+ with_items: NGINX_SERVER_HTML_FILES
+
- name: Write out htpasswd file
htpasswd: >
name={{ COMMON_HTPASSWD_USER }}
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics-api.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics-api.j2
index 7d93d125dc7..9e29a9abb7b 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics-api.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics-api.j2
@@ -13,12 +13,6 @@ server {
}
location / {
- {% include "basic-auth.j2" %}
- try_files $uri @proxy_to_app;
- }
-
- # No basic auth security on the heartbeat url, so that ELB can use it
- location /api/v0/status {
try_files $uri @proxy_to_app;
}
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/basic-auth.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/basic-auth.j2
index ccddaef6c97..2f7da8eca2c 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/basic-auth.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/basic-auth.j2
@@ -4,6 +4,12 @@
allow 127.0.0.1;
allow 192.168.0.0/16;
allow 172.16.0.0/12;
+
+ allow 10.3.110.0/24;
+ allow 10.3.120.0/24;
+ allow 10.8.110.0/24;
+ allow 10.8.120.0/24;
+
deny all;
auth_basic "Restricted";
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2
index fde35e84342..b7df26b1896 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2
@@ -13,10 +13,13 @@ upstream cms-backend {
server {
# CMS configuration file for nginx, templated by ansible
+ # 500 error pages
+ error_page 500 502 504 /server/server-error.html;
+
{% if NGINX_ENABLE_SSL %}
- listen {{EDXAPP_CMS_NGINX_PORT}} {{default_site}};
- listen {{EDXAPP_CMS_SSL_NGINX_PORT}} ssl;
+ listen {{ EDXAPP_CMS_NGINX_PORT }} {{ default_site }};
+ listen {{ EDXAPP_CMS_SSL_NGINX_PORT }} ssl;
ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
@@ -24,12 +27,12 @@ server {
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
{% else %}
- listen {{EDXAPP_CMS_NGINX_PORT}} {{default_site}};
+ listen {{ EDXAPP_CMS_NGINX_PORT }} {{ default_site }};
{% endif %}
server_name {{ CMS_HOSTNAME }};
- access_log {{ nginx_log_dir }}/access.log;
+ access_log {{ nginx_log_dir }}/access.log {{ NGINX_LOG_FORMAT_NAME }};
error_log {{ nginx_log_dir }}/error.log error;
# CS184 requires uploads of up to 4MB for submitting screenshots.
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2
index dbd3338acbb..353a837304a 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2
@@ -32,7 +32,7 @@ upstream forum_app_server {
server {
server_name forum.*;
- listen {{ FORUM_NGINX_PORT }} {{default_site}};
+ listen {{ FORUM_NGINX_PORT }} {{ default_site }};
client_max_body_size 1M;
keepalive_timeout 5;
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2
new file mode 100644
index 00000000000..ed5df512641
--- /dev/null
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2
@@ -0,0 +1,35 @@
+upstream insights_app_server {
+ {% for host in nginx_insights_gunicorn_hosts %}
+ server {{ host }}:{{ insights_gunicorn_port }} fail_timeout=0;
+ {% endfor %}
+}
+
+server {
+ listen {{ INSIGHTS_NGINX_PORT }} default_server;
+
+ location ~ ^/static/(?P.*) {
+ root {{ COMMON_DATA_DIR }}/{{ insights_service_name }};
+ try_files /staticfiles/$file =404;
+ }
+
+ location / {
+ try_files $uri @proxy_to_app;
+ }
+
+ # No basic auth security on the heartbeat url, so that ELB can use it
+ location /status {
+ try_files $uri @proxy_to_app;
+ }
+
+ {% include "robots.j2" %}
+
+location @proxy_to_app {
+ proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+ proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+ proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+ proxy_set_header Host $http_host;
+
+ proxy_redirect off;
+ proxy_pass http://insights_app_server;
+ }
+}
diff --git a/playbooks/roles/nginx/templates/kibana.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2
similarity index 89%
rename from playbooks/roles/nginx/templates/kibana.j2
rename to playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2
index f07a1d7d9c9..e9a3152bd23 100755
--- a/playbooks/roles/nginx/templates/kibana.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2
@@ -13,14 +13,14 @@ server {
{% if NGINX_ENABLE_SSL %}
- listen {{KIBANA_NGINX_PORT}} {{default_site}};
- listen {{KIBANA_SSL_NGINX_PORT}} {{default_site}} ssl;
+ listen {{ KIBANA_NGINX_PORT }} {{ default_site }};
+ listen {{ KIBANA_SSL_NGINX_PORT }} {{ default_site }} ssl;
ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
{% else %}
- listen {{KIBANA_NGINX_PORT}} {{default_site}};
+ listen {{ KIBANA_NGINX_PORT }} {{ default_site }};
{% endif %}
server_name {{ KIBANA_SERVER_NAME }};
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2
index 4b4631d7264..800680a31a3 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2
@@ -7,7 +7,7 @@ upstream lms-preview-backend {
server {
# LMS-preview configuration file for nginx, templated by ansible
- listen {{EDXAPP_LMS_PREVIEW_NGINX_PORT}};
+ listen {{ EDXAPP_LMS_PREVIEW_NGINX_PORT }};
server_name preview.*;
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2
index 9d84bf32bcc..fc2f86a7ece 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2
@@ -10,13 +10,21 @@ upstream lms-backend {
{% endfor %}
}
+{%- if EDXAPP_ENABLE_RATE_LIMITING -%}
+# Make Zone
+limit_req_zone $cookie_{{ EDXAPP_SESSION_COOKIE_NAME }} zone=cookies:10m rate={{ EDXAPP_COURSES_REQUEST_RATE }};
+{%- endif -%}
+
server {
# LMS configuration file for nginx, templated by ansible
+ # 500 error pages
+ error_page 500 502 504 /server/server-error.html;
+
{% if NGINX_ENABLE_SSL %}
- listen {{EDXAPP_LMS_NGINX_PORT}} {{default_site}};
- listen {{EDXAPP_LMS_SSL_NGINX_PORT}} {{default_site}} ssl;
+ listen {{ EDXAPP_LMS_NGINX_PORT }} {{ default_site }};
+ listen {{ EDXAPP_LMS_SSL_NGINX_PORT }} {{ default_site }} ssl;
ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
@@ -24,10 +32,10 @@ server {
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
{% else %}
- listen {{EDXAPP_LMS_NGINX_PORT}} {{default_site}};
+ listen {{ EDXAPP_LMS_NGINX_PORT }} {{ default_site }};
{% endif %}
- access_log {{ nginx_log_dir }}/access.log;
+ access_log {{ nginx_log_dir }}/access.log {{ NGINX_LOG_FORMAT_NAME }};
error_log {{ nginx_log_dir }}/error.log error;
# CS184 requires uploads of up to 4MB for submitting screenshots.
@@ -57,17 +65,41 @@ server {
{% include "basic-auth.j2" %}
try_files $uri @proxy_to_lms_app;
}
+ # No basic auth for /segmentio/event
+ location /segmentio/event {
+ try_files $uri @proxy_to_lms_app;
+ }
+
+ location /notifier_api {
+ try_files $uri @proxy_to_lms_app;
+ }
# No basic auth security on the github_service_hook url, so that github can use it for cms
location /github_service_hook {
try_files $uri @proxy_to_lms_app;
}
+ # No basic auth security on oath2 endpoint
+ location /oauth2 {
+ try_files $uri @proxy_to_lms_app;
+ }
+
# No basic auth security on the heartbeat url, so that ELB can use it
location /heartbeat {
try_files $uri @proxy_to_lms_app;
}
+ location /courses {
+ {%- if EDXAPP_ENABLE_RATE_LIMITING -%}
+ # Set Limit
+ limit_req zone=cookies burst={{ EDXAPP_COURSE_REQUEST_BURST_RATE }};
+ error_page 503 = /server/rate-limit.html;
+ {%- endif -%}
+
+ {%- include "basic-auth.j2" %}
+ try_files $uri @proxy_to_lms_app;
+ }
+
{% include "robots.j2" %}
{% include "static-files.j2" %}
diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/static-files.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/static-files.j2
index b6e547a0862..3cf4b9e89f3 100644
--- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/static-files.j2
+++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/static-files.j2
@@ -1,3 +1,9 @@
+ # static pages for server status
+ location ~ ^/server/(?P.*) {
+ root /edx/var/nginx/server-static;
+ try_files /$file =404;
+ }
+
location ~ ^/static/(?P.*) {
root {{ edxapp_data_dir }};
try_files /staticfiles/$file /course_static/$file =404;
diff --git a/playbooks/roles/nginx/templates/edx/var/nginx/server-static/server-template.j2 b/playbooks/roles/nginx/templates/edx/var/nginx/server-static/server-template.j2
new file mode 100644
index 00000000000..3593161e21d
--- /dev/null
+++ b/playbooks/roles/nginx/templates/edx/var/nginx/server-static/server-template.j2
@@ -0,0 +1,42 @@
+
+
+
+
+ {{ item.title }}
+
+
+
+
+
+
+
{{ item.heading }}
+
+
{{ item.title }}
+
{{ item.msg }}
+