diff --git a/playbooks/jenkins_data_engineering_new.yml b/playbooks/jenkins_data_engineering_new.yml index 5d7176f8198..afb18708d8f 100644 --- a/playbooks/jenkins_data_engineering_new.yml +++ b/playbooks/jenkins_data_engineering_new.yml @@ -1,18 +1,18 @@ --- -- name: Bootstrap instance(s) - hosts: all - gather_facts: no - become: True - roles: - - python +# - name: Bootstrap instance(s) +# hosts: all +# gather_facts: no +# become: True +# roles: +# - python -- name: Mount EBS - hosts: all - become: True - vars: - volumes: "{{ JENKINS_VOLUMES }}" - roles: - - mount_ebs +# - name: Mount EBS +# hosts: all +# become: True +# vars: +# volumes: "{{ JENKINS_VOLUMES }}" +# roles: +# - mount_ebs - name: Configure instance(s) hosts: all @@ -23,17 +23,17 @@ COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: True COMMON_SECURITY_UPDATES: yes SECURITY_UPGRADE_ON_ANSIBLE: true - ansible_distribution_release: bionic + ansible_distribution_release: focal roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - docker-tools + # - role: aws + # when: COMMON_ENABLE_AWS_ROLE + # - docker-tools - jenkins_data_engineering_new - role: newrelic_infrastructure - when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - tags: - - newreliconly - - role: aws_cloudwatch_agent - tags: - - cloudwatch + # when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + # tags: + # - newreliconly + # - role: aws_cloudwatch_agent + # tags: + # - cloudwatch diff --git a/playbooks/roles/common/meta/main.yml b/playbooks/roles/common/meta/main.yml index 5d6d490e437..5f256ad7233 100644 --- a/playbooks/roles/common/meta/main.yml +++ b/playbooks/roles/common/meta/main.yml @@ -1,8 +1,8 @@ --- dependencies: - common_vars - - server_utils - - role: user - user_info: "{{ COMMON_USER_INFO }}" - - role: security - when: COMMON_SECURITY_UPDATES + # - server_utils + # - role: user + # user_info: "{{ COMMON_USER_INFO }}" + # - role: security + # when: COMMON_SECURITY_UPDATES diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index b406c55a021..1490c76307d 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -209,13 +209,13 @@ src: "{{ item.src }}" owner: root group: root - mode: "{{ item.mode | default(0644) }}" + mode: "{{ item.mode }}" register: config_templates with_items: - - { src: 'edx_rsyslog.j2', dest: '/etc/rsyslog.d/99-edx.conf' } - - { src: 'etc/logrotate.d/hourly/edx_logrotate.j2', dest: '/etc/logrotate.d/hourly/edx-services' } + - { src: 'edx_rsyslog.j2', dest: '/etc/rsyslog.d/99-edx.conf', mode: '0644' } + - { src: 'etc/logrotate.d/hourly/edx_logrotate.j2', dest: '/etc/logrotate.d/hourly/edx-services', mode: '0644' } - { src: 'etc/cron.hourly/logrotate.j2', dest: '/etc/cron.hourly/logrotate', mode: '0555' } - - { src: 'etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2', dest: '/etc/logrotate.d/hourly/tracking.log' } + - { src: 'etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2', dest: '/etc/logrotate.d/hourly/tracking.log', mode: '0644' } - name: Are we in a Docker container shell: echo $(egrep -q 'docker' /proc/self/cgroup && echo 'yes' || echo 'no') diff --git a/playbooks/roles/jenkins_common/meta/main.yml b/playbooks/roles/jenkins_common/meta/main.yml index cc207a01175..89582948f6d 100644 --- a/playbooks/roles/jenkins_common/meta/main.yml +++ b/playbooks/roles/jenkins_common/meta/main.yml @@ -1,21 +1,21 @@ --- dependencies: - - common - - role: nginx - nginx_app_dir: "/etc/nginx" - nginx_log_dir: "/var/log/nginx" - nginx_data_dir: "{{ nginx_app_dir }}" - nginx_conf_dir: "{{ nginx_app_dir }}/conf.d" - nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available" - nginx_sites_enabled_dir: "{{ nginx_app_dir }}/sites-enabled" - nginx_server_static_dir: "{{ nginx_data_dir }}/server-static" - nginx_htpasswd_file: "{{ nginx_app_dir }}/nginx.htpasswd" - nginx_default_sites: "jenkins" - nginx_sites: jenkins - jenkins_nginx_port: "{{ jenkins_common_nginx_port }}" - jenkins_server_name: "{{ JENKINS_SERVER_NAME }}" - jenkins_port: "{{ jenkins_common_port }}" - jenkins_protocol_https: "{{ jenkins_common_protocol_https }}" - tags: jenkins:promote-to-production - - role: oraclejdk - tags: java + # - common + # - role: nginx + # nginx_app_dir: "/etc/nginx" + # nginx_log_dir: "/var/log/nginx" + # nginx_data_dir: "{{ nginx_app_dir }}" + # nginx_conf_dir: "{{ nginx_app_dir }}/conf.d" + # nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available" + # nginx_sites_enabled_dir: "{{ nginx_app_dir }}/sites-enabled" + # nginx_server_static_dir: "{{ nginx_data_dir }}/server-static" + # nginx_htpasswd_file: "{{ nginx_app_dir }}/nginx.htpasswd" + # nginx_default_sites: "jenkins" + # nginx_sites: jenkins + # jenkins_nginx_port: "{{ jenkins_common_nginx_port }}" + # jenkins_server_name: "{{ JENKINS_SERVER_NAME }}" + # jenkins_port: "{{ jenkins_common_port }}" + # jenkins_protocol_https: "{{ jenkins_common_protocol_https }}" + # tags: jenkins:promote-to-production + # - role: oraclejdk + # tags: java diff --git a/playbooks/roles/jenkins_common/tasks/main.yml b/playbooks/roles/jenkins_common/tasks/main.yml index 417193e4c61..4aa6e28b87a 100644 --- a/playbooks/roles/jenkins_common/tasks/main.yml +++ b/playbooks/roles/jenkins_common/tasks/main.yml @@ -1,432 +1,440 @@ --- -- name: Install jenkins specific system packages - apt: - name: '{{ item }}' - state: present - update_cache: yes - with_items: '{{ jenkins_common_debian_pkgs }}' - tags: - - jenkins - - install - - install:system-requirements - -- name: Install jenkins specific snap packages - command: 'snap install {{ item }} --classic' - with_items: '{{ jenkins_common_snap_pkgs }}' - tags: - - jenkins - - install - - install:system-requirements - -# Certain versions of Python are not available as apt-packages, -# depending on your distribution. Use the deadsnakes PPA to have -# access to them -- name: add deadsnakes PPA for newer Python versions - apt_repository: - repo: "ppa:deadsnakes/ppa" - update_cache: yes - tags: - - jenkins - - install - - install:system-requirements - -# Install newer versions of python for testing, but do not set them -# as the default version -- name: Install specific versions of python - apt: - name: '{{ item }}' - state: present - update_cache: yes - with_items: '{{ jenkins_common_python_versions }}' - tags: - - jenkins - - install - - install:system-requirements - -# Install 'dev' packages for each version of python that is installed -- name: Install python dev packages - apt: - name: '{{ item }}-dev' - state: present - update_cache: yes - with_items: '{{ jenkins_common_python_versions }}' - tags: - - jenkins - - install - - install:system-requirements - -- name: Install extra packages required for python3.8 - apt: - name: '{{ item }}' - state: present - update_cache: yes - when: '"python3.8" in jenkins_common_python_versions' - with_list: - - 'python3.8-lib2to3' - - 'python3.8-distutils' - tags: - - jenkins - - install - - install:system-requirements - -- name: Create jenkins group with specified gid - group: - name: '{{ jenkins_common_group }}' - gid: '{{ jenkins_common_group_gid }}' - state: present - when: jenkins_common_group_gid is defined - tags: - - install - - install:system-requirements - -- name: Create jenkins group - group: - name: '{{ jenkins_common_group }}' - state: present - when: jenkins_common_group_gid is not defined or not jenkins_common_group_gid - tags: - - install - - install:system-requirements - -- name: Create the jenkins user with specified uid and add to the group - user: - name: '{{ jenkins_common_user }}' - append: yes - uid: '{{ jenkins_common_user_uid }}' - groups: '{{ jenkins_common_groups }}' - when: jenkins_common_user_uid is defined - tags: - - install - - install:system-requirements - -- name: Create the jenkins user and add to the group - user: - name: '{{ jenkins_common_user }}' - append: yes - groups: '{{ jenkins_common_groups }}' - when: jenkins_common_user_uid is not defined or not jenkins_common_user_uid - tags: - - install - - install:system-requirements - -- name: Create jenkins home and set ownership - file: - path: "{{ jenkins_common_home }}" - state: directory - owner: "{{ jenkins_common_user }}" - group: "{{ jenkins_common_group }}" - mode: 0700 - tags: - - install - - install:system-requirements - -- name: set nofile soft limit for the user jenkins - pam_limits: - domain: "{{ jenkins_common_user }}" - limit_type: soft - limit_item: nofile - value: 4096 - tags: - - install - - install:system-requirements - -- name: set nofile hard limit for the user jenkins - pam_limits: - domain: "{{ jenkins_common_user }}" - limit_type: hard - limit_item: nofile - value: 8096 - tags: - - install - - install:system-requirements - -- name: Create /var/run/jenkins - file: - path: "/var/run/jenkins" - state: directory - owner: "{{ jenkins_common_user }}" - group: "{{ jenkins_common_group }}" - tags: - - install - - install:system-requirements - -- name: Delete any existing jenkins-configuration folders to avoid unwanted configuration - file: - path: '{{ item }}' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - state: absent - with_items: - - '{{ jenkins_common_home }}/init.groovy.d' - - '{{ jenkins_common_home }}/plugins' - - '{{ jenkins_common_home }}/utils' - - '{{ jenkins_common_config_path }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Create necessary folders - file: - path: '{{ item }}' - state: directory - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_items: - - /usr/share/jenkins - - '{{ jenkins_common_home }}/init.groovy.d' - - '{{ jenkins_common_config_path }}' - - '{{ jenkins_common_home }}/utils' - - '{{ jenkins_common_home }}/plugins' - - '{{ jenkins_common_git_home }}' - - /var/log/jenkins - - /var/cache/jenkins - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Repository Signing Key for Jenkins 2.235.3 - apt_key: - url: "https://pkg.jenkins.io/debian-stable/jenkins.io.key" - state: present - tags: - - install - - install:app-requirements - become: yes - -- name: Download Jenkins war file - get_url: - url: '{{ jenkins_common_war_source }}/{{ JENKINS_COMMON_VERSION }}.war' - dest: /usr/share/jenkins/jenkins.war - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - force: yes - tags: - - install - - install:app-requirements - -- name: Add Jenkins systemd configuration - template: - src: "etc/systemd/system/jenkins.service.j2" - dest: "/etc/systemd/system/jenkins.service" - tags: - - install - - install:system-requirements - -- name: Configure logrotate for jenkins application log - template: - src: "etc/logrotate.d/jenkins_log.j2" - dest: "/etc/logrotate.d/jenkins" - tags: - - install - - install:system-requirements - -- name: Add env vars - template: - src: "jenkins-env.sh.j2" - dest: "/etc/profile.d/jenkins-env.sh" - owner: root - group: root - mode: "0755" - tags: - - install - - install:base - -- name: Download jenkins-configuration repo - git: - repo: '{{ jenkins_common_configuration_git_url }}' - dest: '{{ jenkins_common_git_home }}/jenkins-configuration' - version: '{{ jenkins_common_jenkins_configuration_branch }}' - become: true - become_user: '{{ jenkins_common_user }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Run gradle libs - shell: './gradlew libs' - args: - chdir: '{{ jenkins_common_git_home }}/jenkins-configuration' - environment: - UTILS_PATH: '{{ jenkins_common_home }}/utils' - JENKINS_VERSION: '{{ JENKINS_COMMON_VERSION }}' - become: true - become_user: '{{ jenkins_common_user }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy init scripts into init.groovy.d - command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/' - with_items: '{{ jenkins_common_configuration_scripts }}' - become: true - become_user: '{{ jenkins_common_user }}' - register: init_scripts_copied - tags: - - install - - install:base - - install:plugins - -- name: Copy all init scripts other than oauth and security for local dev - command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/' - with_items: '{{ jenkins_common_configuration_scripts }}' - become: true - become_user: '{{ jenkins_common_user }}' - when: 'item != "4configureGHOAuth.groovy" and item != "4configureSecurity.groovy" and init_scripts_copied is not defined' - tags: - - jenkins:local-dev - -- name: Create jenkins config sub folders - file: - path: '{{ item }}' - state: directory - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_items: - - '{{ jenkins_common_config_path }}/credentials' - - '{{ jenkins_common_config_path }}/ec2' - - '{{ jenkins_common_config_path }}/xml' - - '{{ jenkins_common_config_path }}/splunk' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy non plugins template files - template: - src: '{{ role_path }}/templates/config/{{ item }}.yml.j2' - dest: '{{ jenkins_common_config_path }}/{{ item }}.yml' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_items: '{{ jenkins_common_non_plugin_template_files }}' - register: templates_copied - tags: - - install - - install:base - - install:plugins - -- name: For local dev, copy any config files other than oauth and security - template: - src: '{{ role_path }}/templates/config/{{ item }}.yml.j2' - dest: '{{ jenkins_common_config_path }}/{{ item }}.yml' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_items: '{{ jenkins_common_non_plugin_template_files }}' - when: 'item != "github_oauth" and item != "security" and templates_copied is not defined' - tags: - - jenkins:local-dev - -- name: Update Github OAUTH settings when promoting jenkins instance to production - template: - src: '{{ role_path }}/templates/config/security.yml.j2' - dest: '{{ jenkins_common_config_path }}/security.yml' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - when: '"security" in jenkins_common_non_plugin_template_files and templates_copied is not defined' - tags: - - jenkins:promote-to-production - -- name: Copy plugins.yml config file - template: - src: '{{ role_path }}/templates/config/plugins.yml.j2' - dest: '{{jenkins_common_config_path }}/plugins.yml' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy ec2 config files - template: - src: '{{ item }}' - dest: '{{ jenkins_common_config_path }}/ec2/' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_fileglob: - - '{{ role_path }}/files/ec2/*' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy xml config files - template: - src: '{{ item }}' - dest: '{{ jenkins_common_config_path }}/xml/' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_fileglob: - - '{{ jenkins_common_seed_job_source }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy splunk config script - template: - src: '{{ item }}' - dest: '{{ jenkins_common_config_path }}/splunk/' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - with_fileglob: - - '{{ jenkins_common_splunk_file_path }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Run plugins.gradle to download plugins - shell: './gradlew -b plugins.gradle plugins' - args: - chdir: '{{ jenkins_common_git_home }}/jenkins-configuration' - environment: - PLUGIN_OUTPUT_DIR: '{{ jenkins_common_home }}/plugins' - PLUGIN_CONFIG: '{{ jenkins_common_config_path }}/plugins.yml' - become: true - become_user: '{{ jenkins_common_user }}' - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy secret file credentials - copy: - content: "{{ item.content }}" - dest: '{{ jenkins_common_config_path }}/credentials/{{ item.name }}' - with_items: '{{ JENKINS_SECRET_FILES_LIST }}' - no_log: yes - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev - -- name: Copy ec2 key - copy: - content: '{{ JENKINS_EC2_PRIVATE_KEY }}' - dest: '{{ jenkins_common_config_path }}/ec2/id_rsa' - owner: '{{ jenkins_common_user }}' - group: '{{ jenkins_common_group }}' - no_log: yes - tags: - - install - - install:base - - install:plugins - - jenkins:local-dev +# - name: Install jenkins specific system packages +# apt: +# name: '{{ item }}' +# state: present +# update_cache: yes +# with_items: '{{ jenkins_common_debian_pkgs }}' +# tags: +# - jenkins +# - install +# - install:system-requirements + +# - name: Install jenkins specific snap packages +# command: 'snap install {{ item }} --classic' +# with_items: '{{ jenkins_common_snap_pkgs }}' +# tags: +# - jenkins +# - install +# - install:system-requirements + +# # Certain versions of Python are not available as apt-packages, +# # depending on your distribution. Use the deadsnakes PPA to have +# # access to them +# - name: add deadsnakes PPA for newer Python versions +# apt_repository: +# repo: "ppa:deadsnakes/ppa" +# update_cache: yes +# tags: +# - jenkins +# - install +# - install:system-requirements + +# # Install newer versions of python for testing, but do not set them +# # as the default version +# - name: Install specific versions of python +# apt: +# name: '{{ item }}' +# state: present +# update_cache: yes +# with_items: '{{ jenkins_common_python_versions }}' +# tags: +# - jenkins +# - install +# - install:system-requirements + +# # Install 'dev' packages for each version of python that is installed +# - name: Install python dev packages +# apt: +# name: '{{ item }}-dev' +# state: present +# update_cache: yes +# with_items: '{{ jenkins_common_python_versions }}' +# tags: +# - jenkins +# - install +# - install:system-requirements + +# - name: Install extra packages required for python3.8 +# apt: +# name: '{{ item }}' +# state: present +# update_cache: yes +# when: '"python3.8" in jenkins_common_python_versions' +# with_list: +# - 'python3.8-lib2to3' +# - 'python3.8-distutils' +# tags: +# - jenkins +# - install +# - install:system-requirements + +# - name: Create jenkins group with specified gid +# group: +# name: '{{ jenkins_common_group }}' +# gid: '{{ jenkins_common_group_gid }}' +# state: present +# when: jenkins_common_group_gid is defined +# tags: +# - install +# - install:system-requirements + +# - name: Create jenkins group +# group: +# name: '{{ jenkins_common_group }}' +# state: present +# when: jenkins_common_group_gid is not defined or not jenkins_common_group_gid +# tags: +# - install +# - install:system-requirements + +# - name: Create the jenkins user with specified uid and add to the group +# user: +# name: '{{ jenkins_common_user }}' +# append: yes +# uid: '{{ jenkins_common_user_uid }}' +# groups: '{{ jenkins_common_groups }}' +# when: jenkins_common_user_uid is defined +# tags: +# - install +# - install:system-requirements + +# - name: Create the jenkins user and add to the group +# user: +# name: '{{ jenkins_common_user }}' +# append: yes +# groups: '{{ jenkins_common_groups }}' +# when: jenkins_common_user_uid is not defined or not jenkins_common_user_uid +# tags: +# - install +# - install:system-requirements + +# - name: Create jenkins home and set ownership +# file: +# path: "{{ jenkins_common_home }}" +# state: directory +# owner: "{{ jenkins_common_user }}" +# group: "{{ jenkins_common_group }}" +# mode: 0700 +# tags: +# - install +# - install:system-requirements + +# - name: set nofile soft limit for the user jenkins +# pam_limits: +# domain: "{{ jenkins_common_user }}" +# limit_type: soft +# limit_item: nofile +# value: 4096 +# tags: +# - install +# - install:system-requirements + +# - name: set nofile hard limit for the user jenkins +# pam_limits: +# domain: "{{ jenkins_common_user }}" +# limit_type: hard +# limit_item: nofile +# value: 8096 +# tags: +# - install +# - install:system-requirements + +# - name: Create /var/run/jenkins +# file: +# path: "/var/run/jenkins" +# state: directory +# owner: "{{ jenkins_common_user }}" +# group: "{{ jenkins_common_group }}" +# tags: +# - install +# - install:system-requirements + +# - name: Delete any existing jenkins-configuration folders to avoid unwanted configuration +# file: +# path: '{{ item }}' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# state: absent +# with_items: +# - '{{ jenkins_common_home }}/init.groovy.d' +# - '{{ jenkins_common_home }}/plugins' +# - '{{ jenkins_common_home }}/utils' +# - '{{ jenkins_common_config_path }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Create necessary folders +# file: +# path: '{{ item }}' +# state: directory +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_items: +# - /usr/share/jenkins +# - '{{ jenkins_common_home }}/init.groovy.d' +# - '{{ jenkins_common_config_path }}' +# - '{{ jenkins_common_home }}/utils' +# - '{{ jenkins_common_home }}/plugins' +# - '{{ jenkins_common_git_home }}' +# - /var/log/jenkins +# - /var/cache/jenkins +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Repository Signing Key for Jenkins 2.235.3 +# apt_key: +# url: "https://pkg.jenkins.io/debian-stable/jenkins.io.key" +# state: present +# tags: +# - install +# - install:app-requirements +# become: yes + +# # - name: Download Jenkins war file +# # get_url: +# # url: '{{ jenkins_common_war_source }}/{{ JENKINS_COMMON_VERSION }}.war' +# # dest: /usr/share/jenkins/jenkins.war +# # owner: '{{ jenkins_common_user }}' +# # group: '{{ jenkins_common_group }}' +# # force: yes +# # tags: +# # - install +# # - install:app-requirements +# - name: Set ownership and group for Jenkins war file +# file: +# path: /usr/share/jenkins/jenkins.war +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# tags: +# - install +# - install:app-requirements + +# - name: Add Jenkins systemd configuration +# template: +# src: "etc/systemd/system/jenkins.service.j2" +# dest: "/etc/systemd/system/jenkins.service" +# tags: +# - install +# - install:system-requirements + +# - name: Configure logrotate for jenkins application log +# template: +# src: "etc/logrotate.d/jenkins_log.j2" +# dest: "/etc/logrotate.d/jenkins" +# tags: +# - install +# - install:system-requirements + +# - name: Add env vars +# template: +# src: "jenkins-env.sh.j2" +# dest: "/etc/profile.d/jenkins-env.sh" +# owner: root +# group: root +# mode: "0755" +# tags: +# - install +# - install:base + +# - name: Download jenkins-configuration repo +# git: +# repo: '{{ jenkins_common_configuration_git_url }}' +# dest: '{{ jenkins_common_git_home }}/jenkins-configuration' +# version: '{{ jenkins_common_jenkins_configuration_branch }}' +# become: true +# become_user: '{{ jenkins_common_user }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Run gradle libs +# shell: './gradlew libs' +# args: +# chdir: '{{ jenkins_common_git_home }}/jenkins-configuration' +# environment: +# UTILS_PATH: '{{ jenkins_common_home }}/utils' +# JENKINS_VERSION: '{{ JENKINS_COMMON_VERSION }}' +# become: true +# become_user: '{{ jenkins_common_user }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy init scripts into init.groovy.d +# command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/' +# with_items: '{{ jenkins_common_configuration_scripts }}' +# become: true +# become_user: '{{ jenkins_common_user }}' +# register: init_scripts_copied +# tags: +# - install +# - install:base +# - install:plugins + +# - name: Copy all init scripts other than oauth and security for local dev +# command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/' +# with_items: '{{ jenkins_common_configuration_scripts }}' +# become: true +# become_user: '{{ jenkins_common_user }}' +# when: 'item != "4configureGHOAuth.groovy" and item != "4configureSecurity.groovy" and init_scripts_copied is not defined' +# tags: +# - jenkins:local-dev + +# - name: Create jenkins config sub folders +# file: +# path: '{{ item }}' +# state: directory +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_items: +# - '{{ jenkins_common_config_path }}/credentials' +# - '{{ jenkins_common_config_path }}/ec2' +# - '{{ jenkins_common_config_path }}/xml' +# - '{{ jenkins_common_config_path }}/splunk' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy non plugins template files +# template: +# src: '{{ role_path }}/templates/config/{{ item }}.yml.j2' +# dest: '{{ jenkins_common_config_path }}/{{ item }}.yml' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_items: '{{ jenkins_common_non_plugin_template_files }}' +# register: templates_copied +# tags: +# - install +# - install:base +# - install:plugins + +# - name: For local dev, copy any config files other than oauth and security +# template: +# src: '{{ role_path }}/templates/config/{{ item }}.yml.j2' +# dest: '{{ jenkins_common_config_path }}/{{ item }}.yml' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_items: '{{ jenkins_common_non_plugin_template_files }}' +# when: 'item != "github_oauth" and item != "security" and templates_copied is not defined' +# tags: +# - jenkins:local-dev + +# - name: Update Github OAUTH settings when promoting jenkins instance to production +# template: +# src: '{{ role_path }}/templates/config/security.yml.j2' +# dest: '{{ jenkins_common_config_path }}/security.yml' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# when: '"security" in jenkins_common_non_plugin_template_files and templates_copied is not defined' +# tags: +# - jenkins:promote-to-production + +# - name: Copy plugins.yml config file +# template: +# src: '{{ role_path }}/templates/config/plugins.yml.j2' +# dest: '{{jenkins_common_config_path }}/plugins.yml' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy ec2 config files +# template: +# src: '{{ item }}' +# dest: '{{ jenkins_common_config_path }}/ec2/' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_fileglob: +# - '{{ role_path }}/files/ec2/*' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy xml config files +# template: +# src: '{{ item }}' +# dest: '{{ jenkins_common_config_path }}/xml/' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_fileglob: +# - '{{ jenkins_common_seed_job_source }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy splunk config script +# template: +# src: '{{ item }}' +# dest: '{{ jenkins_common_config_path }}/splunk/' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# with_fileglob: +# - '{{ jenkins_common_splunk_file_path }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Run plugins.gradle to download plugins +# shell: './gradlew -b plugins.gradle plugins' +# args: +# chdir: '{{ jenkins_common_git_home }}/jenkins-configuration' +# environment: +# PLUGIN_OUTPUT_DIR: '{{ jenkins_common_home }}/plugins' +# PLUGIN_CONFIG: '{{ jenkins_common_config_path }}/plugins.yml' +# become: true +# become_user: '{{ jenkins_common_user }}' +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy secret file credentials +# copy: +# content: "{{ item.content }}" +# dest: '{{ jenkins_common_config_path }}/credentials/{{ item.name }}' +# with_items: '{{ JENKINS_SECRET_FILES_LIST }}' +# no_log: yes +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev + +# - name: Copy ec2 key +# copy: +# content: '{{ JENKINS_EC2_PRIVATE_KEY }}' +# dest: '{{ jenkins_common_config_path }}/ec2/id_rsa' +# owner: '{{ jenkins_common_user }}' +# group: '{{ jenkins_common_group }}' +# no_log: yes +# tags: +# - install +# - install:base +# - install:plugins +# - jenkins:local-dev - name: Start Jenkins Service systemd: diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index e4373020299..63440eb498d 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -3,7 +3,7 @@ jenkins_home: '/var/lib/jenkins' -JENKINS_SERVER_NAME: 'jenkins-new.analytics.edx.org' +JENKINS_SERVER_NAME: 'jenkins-test.analytics.edx.org' jenkins_user: 'jenkins' jenkins_group: 'jenkins' # the groups that the jenkins user must belong to @@ -28,7 +28,7 @@ de_jenkins_seed_job_source: '{{ role_path }}/../jenkins_data_engineering_new/fil de_jenkins_user_uid: 900 de_jenkins_group_gid: 900 -DE_JENKINS_VERSION: jenkins_2.222.3 +DE_JENKINS_VERSION: jenkins_2.387.3 de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16g -Djenkins.install.runSetupWizard=false' de_jenkins_snap_pkgs: diff --git a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml index 1903d20353f..6f7575ba530 100644 --- a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml @@ -1,6 +1,6 @@ --- dependencies: - - common + # - common - role: jenkins_common jenkins_node_usage: 'NORMAL' JENKINS_COMMON_VERSION: '{{ DE_JENKINS_VERSION }}' diff --git a/playbooks/roles/python/defaults/main.yml b/playbooks/roles/python/defaults/main.yml index a020e493d8b..e00034c83f5 100644 --- a/playbooks/roles/python/defaults/main.yml +++ b/playbooks/roles/python/defaults/main.yml @@ -1,4 +1,6 @@ # Install python2.7 + the /usr/bin/python symlink. python_packages: - - python-minimal + - python + - python3-minimal + # - python-minimal diff --git a/playbooks/roles/python/tasks/main.yml b/playbooks/roles/python/tasks/main.yml index 490687052e6..f8478213a3f 100644 --- a/playbooks/roles/python/tasks/main.yml +++ b/playbooks/roles/python/tasks/main.yml @@ -9,7 +9,8 @@ - name: Update apt-get raw: apt-get update -qq register: python_update_result - until: python_update_result.rc == 0 + # until: python_update_result.rc == 0 + until: python_update_result is succeeded retries: 10 delay: 10 diff --git a/playbooks/roles/user/tasks/main.yml b/playbooks/roles/user/tasks/main.yml index 6c316d67ff1..14faf519a65 100644 --- a/playbooks/roles/user/tasks/main.yml +++ b/playbooks/roles/user/tasks/main.yml @@ -137,15 +137,21 @@ # We skip users in the previous task, and they end up with no content_length when: (USER_FAIL_MISSING_KEYS and 'content' in item and item.content == "") +- name: Print the github + debug: + msg: "{{ item.github }} {{ item.get('state', 'present') }}" + with_items: "{{ user_info }}" + - name: Get github key(s) and update the authorized_keys file authorized_key: user: "{{ item.name }}" + path: /home/{{ item.name }}/.ssh/authorized_keys exclusive: yes key: "https://github.com/{{ item.name }}.keys" when: item.github is defined and item.get('state', 'present') == 'present' register: task_result until: task_result is succeeded - retries: 5 + retries: 1 with_items: "{{ user_info }}" - name: Create ~/.ssh directory for non github users