Merge pull request #98 from oracle-devrel/develop

Automation Toolkit Release v2024.2.2

Dockerfile

@@ -1,56 +1,58 @@
-FROM oraclelinux:7-slim
+FROM oraclelinux:9-slim
 LABEL maintainer="Team at Oracle"
 LABEL description="OCI format to generate CD3 image"
 
 ARG USERNAME=cd3user
 ARG USER_UID=1001
 ARG USER_GID=$USER_UID
 
-RUN yum install sudo -y && groupadd --gid $USER_GID $USERNAME \
-&& useradd --uid $USER_UID --gid $USER_GID -d /$USERNAME -m $USERNAME \
-&& echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
-&& chmod 0440 /etc/sudoers.d/$USERNAME \
-&& mkdir -p /cd3user/tenancies && sudo chown -R $USERNAME:$USERNAME /cd3user/tenancies/ \
-&& yum install -y vim && echo 'alias vi="vim"' >> /etc/bashrc
-
+RUN microdnf install -y sudo && \
+    groupadd --gid $USER_GID $USERNAME && \
+    useradd --uid $USER_UID --gid $USER_GID -d /$USERNAME -m $USERNAME && \
+    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
+    chmod 0440 /etc/sudoers.d/$USERNAME && \
+    mkdir -p /cd3user/tenancies && \
+    chown -R $USERNAME:$USERNAME /cd3user/tenancies/ && \
+    microdnf install -y vim && \
+    microdnf install -y dnf && \
+    echo 'alias vi="vim"' >> /etc/bashrc
 
 USER $USERNAME
 WORKDIR /cd3user/oci_tools/
 COPY cd3_automation_toolkit cd3_automation_toolkit/
 
 WORKDIR /cd3user/
 
-RUN sudo yum install -y oracle-softwarecollection-release-el7 \
-&& sudo chown -R $USERNAME:$USERNAME /cd3user/
-
-RUN sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
-&& bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
-&& sudo chown -R cd3user:cd3user /cd3user/ && sudo yum clean all && sudo rm -rf /var/cache/yum \
-&& sudo chmod -R 740 /cd3user/
+RUN  sudo dnf install -y oraclelinux-release-el9 && \
+sudo chown -R $USERNAME:$USERNAME /cd3user/ && \
+sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
+bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
+sudo chown -R cd3user:cd3user /cd3user/ && \
+sudo dnf clean all && \
+sudo rm -rf /var/cache/dnf && \
+sudo chmod -R 740 /cd3user/
 
 
 ##################################### START INSTALLING JENKINS ###################################
-ARG JENKINS_VERSION=2.401.1
-ARG JENKINS_SHA=600b73eabf797852e39919541b84f7686ff601b97c77b44eb00843eb91c7dd6c
+ARG JENKINS_VERSION=2.444
+ARG JENKINS_SHA=ab093a455fc35951c9b46361002e17cc3ed7c59b0943bbee3a57a363f3370d2e
 ARG JENKINS_PLUGIN_MANAGER_VERSION=2.12.13
 ARG PLUGIN_CLI_URL=https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/${JENKINS_PLUGIN_MANAGER_VERSION}/jenkins-plugin-manager-${JENKINS_PLUGIN_MANAGER_VERSION}.jar
 
 ARG JENKINS_HOME=/cd3user/tenancies/jenkins_home
 ARG JENKINS_INSTALL=/usr/share/jenkins
 ARG REF=/usr/share/jenkins/ref
 
-RUN sudo yum remove java-1.8.0-openjdk-1.8.0.345.b01-1.el7_9.x86_64 \
-&& sudo yum install -y java-11-openjdk  \
-&& sudo yum install -y java-11-openjdk-devel \
-&& sudo yum install unzip -y \
-&& sudo yum install git -y \
-&& sudo mkdir -p ${REF}/init.groovy.d \
-&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
-&& sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war \
-&& echo "${JENKINS_SHA}  ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - \
-&& sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
+RUN sudo microdnf install -y java-21-openjdk && \
+    sudo microdnf install -y java-21-openjdk-devel && \
+    sudo microdnf install git-2.39.3 -y && \
+    sudo mkdir -p ${REF}/init.groovy.d && \
+    sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
+    sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
+    echo "${JENKINS_SHA}  ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - && \
+    sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
 
-ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-11.0.17.0.8-2.el8_6.x86_64
+ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk
 ENV JENKINS_HOME ${JENKINS_HOME}
 ENV JENKINS_INSTALL ${JENKINS_INSTALL}
 ENV REF ${REF}
@@ -65,7 +67,6 @@ COPY --chown=cd3user:cd3user jenkins_install ${JENKINS_INSTALL}/
 COPY --chown=cd3user:cd3user jenkins_install/init/*.groovy ${REF}/init.groovy.d/
 COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
 
-
-RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt \
-&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
-&& sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh
+RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt && \
+    sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
+    sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh

OCIWorkVMStack/data_sources.tf

@@ -19,16 +19,17 @@ data "oci_identity_compartment" "compartment" {
 
 data "oci_core_images" "oracle_linux" {
   compartment_id   = var.tenancy_ocid
-  operating_system = "Oracle Linux"
-  shape            = var.instance_shape
+  #operating_system = "Oracle Linux"
+  #shape            = var.instance_shape
+  display_name     = var.instance_os_version
   sort_by          = "TIMECREATED"
   sort_order       = "DESC"
   state            = "AVAILABLE"
 
   # filter restricts to OL
-  filter {
-    name   = "operating_system_version"
-    values = ["${local.os_version}"]
-    regex  = false
-  }
+  #filter {
+  #  name   = "operating_system_version"
+  #  values = ["${local.os_version}"]
+  #  regex  = false
+  #}
 }

OCIWorkVMStack/locals.tf

@@ -19,7 +19,7 @@ locals {
   listing_resource_id      = var.mp_listing_resource_id
   listing_resource_version = var.mp_listing_resource_version
 
-  os_version = var.instance_os_version == "Oracle-Linux-9" ? 9 : (var.instance_os_version == "Oracle-Linux-8" ? 8 : 7.9)
+  #os_version = var.instance_os_version == "Oracle-Linux-9" ? 9 : (var.instance_os_version == "Oracle-Linux-8" ? 8 : 7.9)
 
   instance_image_ocid = data.oci_core_images.oracle_linux.images[0].id
 

OCIWorkVMStack/schema.yaml

@@ -127,8 +127,10 @@ variables:
     description: Oracle Linux image OCID for VM provisioning
     type: enum
     enum:
-    - "Oracle-Linux-7"
-    default: "Oracle-Linux-7"
+    - "Oracle-Linux-7.9-2024.02.26-0"
+    - "Oracle-Linux-8.9-2024.02.26-0"
+    - "Oracle-Linux-9.3-2024.02.26-0"
+    default: "Oracle-Linux-9.3-2024.02.26-0"
     #pattern: '^ocid1\.([a-z0-9_-]{1,32})\.([a-z0-9_-]{1,15})\.([a-z0-9]{0,24})\.([a-z0-9]{60})$'
     required: true
   instance_shape:
@@ -429,4 +431,4 @@ variables:
   mp_listing_resource_version:
     type: string
   tenancy_ocid:
-    type: string
+    type: string

OCIWorkVMStack/scripts/installToolkit.sh

@@ -67,7 +67,7 @@ echo "********************************************************" >> $logfile 2>&1
 echo "########################################################" >> $logfile 2>&1
 echo "Downloading CD3 Automation Toolkit Code from Github " >> $logfile 2>&1
 echo "########################################################" >> $logfile 2>&1
-sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git -b develop $toolkit_dir >> $logfile 2>&1
+sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git -b develop  $toolkit_dir >> $logfile 2>&1
 stop_exec
 sudo ls -la /tmp/githubCode >> $logfile 2>&1
 echo "Downloading CD3 Automation Toolkit Code from Github completed successfully" >> $logfile 2>&1

README.md

@@ -7,7 +7,7 @@
 
 <br>
 
-  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.2.1) &nbsp;&nbsp;•&nbsp; &nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/excel-templates) &nbsp;&nbsp;•&nbsp; &nbsp; [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp; &nbsp;•&nbsp;&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;&nbsp;•&nbsp;&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/tutorials/) &nbsp;&nbsp;•&nbsp;&nbsp;[OCI CD3-Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)
+  [What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.2.2) &nbsp;&nbsp;•&nbsp; &nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/excel-templates) &nbsp;&nbsp;•&nbsp; &nbsp; [CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp; &nbsp;•&nbsp;&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;&nbsp;•&nbsp;&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/tutorials/) &nbsp;&nbsp;•&nbsp;&nbsp;[OCI CD3-Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724)
 
 <br>
 

cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py

@@ -216,23 +216,23 @@ def create_terraform_tags(inputfile, outdir, service_dir, prefix, ct):
                             if default_value != "" and str(default_value).lower() != "nan":
                                 if '$' in default_value and default_value.count('$') == 1:
                                     default_value = str(default_value).strip().replace('$','$$')
-                                #is_required = 'false' #Uncomment this line if needed
-                                columnvalue = key_tf_name+"="+default_compartment+"="+default_value#+"="+is_required #Uncomment this if needed
+                                is_required = 'false' #Uncomment this line if needed
+                                columnvalue = key_tf_name+"="+default_compartment+"="+default_value+"="+is_required #Uncomment this if needed
                                 if columnvalue not in default_tags:
                                     default_tags.append(columnvalue)
                             else:
                                 if default_value == '' or default_value.strip().lower() == 'nan':
                                     if str(df.loc[i,'Validator']).strip() != '' and  str(df.loc[i,'Validator']).strip().lower() != 'nan' and str(df.loc[i,'Validator']).strip() != []:
-                                        #is_required_updated = 'true' #Uncomment this if needed
+                                        is_required_updated = 'true' #Uncomment this if needed
                                         default_value = values_list[0]
-                                        columnvalue = key_tf_name+"="+default_compartment+"="+default_value#+"="+is_required_updated #Uncomment this if needed
+                                        columnvalue = key_tf_name+"="+default_compartment+"="+default_value+"="+is_required_updated #Uncomment this if needed
                                         if columnvalue not in default_tags:
                                             default_tags.append(columnvalue)
                                     else:
                                         if str(df.loc[i, 'Validator']).strip() == '' or  str(df.loc[i, 'Validator']).strip().lower() == 'nan':
-                                            #is_required_updated = 'true' #Uncomment this if needed
+                                            is_required_updated = 'true' #Uncomment this if needed
                                             default_value = '-'
-                                            columnvalue = key_tf_name+"="+default_compartment+"="+default_value#+"="+is_required_updated #Uncomment this if needed
+                                            columnvalue = key_tf_name+"="+default_compartment+"="+default_value+"="+is_required_updated #Uncomment this if needed
                                             if columnvalue not in default_tags:
                                                 default_tags.append(columnvalue)
 

cd3_automation_toolkit/Governance/Tagging/export_tags_nonGreenField.py

@@ -149,7 +149,10 @@ def export_tags_nongreenfield(inputfile, outdir, service_dir, config, signer, ct
             if tag_defaults.data != []:
                 for tag_default in tag_defaults.data:
                     if tag_default.tag_definition_name != '(deleted tag definition)':
-                        add_values_in_dict(tag_default_comps_map, tag_default.tag_definition_id+"="+tag_default.tag_definition_name, [ntk_compartment_name+"="+tag_default.value])
+                        my_val=tag_default.value
+                        if tag_default.is_required==True:
+                            my_val=""
+                        add_values_in_dict(tag_default_comps_map, tag_default.tag_definition_id+"="+tag_default.tag_definition_name, [ntk_compartment_name+"="+my_val])
                     defaultcomp_to_tagid_map.update({ commonTools.check_tf_variable(str(tag_default.tag_definition_name).replace('\\','\\\\'))+"-"+commonTools.check_tf_variable(ntk_compartment_name) : tag_default.id })
 
     comp_ocid_done = []

cd3_automation_toolkit/Governance/Tagging/templates/tags-defaults-template

@@ -22,7 +22,7 @@ tag_defaults = {
             compartment_id = "{{ tags.split('=')[1] }}"
             value = "{{ tags.split('=')[2] }}"
 
-            {# is_required =  {{ tags.split('=')[3] }} #} {# Uncomment this line if needed #}
+            is_required =  {{ tags.split('=')[3] }}
             },
     {% endfor %}
 

cd3_automation_toolkit/Identity/Compartments/create_terraform_compartments.py

@@ -243,7 +243,7 @@ def travel(parent, keys, values, c):
         oname[reg].close()
         print(outfile[reg] + " for Compartments has been created for region " + reg)
 
-        fetch_comp_file = f'{outdir}/fetchcompinfo.safe'
+        fetch_comp_file = f'{outdir}/.safe/fetchcompinfo.safe'
         with open(fetch_comp_file, 'w') as f:
             f.write('run_fetch_script=1')
         f.close()

cd3_automation_toolkit/ManagementServices/EventsAndNotifications/export_events_notifications_nonGreenField.py

@@ -118,15 +118,16 @@ def print_events(values_for_column_events, region, ntk_compartment_name, event,
                    data = str(condition["data"])
                else:
                    data = "{}"
-               for val in condition["eventType"]:
-                 if "oraclecloud" in val:
-                   service = val.split("com.oraclecloud.")[1]
-                 elif "oracle" in val:
-                   service = val.split("com.oracle.")[1]
-                 event_prod = service.split('.', 1)[0]
-                 event_res = service.split('.', 1)[1]
-                 if ( action_name != "" ):
-                     events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
+               if "eventType" in condition:
+                   for val in condition["eventType"]:
+                     if "oraclecloud" in val:
+                       service = val.split("com.oraclecloud.")[1]
+                     elif "oracle" in val:
+                       service = val.split("com.oracle.")[1]
+                     event_prod = service.split('.', 1)[0]
+                     event_res = service.split('.', 1)[1]
+                     if ( action_name != "" ):
+                         events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
           if ( i > 0 and action_name != ""):
              events_rows(values_for_column_events, region, ntk_compartment_name, event_name, event_desc, action_type, action_is_enabled, action_description, event_prod, event_res,data,  event_is_enabled, action_name, event, event_info)
           i = i + 1

cd3_automation_toolkit/ManagementServices/Monitoring/create_terraform_alarms.py

@@ -161,7 +161,7 @@ def create_terraform_alarms(inputfile, outdir, service_dir, prefix, ct):
 
 
         # Write all info to TF string
-        tfStr[region]=tfStr[region][:-1] +alarms_template.render(tempStr)
+        tfStr[region]=tfStr[region][:-2] +alarms_template.render(tempStr)
 
     # Write to output
     for reg in ct.all_regions:

cd3_automation_toolkit/ManagementServices/Monitoring/templates/alarms-template

@@ -22,7 +22,13 @@ alarms = {
         query = "{{ query }}"
         severity = "{{ severity }}"
         {% if body and body != "" %}
+        {% if '\n' not in body %}
         body = "{{ body }}"
+        {% else %}
+        body = <<-EOF
+        {{ body }}
+        EOF
+        {% endif %}
         {% endif %}
 
         {% if message_format and message_format != "" %}

cd3_automation_toolkit/Network/BaseNetwork/create_all_tf_objects.py

@@ -36,10 +36,11 @@ def create_all_tf_objects(inputfile, outdir, service_dir,prefix, ct, non_gf_tena
     with section('Process DRGs tab for DRG Route Tables and Route Distribution creation'):
         create_terraform_drg_route(inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy, network_connectivity_in_setupoci, modify_network)
 
+    #Create Workflow
     if non_gf_tenancy == False:
         with section('Process Subnets tab for Routes creation'):
             create_terraform_route(inputfile, outdir, service_dir_network, prefix, ct, non_gf_tenancy, network_vlan_in_setupoci, modify_network)
-
+    # Create Workflow
     if non_gf_tenancy == False:
         with section('Process Subnets for Seclists creation'):
             create_terraform_seclist(inputfile, outdir, service_dir_network, prefix, ct, modify_network)