Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement namespace filtering, to support multiple installations of a… #27

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

implement namespace filtering, to support multiple installations of a… #27

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 27 additions & 17 deletions generate_postmortem.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ for switch in $@; do
echo -e "Available switches:"
echo -e "--cp4i Specify if API Connect is deployed in a CloudPak 4i environment."
echo -e "--extra-namespaces: Extra namespaces separated with commas. Example: --extra-namespaces=dev1,dev2,dev3"
echo -e "--filter-namespaces: Filter helm installations on namespaces matching this string. Example: --filter-namespaces=utst"
echo -e "--log-limit: Set the number of lines to collect from each pod logs."
echo -e "--ova: Only set if running inside an OVA deployment."
echo -e "--pull-appliance-logs: Call [apic logs] command then package into archive file."
Expand Down Expand Up @@ -77,6 +78,9 @@ for switch in $@; do
extra_namespaces=`echo "${switch}" | cut -d'=' -f2 | tr ',' ' '`
NAMESPACE_LIST="kube-system ${extra_namespaces}"
;;
"--filter-namespaces"*)
filter_namespaces=`echo "${switch}" | cut -d'=' -f2`
;;
"--pull-appliance-logs")
PULL_APPLIANCE_LOGS=1
;;
Expand All @@ -87,6 +91,7 @@ for switch in $@; do
NO_HISTORY=1
;;
*)
echo "got: ${switch}"
if [[ -z "$DEBUG_SET" ]]; then
set +e
fi
Expand Down Expand Up @@ -350,7 +355,12 @@ SUBSYS_GATEWAY=""
SUBSYS_INGRESS=""

if [[ $CP4I -ne 1 ]]; then
OUTPUT=`helm ls -a 2>/dev/null`
if [ -v filter_namespaces ] ; then
echo "Filtering namepace list on ${filter_namespaces}."
OUTPUT=`helm ls -a 2>/dev/null | grep ${filter_namespaces}`
else
OUTPUT=`helm ls -a 2>/dev/null`
fi

if [[ ${#OUTPUT} -eq 0 ]]; then
warning1="WARNING! Helm is not reporting any deployments, this indicates there is problem communicating with the [tiller] pod."
Expand Down Expand Up @@ -396,7 +406,7 @@ if [[ $CP4I -ne 1 ]]; then
namespace=""

case $chart in
*"apic-analytics"*)
*"apic-analytics"*)
namespace=$ns
SUBSYS_ANALYTICS+=" $release"
;;
Expand Down Expand Up @@ -461,7 +471,7 @@ else
release=`echo "${pod}" | awk -F'-' '{print $1}'`

case $pod in
*"apim-v2"*)
*"apim-v2"*)
SUBSYS_MANAGER+=" ${release}"
;;
*"analytics"*)
Expand Down Expand Up @@ -594,8 +604,8 @@ if [[ $? -eq 0 && ${#OUTPUT} -gt 0 ]]; then
echo -e "${warning1}${name}${warning2}" >> "${K8S_DATA}/warnings.out"
fi
fi


done <<< "$OUTPUT"

if [[ $OUTPUT_METRICS -eq 0 ]]; then
Expand Down Expand Up @@ -669,9 +679,9 @@ if [[ $PERFORMANCE_CHECK -eq 1 ]]; then
ETCD_CA_FILE=`kubectl describe pod -n kube-system ${ETCD_POD} | grep "\--trusted-ca-file" | cut -f2 -d"=" 2>/dev/null`
ETCD_CERT_FILE=`kubectl describe pod -n kube-system ${ETCD_POD} | grep "\--cert-file" | cut -f2 -d"=" 2>/dev/null`
ETCD_KEY_FILE=`kubectl describe pod -n kube-system ${ETCD_POD} | grep "\--key-file" | cut -f2 -d"=" 2>/dev/null`

OUTPUT=`kubectl exec -n kube-system ${ETCD_POD} -- sh -c "export ETCDCTL_API=3; etcdctl member list --cacert=${ETCD_CA_FILE} --cert=${ETCD_CERT_FILE} --key=${ETCD_KEY_FILE} 2>/dev/null"`

# parsing endpoints from etcd member list
ENDPOINTS=''
while read line; do
Expand Down Expand Up @@ -724,7 +734,7 @@ for NAMESPACE in $NAMESPACE_LIST; do

K8S_NAMESPACES_JOB_DATA="${K8S_NAMESPACES_SPECIFIC}/jobs"
K8S_NAMESPACES_JOB_DESCRIBE_DATA="${K8S_NAMESPACES_JOB_DATA}/describe"

K8S_NAMESPACES_POD_DATA="${K8S_NAMESPACES_SPECIFIC}/pods"
K8S_NAMESPACES_POD_DESCRIBE_DATA="${K8S_NAMESPACES_POD_DATA}/describe"
K8S_NAMESPACES_POD_DIAGNOSTIC_DATA="${K8S_NAMESPACES_POD_DATA}/diagnostic"
Expand Down Expand Up @@ -792,7 +802,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
mkdir -p $K8S_NAMESPACES_STS_DESCRIBE_DATA

if [[ $DIAG_MANAGER -eq 1 ]]; then
mkdir -p "${K8S_NAMESPACES_POD_DIAGNOSTIC_DATA}/manager"
mkdir -p "${K8S_NAMESPACES_POD_DIAGNOSTIC_DATA}/manager"
fi

#for icp4i platform, grab equivalent "apiconnect-up.yml"
Expand Down Expand Up @@ -864,7 +874,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
DOWNLOAD_RESULT=`curl --write-out %{http_code} -s -o ${file_path} https://raw.githubusercontent.com/ibm-apiconnect/v2018-postmortem/master/identifyServicesState.js`
else
DOWNLOAD_RESULT=200
fi
fi
if [[ $DOWNLOAD_RESULT -eq 200 ]]; then
cat "${file_path}" | kubectl exec -n $NAMESPACE -it $pod -- node &>"${K8S_NAMESPACES_POD_DIAGNOSTIC_DATA}/manager/${pod}-identifyServicesState.out"
[ $? -eq 0 ] || rm -fr "${K8S_NAMESPACES_POD_DIAGNOSTIC_DATA}/manager/${pod}-identifyServicesState.out"
Expand Down Expand Up @@ -995,7 +1005,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
*"calico"*|*"flannel"*) SUBFOLDER="networking";;
*"coredns"*) SUBFOLDER="coredns";;
*"etcd"*) SUBFOLDER="etcd";;
*"ingress"*)
*"ingress"*)
IS_INGRESS=1
SUBFOLDER="ingress"
;;
Expand Down Expand Up @@ -1035,7 +1045,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
DESCRIBE_TARGET_PATH="${K8S_NAMESPACES_POD_DESCRIBE_DATA}/${SUBFOLDER}"
LOG_TARGET_PATH="${K8S_NAMESPACES_POD_LOG_DATA}/${SUBFOLDER}";;
esac

#make sure directories exist
if [[ ! -d "$DESCRIBE_TARGET_PATH" ]]; then
mkdir -p $DESCRIBE_TARGET_PATH
Expand All @@ -1048,7 +1058,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
if [[ ( "${pod}" == *"apim-v2"* ) || ( "${pod}" == *"analytics-client"* ) || ( "${pod}" == *"-apic-portal-www"* ) ]]; then
PERFORM_NSLOOKUP=1
fi

if [[ $PERFORM_NSLOOKUP -eq 1 ]]; then
#grab ingress or routes
ingress_list=`kubectl get ingress -n $NAMESPACE 2>/dev/null`
Expand Down Expand Up @@ -1108,7 +1118,7 @@ for NAMESPACE in $NAMESPACE_LIST; do

#only proceed with error report if response status code is 200
if [[ $response -eq 200 ]]; then

#pull error report
echo -e "Pausing for error report to generate..."
sleep $ERROR_REPORT_SLEEP_TIMEOUT
Expand Down Expand Up @@ -1227,7 +1237,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
"db")
mkdir -p $PORTAL_DIAGNOSTIC_DATA
OUTPUT1=`kubectl exec -n $NAMESPACE -c $container $pod -- bash -ic "mysqldump portal" 2>"/dev/null"`
echo "$OUTPUT1" >"${PORTAL_DIAGNOSTIC_DATA}/portal.dump"
echo "$OUTPUT1" >"${PORTAL_DIAGNOSTIC_DATA}/portal.dump"
OUTPUT1=`kubectl exec -n $NAMESPACE -c $container $pod -- bash -ic "ls -lRAi --author --full-time" 2>"/dev/null"`
echo "$OUTPUT1" >"${PORTAL_DIAGNOSTIC_DATA}/listing-all.out"
OUTPUT1=`kubectl exec -n $NAMESPACE -c $container $pod -- bash -ic "ps -efHww --sort=-pcpu" 2>"/dev/null"`
Expand Down Expand Up @@ -1307,7 +1317,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
else
rm -fr $K8S_NAMESPACES_ROLEBINDING_DATA
fi

#grab role service account data
OUTPUT=`kubectl get sa -n $NAMESPACE 2>/dev/null`
if [[ $? -eq 0 && ${#OUTPUT} -gt 0 ]]; then
Expand All @@ -1327,7 +1337,7 @@ for NAMESPACE in $NAMESPACE_LIST; do
echo "$OUTPUT" > "${K8S_NAMESPACES_SERVICE_DATA}/services.out"
while read line; do
svc=`echo "$line" | cut -d' ' -f1`

kubectl describe svc $svc -n $NAMESPACE &>"${K8S_NAMESPACES_SERVICE_DESCRIBE_DATA}/${svc}.out"
[ $? -eq 0 ] || rm -f "${K8S_NAMESPACES_SERVICE_DESCRIBE_DATA}/${svc}.out"

Expand Down