DO280.md

Red Hat OpenShift Admin I (v3.9) DO280/EX280

Fri Dec 27 10:28:55 UTC 2019

1. Installation - Ansible inventory file & vars

[workstations]
[nfs]
[masters]
[etcd]
[nodes]
  openshift_node_labels                           # e.g. "{'region':'infra', 'node-role.kubernetes.io/compute':'true'}"
[OSEv3:children]
[nodes:vars]
  # pre-installation vars
  registry_local                                  # registry.lab.example.com
  use_overlay_driver                              # true
  insecure_registry                               # false
  run_docker_offline                              # true
  docker_storage_device                           # /dev/vdb
[OSEv3:vars]
  # general vars
  openshift_deployment_type                       # openshift-enterprise
  openshift_release                               # v3.9
  openshift_image_tag                             # v3.9.14
  openshift_disable_check                         # disk_availability,docker_storage,memory_availability
  # networking
  os_firewall_use_firewalld                       # true
  openshift_master_api_port                       # 443
  openshift_master_console_port                   # 443
  openshift_master_default_subdomain              # apps.lab.example.com
  # authentication
  openshift_master_identity_providers             # [{'name':'htpasswd_auth', 'login':'true', 'challenge':'true',
    'kind':'HTPasswdPasswordIdentityProvider','filename':'/etc/origin/master/htpasswd'}]
  openshift_master_htpasswd_users                 # {'user':'<<HASH>>'}
                                                  # openssl passwd -apr1 <PASSWORD> or htpasswd -nbm <USER> <PASSWORD>
  # nfs
  openshift_enable_unsupported_configurations     # true
  openshift_hosted_registry_storage_kind          # nfs
  openshift_hosted_registry_storage_access_modes  # ReadWriteMany
  openshift_hosted_registry_storage_nfs_directory # /exports
  openshift_hosted_registry_storage_nfs_options   # "*(rw,root_squash)"
  openshift_hosted_registry_storage_volume_name   # registry
  openshift_hosted_registry_storage_volume_size   # 40Gi
  # etcd
  openshift_hosted_etcd_storage_kind              # nfs
  openshift_hosted_etcd_storage_access_modes      # ["ReadWriteOnce"]
  openshift_hosted_etcd_storage_nfs_directory     # /exports
  openshift_hosted_etcd_storage_nfs_options       # "*(rw,root_squash,sync,no_wdelay)"
  openshift_hosted_etcd_storage_volume_name       # etcd-vol2
  openshift_hosted_etcd_storage_volume_size       # 1G
  openshift_hosted_etcd_storage_labels            # {'storage':'etcd'}
  # disconnected installation
  oreg_url                                        # registry.lab.example.com/openshift3/ose-${component}:${version}
  openshift_examples_modify_imagestreams          # true
  openshift_docker_additional_registries          # registry.lab.example.com
  openshift_docker_blocked_registries             # registry.lab.example.com,docker.io
  # image prefixes
  openshift_web_console_prefix                    # registry.lab.example.com/openshift3/ose-
  openshift_cockpit_deployer_prefix               # 'registry.lab.example.com/openshift3'
  openshift_service_catalog_image_prefix          # registry.lab.example.com/openshift3/ose-
  openshift_service_broker_prefix                 # registry.lab.example.com/openshift3/ose-
  openshift_service_broker_image_prefix           # registry.lab.example.com/openshift3/ose-
  openshift_service_broker_etcd_image_prefix      # registry.lab.example.com/rhel7
  # metrics
  openshift_metrics_install_metrics               # true

2. Installation process

sudo yum install atomic-openshift-utils
# Prerequisites - FROM THE DIR WITH 'ansible.cfg'!
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml
# Deploy - FROM THE DIR WITH 'ansible.cfg'!
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deplpoy_cluster.yml

3. Post-installation process

oc login -u <USER> -p <PASSWORD> --insecure-skip-tls-verify=true
oc get nodes --show labels
ssh master.lab.example.com
sudo -i
oc adm policy add-cluster-role-to-user cluster-admin <USER>
oc explain

4. Creating a route

a/ Generate private key

openssl genrsa -out <hello.apps.lab.example.com.key> 2048

b/ Generate CSR (request)

openssl req -new -key <hello.apps.lab.example.com.key> -out <hello.apps.lab.example.com.csr> -subj "/C=US/ST=NC/L=Raileigh/O=RedHat/OU=RH/CN=hello.apps.lab.example.com"

c/ Generate certificate

openssl x509 -req -days 365 -in <hello.apps.lab.example.com.csr> -signkey <hello.apps.lab.example.com.key> -out <hello.apps.lab.example.com.crt>

d/ Create secure edge-terminated route

oc create route edge --service=hello --hostname=hello.apps.lab.example.com --key=hello.apps.lab.example.com --cert=hello.apps.lab.example.com.crt
oc types
oc get routes
oc get route/hello -o yaml
oc get pods -o wide
ssh node1 curl -vvv http://<IP>:8080              # IP from the previous command

# Troubleshooting:
oc describe svc hello-openshift [-n <NAMESPACE>]
oc describe pod <hello-openshift-1-abcd>
oc edit svc hello-openshift
oc edit route hello-openshift

5. ImageStreams

oc new-app --name=hello -i php:5.4 http://services/lab/example.com/php-helloworld # -i = imagestream + git repository
oc describe is php -n openshift
oc get pods -o wide
oc logs hello-1-build
oc get events
ssh root@master oc get nodes
ssh root@node1 systemctl status atomic-openshift-node
ssh root@node1 systemctl status docker
oc describe is

6. Common problems

oc delete all -l app=<node-hello>
oc get all
oc describe pod <hello-1-deploy>
oc get events --sort-by='.metadata.creationTimestamp'
oc get dc <hello> -o yaml
sudo vi /etc/sysconfig/docker
oc rollout latest hellp
oc logs <hello-2-abcd>
oc expose service --hostname=hello.apps.lab.example.com <node-hello>
oc debug pod <PODNAME>

7. Secrets

oc create secret generic <mysql> --from-literal='database-user'='mysql' --from-literal='database-password'='r3dh4t' --from-literal='database-root-password'='redhat'
oc get secret <mysql> -o yaml
oc new-app --file=mysql.yml
oc port-forward <pod> <local>:<on the pod>        # oc port-forward mysql-1-abcd 3306:3306

8. User accounts, access

ssh root@master htpasswd /etc/origin/master/htpasswd <USER>

a/ Remove capability to create projects for all regular users

oc login -u <admin> -p <redhat> <master>
oc adm policy remove-cluster-role-from-group self-provisioner system:authenticated system:authenticated:oauth

b/ Associate user with secure project

oc login -u <admin> -p <redhat>
oc new-project <secure>
oc project <secure>                               # you don't have to do this, if you then specify -n (last command)
oc policy add-role-to-user edit <user>
oc policy add-role-to-user edit <user> -n <secure># you don't have to do this, if you switched to the namespace already

c/ Pass environment variable to the new app

oc new-app --name=phpmyadmin --docker-image=registry.lab.example.com/phpmyadmin:4.7 -e PMA_HOST=mysql.secure-review.svc.cluster.local

d/ Failed deployment because of the default security

Enable container to run with root privileges:

oc login -u <admin> -p <redhat>
oc create serviceaccount <phpmyadmin-account>
oc adm policy add-scc-to-user anyuid -z <phpmyadmin-account>

e/ Use & update deployment with the new service account

oc edit dc/phpmyadmin # or this command: oc patch dc/phpmyadmin --patch '{"spec":{"template":{"spec":{"serviceAccountName":"<phpmyadmin-account>"}}}}'

JSON representation of the above:

{
  "spec": {
    "template": {
      "spec": {
        "serviceAccountName": "<phpmyadmin-account>"
      }
    }
  }
}

9. Persistent volume

cat mysqldb-volume.yml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysqldb-volume
spec:
  capacity:
    storage: 3Gi
  accessModes:
    - ReadWriteMany
  nfs:
    path: /var/export/dbvol
    server: services.lab.example.com
  persistentVolumeReclaimPolicy: Recycle

oc create -f <mysqldb-volume.yml>
oc get pv
oc status -v
oc describe pod <mysqldb>
oc set volume dc/<mysqldb> --add --overwrite --name=<mysqldb-volume-1> -t pvc --claim-name=<mysqldb-pvclaim> --claim-size=<3Gi> --claim-mode=<'ReadWriteMany'>
oc get pvc

# Important knowledge about PV/PVC:
  - PV doesn't have a namespace
  - Allocated capacity of PVC may be bigger than requested capacity, imagine a scenario:
    1. Create 'review-pv' PV of 3Gi
    2. Create a new app from template with PVC called 'mysql-pvc' of 1Gi with 'review-pv' selector (step 1.)
    3. In the template, there is "container" in the "DeploymentConfig" using "volumeMounts" with the name of
       'mysql-data' mounting it to '/var/lib/mysql/data'
    4. In the template, there is "volumes" object 'mysql-data' using "persistenVolumeClaim" with "claimName" of mysql-pvc
    - What happens is following:
      - 'mysql-pvc' is bound to volume 'review-pv'
      - it has requested capacity of 1GiB, but was allocated 3GiB
      - if the selector in PVC is not specified, it will automatically find the closest one

10. Controlling scheduling & scaling

# Scaling:
oc new-app -o yaml -i php:7.0 http://registry.lab.example.com/scaling > scaling.yml
oc describe dc <scaling> | grep 'Replicas'
oc scale --replicas=5 dc <scaling>

oc get nodes -L region
oc label node <node2.lab.example.com> region=<apps> --overwrite
oc get dc/hello -o yaml > <hello.yml>

cat hello.yml

nodeSelector:
  region: apps

oc apply -f <hello.yml> oc label node node1.lab.example.com region=apps --overwrite

a/ Disable scheduling on node2

oc adm manage-node --schedulable=false <node2.lab.example.com>

b/ Delete/drain all pods on node2

oc adm drain <node2.lab.example.com> --delete-local-data

c/ Load Docker image locally

docker load -i <phpmyadmin-latest.tar>

d/ Tag local image ID

docker tag <123abcdef> <docker-registry-default.apps.lab.example.com/phpmyadmin:4> docker images

e/ Login to OpenShift internal image registry

TOKEN=$(oc whoami -t)

docker login -n developer -p ${TOKEN} docker-registry-default.apps.lab.example.com

# Certificate signed by unknown authority:
scp registry.crt root@master:/etc/origin/master/registry.crt
/etc/pki/ca-trust/source/anchors/docker-registry-default.apps.lab.example.com.crt
update-ca-trust
systemctl restart docker
<<RUN DOCKER LOGIN AGAIN>>>

11. Metrics subsystem

a/ Verify images required by metrics

docker-registry-cli <registry.lab.example.com> search <metrics-cassandra> ssl

# Output:
openshift3/ose-metrics-hawkular-metrics:v3.9
openshift3/ose-metrics-heapster:v3.9
openshift3/ose-metrics-cassandra:v3.9
openshift3/ose-metrics-recycler:v3.9

b/ Check NFS

ssh root@services cat /etc/exports.d/openshift-ansible.exports

c/ Create PV for NFS share

cat metrics-pv.yml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: metrics
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce                               # Must have this!
  nfs:
    path: /exports/metrics
    server: services.lab.example.com
  persistenVolumeReclaimPolicy: Recycle

oc get pv

d/ Add to Ansible inventory file

[OSEv3:vars]
  openshift_metrics_install_metrics               # true
  openshift_metrics_image_prefix                  # registry.lab.example.com/openshift3/ose-
  openshift_metrics_image_version                 # v3.9
  openshift_metrics_heapster_request_memory       # 300M
  openshift_metrics_hawkular_request_memory       # 750M
  openshift_metrics_cassandra_request_memory      # 750M
  openshift_metrics_cassandra_storage_type        # pv
  openshift_metrics_cassandra_pvc_size            # 5Gi
  openshift_metrics_cassandra_pvc_prefix          # metrics

e/ Run Ansible, verify if it's OK

oc get pvc -n openshift-infra
oc get pod -n openshift-infra
oc adm diagnostics MetricsApiProxy

f/ Top command as admin

oc adm top node --heapster-namespace=openshift-infra --heapster-scheme=https

12. Limits

oc describe node <node1.lab.example.com>
oc describe node <node2.lab.example.com>
# Look for allocated resources (| grep -A 4 Allocated)
# After you deploy new app, allocated resources do NOT change

cat limits.yml

apiVersion: v1
kind: LimitRange
metadata:
  name: project-limits
spec:
  limits:
    - type: container
      default:
        cpu: 250m

oc describe limits

cat quota.yml

apiVersion: v1
kind: ResourceQuota
metadata:
  name: project-quota
spec:
  hard:
    cpu: 900m

# Same as:
oc quota project-quota --hard=cpu=900m
oc describe quota
# After you deploy an app, it will consume the project quota

oc describe pod <hello-1-abcdef> | grep -A 2 Requests
# When you scale up and get over the quota, resources will not be created
oc get resourcequota --list-all-quotas
oc get events | grep -i error
oc set resources dc hello --requests=memory=256Mi
# Memory request is not counted against the project quota

13. Readiness/liveness

oc status
curl http://probe.apps.lab.example.com/health
curl http://probe.apps.lab.example.com/ready
<<CREATE PROBES IN WEB GUI>>
oc get events --sort-by='.metadata.CreationTimestamp' | grep 'probe failed'

readinessProbe:
  httpGet:                                        # TCP Socket
    path: /health
    port: 8888
  initialDelaySeconds: 15
  timeoutSeconds: 1

livenessProbe:
  exec:
    command:
      - cat
      - /tmp/health
  initialDelaySeconds: 15
  timeoutSeconds: 1

14. FAQs

a/ Import the template into OpenShift

oc apply -n openshift -f <template.yml>

b/ Import the Docker image to OpenShift

oc import-image <stream> --from=registry.lab.example.com/todoapp/todoui --confirm -n <todoapp>

c/ Turn service into NodePort

oc edit svc <hello>

. . .
  ports:
    - name: 8080-tcp
      . . .
      nodePort: 30800
  type: NodePort
. . .

d/ Access shell inside the pod

oc rsh <hello-1-abcdef>

e/ Export resource to YAML

oc export pod <hello-1-abcdef> > pod.yml
# As template:
oc export svc,dc hello --as-template=docker-hello > template.yml

f/ Configure router to handle wildcard routes

oc scale dc/router --replicas=0
oc set env dc/router ROUTER_ALLOW_WILDCARD_ROUTES=true
oc scale dc/router --replicas=3
oc expose svc test --wildcard-policy-subdomain --hostname='www.lab.example.com'

g/ Autocomplete

source /etc/bash_completion.d/oc

h/ Troubleshooting policies

oc describe clusterPolicyBindings :default
oc describe policyBindings :default

i/ Security Context Constraints (SCCs)

oc get scc
oc create serviceaccount <account>

# SCCs:
- anyuid
- hostaccess
- hostmount-anyuid
- nouroot
- privileged
- restricted

# Default SELinux policies do not allow containers to access NFS shares!
setsebool -p virt_use_nfs=true
setsebool -p virt_sandbox_use_nfs=true

j/ ConfigMap

oc create configmap <special-config> --from-literal=serverAddress=172.20.30.40

# ConfigMaps:
  --from-literal=KEY=VALUE
  --from-file=directory/file
  --from-file=directory/
  --from-file=KEY=directory/file
# Consuming using "configMapKeyRef"

# List all ENV:
oc env dc/printenv --list

k/ RBAC table

Name of the role	Permissions
cluster-admin	superuser
cluster-status	read-only
edit	no admin, no quota, no access mgmt
basic-user	read account
self-provisioner	cluster role to create new project(s)
admin	anything

l/ Autoscale pods

oc autoscale dc/myapp --min 1 --max 5 --cpu-percent=80
oc get hpa/frontend

m/ Tag images

oc tag <ruby:latest> <ruby:2.0>
# Options:
#   --alias=true
#   --scheduled=true
#   --reference-policy=local

n/ Docker import vs Docker load

# Docker import
  - Create an empty filesystem image and import the contents of the tarball into it.
# Docker load
  - Load an image from a file or STDIN. Restores both images & tags. Write image names or IDs imported into STDOUT.

o/ OpenShift output vs export

oc get <RES> -o yaml
oc export <RES>
# Export will show object definition without any runtime specifics

p/ A/B routing

oc set route-backends <ROUTE> <svc1=weight>
oc set route-backends cotd cotd1=50 cotd2=50

q/ Link secret with service account

oc secret link <service-account> <secret-name>

r/ Process template into a list of resources

oc process -f <TEMPLATE> | oc create -f -         # examines template, generates parameters. To override params, add -v

s/ Examine pod contents

/usr/local/s2i/run
/var/run/secrets/kubernetes.io/serviceaccount
/root/buildinfo

t/ Delete environment variable

oc set env dc/d1 ENV1- ENV2- ENV3-

u/ Using secrets with ENV & Volumes

oc env dc/printenv --from=secret/printsecret

oc env dc/printenv --from=secret/printsecret --prefix=DB_

oc set volume dc/printenv --add --overwrite --name=db-conf-volume --mount-path /debconf/ --secret-name=printenv-secret

v/ Turn off automatic triggers

oc set triggers dc <NAME> --manual

w/ Allow Jenkins to build & deploy the app

Jenkins container has to be deployed first: Service Catalog > CI/CD > Jenkins (persistent)

oc policy add-role-to-user edit system serviceaccount:<PROJECT_NAME>:jenkins -n <NAMESPACE>

Because Jenkins is in a different project than the application

x/ Generate values in templates

parameters:
  - name: PASSWORD
    description: "Random password"
    generate: expression
    from: "[a-zA-Z0-9]{12}"

# Create an application from template:
oc new-app --template=ruby-hello --param=A=B

To make template available accross the cluster, cluster admin must add it to the openshift namespace

List all parameters from mysql template: oc process --parameters=true -n openshift mysql