From 3e8c25f25735ba135d55fe45dd13421e303a7d75 Mon Sep 17 00:00:00 2001 From: Jan Walzer Date: Wed, 13 Mar 2024 09:58:49 +0100 Subject: [PATCH 1/2] need to store credentials in ansible vault --- .gitignore | 3 ++- ansible.cfg | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 66d81482..1cffe773 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ cluster.yml .DS_Store certificate/ ansible-navigator.log -context/ \ No newline at end of file +context/ +.vaultpasswd \ No newline at end of file diff --git a/ansible.cfg b/ansible.cfg index 756ca65a..7b6f905f 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -20,6 +20,7 @@ retry_files_save_path = ~/ansible-installer-retries nocows = True remote_user = root roles_path = ~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:ansible/roles/:ansible/add-on-roles/ +vault_password_file = .vaultpasswd gathering = smart # fact_caching = jsonfile From 86893cbcbfaffead1e77fc2ce0a5fb46dd20e185 Mon Sep 17 00:00:00 2001 From: Jan Walzer Date: Thu, 14 Mar 2024 21:41:32 +0100 Subject: [PATCH 2/2] setup a cryptroot-installation including necessary steps for unlocking via ssh after reboot --- .../roles/provision-hetzner/defaults/main.yml | 5 + .../tasks/provision-server.yml | 155 +++++++++++++++++- .../provision-hetzner/templates/autosetup | 12 +- .../provision-hetzner/templates/post-install | 68 ++++++++ cluster-example.yml | 6 + 5 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/provision-hetzner/templates/post-install diff --git a/ansible/roles/provision-hetzner/defaults/main.yml b/ansible/roles/provision-hetzner/defaults/main.yml index 5952dc1b..668344da 100644 --- a/ansible/roles/provision-hetzner/defaults/main.yml +++ b/ansible/roles/provision-hetzner/defaults/main.yml @@ -4,14 +4,19 @@ hetzner_webservice_password: hetzner_hostname: # needed for Hetzner server computer name hetzner_ip: # needed for Hetzner Web UI and Ansible hetzner_autosetup_file: templates/autosetup +hetzner_postinst_file: templates/post-install hetzner_disk1: sda hetzner_disk2: sdb hetzner_raid_level: 0 hetzner_vg_name: vg0 + +hetzner_crypt_bootstrap_password: InitialCryptPW + hetzner_image: "/root/.oldroot/nfs/install/../images/CentOS-90-stream-amd64-base.tar.gz" hetzner_image_ignore_errors: false hetzner_size_of_libvirt_images: all + robot_base: https://robot-ws.your-server.de/ needs_reprovision: false already_in_rescue: false diff --git a/ansible/roles/provision-hetzner/tasks/provision-server.yml b/ansible/roles/provision-hetzner/tasks/provision-server.yml index fd03dff6..b6c1db9f 100644 --- a/ansible/roles/provision-hetzner/tasks/provision-server.yml +++ b/ansible/roles/provision-hetzner/tasks/provision-server.yml @@ -142,14 +142,27 @@ mode: 0644 delegate_to: "{{ hetzner_ip }}" +- name: Copy postinstall file for cryptroot + template: + src: "{{ hetzner_postinst_file }}" + dest: /root/post-install.ansible + owner: root + group: root + mode: 0755 + delegate_to: "{{ hetzner_ip }}" + when: hetzner_crypt_password is defined + + - name: Run installimage - command: "/root/.oldroot/nfs/install/installimage -a -c /root/autosetup.ansible" + command: "/root/.oldroot/nfs/install/installimage -a -c /root/autosetup.ansible {{postinst_string}}" environment: TERM: "vt100" register: result changed_when: true failed_when: false delegate_to: "{{ hetzner_ip }}" + vars: + postinst_string: "{% if hetzner_crypt_password is defined %}-x /root/post-install.ansible{% endif %}" - name: Print installimage output with -v debug: @@ -157,6 +170,12 @@ verbosity: 1 delegate_to: localhost +- name: Print installimage stderroutput with -v + debug: + var: result.stderr_lines + verbosity: 1 + delegate_to: localhost + - name: Check stderr from installimage debug: msg: "Something want wrong at installimage: {{ result.stderr_lines | join('\n') }}" @@ -191,6 +210,97 @@ changed_when: '"updated" in output.stdout' delegate_to: localhost +- name: Crypt-Unlocking twice + when: hetzner_crypt_password is defined + block: + ## + ## We have to unlock twice, because of auto-relabeling there are two boots + ## + + # wait for first reboot + - name: Wait 600 seconds for port crypt-shell to become open + wait_for: + port: "{{ hetzner_crypt_network_ssh_port }}" + host: '{{ hetzner_ip }}' + delay: 10 + timeout: 600 + connection: local + + # Show console before auth + - name: Read the console + ansible.builtin.raw: | + console_peek + timeout: 5 + register: cryptroot_peek_reg + delegate_to: "{{ hetzner_ip }}" + ignore_errors: true + - debug: var=cryptroot_peek_reg + + # this needs to be done manually, because the "console_auth" tool + # wants interactive input and we need to fake that + - name: Unlock cryptroot + ansible.builtin.shell: "echo {{ hetzner_crypt_bootstrap_password }} | ssh -o StrictHostKeyChecking=no -l root -t {{ hetzner_ip }} 'console_auth'" + timeout: 5 + register: cryptroot_unlock_reg + ignore_errors: true + delegate_to: "localhost" + + # Show console after auth + - name: Pause a bit for the hardware reset to kick in + pause: seconds=2 + - name: Read the console + ansible.builtin.raw: | + console_peek + timeout: 5 + register: cryptroot_peek_reg + delegate_to: "{{ hetzner_ip }}" + ignore_errors: true + - debug: var=cryptroot_peek_reg + + - name: Pause a bit for the hardware reset to kick in + pause: seconds=15 + + # wait for second reboot + - name: Wait 600 seconds for port crypt-shell to become open + wait_for: + port: "{{ hetzner_crypt_network_ssh_port }}" + host: '{{ hetzner_ip }}' + delay: 10 + timeout: 600 + connection: local + + # Show console before auth + - name: Read the console + ansible.builtin.raw: | + console_peek + timeout: 5 + register: cryptroot_peek_reg + delegate_to: "{{ hetzner_ip }}" + ignore_errors: true + - debug: var=cryptroot_peek_reg + + # this needs to be done manually, because the "console_auth" tool + # wants interactive input and we need to fake that + - name: Unlock cryptroot + ansible.builtin.shell: "echo {{ hetzner_crypt_bootstrap_password }} | ssh -o StrictHostKeyChecking=no -l root -t {{ hetzner_ip }} 'console_auth'" + timeout: 5 + register: cryptroot_unlock_reg + ignore_errors: true + delegate_to: "localhost" + + # Show console after auth + - name: Pause a bit for the hardware reset to kick in + pause: seconds=2 + - name: Read the console + ansible.builtin.raw: | + console_peek + timeout: 5 + register: cryptroot_peek_reg + delegate_to: "{{ hetzner_ip }}" + ignore_errors: true + - debug: var=cryptroot_peek_reg + + - name: Wait 600 seconds for port 22 to become open wait_for: port: 22 @@ -203,3 +313,46 @@ ansible.builtin.gather_facts: register: host_facts delegate_to: "{{ hetzner_ip }}" + +- name: Change luks passphrase + when: hetzner_crypt_password is defined + delegate_to: "{{ hetzner_ip }}" + block: + - name: Set fact for the LVM PVs of vg0 + set_fact: + luks_pv: "{{ ansible_facts.lvm.pvs | dict2items | selectattr('value.vg', 'equalto', hetzner_vg_name) | map(attribute='key') | list }}" + no_log: true + + - name: Find parent of cryptdevice + ansible.builtin.shell: + cmd: "lsblk -l -o path,pkname | grep {{ luks_pv[0] }} | cut -d ' ' -f 2" + executable: /bin/bash + when: luks_pv | length > 0 + register: luks_cryptdev_reg + ignore_errors: true + + - set_fact: + luks_cryptdev: "{{luks_cryptdev_reg.stdout_lines[0]}}" + + - name: Ensure the LUKS device variable is set + fail: + msg: "LUKS device could not be determined." + when: luks_pv | length == 0 + + - name: Add new passphrase to LUKS device + ansible.builtin.shell: + cmd: "echo -n '{{ hetzner_crypt_password }}' | cryptsetup luksAddKey /dev/{{ luks_cryptdev }} --key-file <(echo -n '{{ hetzner_crypt_bootstrap_password }}')" + executable: /bin/bash + no_log: true + when: luks_pv | length > 0 + register: lukskey_add_reg + ignore_errors: true + + - name: Remove old passphrase from LUKS device + ansible.builtin.shell: + cmd: "echo -n '{{ hetzner_crypt_bootstrap_password }}' | cryptsetup luksRemoveKey /dev/{{ luks_cryptdev }} --key-file <(echo -n '{{ hetzner_crypt_bootstrap_password }}')" + executable: /bin/bash + no_log: true + when: luks_pv | length > 0 + register: lukskey_del_reg + ignore_errors: true diff --git a/ansible/roles/provision-hetzner/templates/autosetup b/ansible/roles/provision-hetzner/templates/autosetup index 642c45d0..72740d4e 100644 --- a/ansible/roles/provision-hetzner/templates/autosetup +++ b/ansible/roles/provision-hetzner/templates/autosetup @@ -5,7 +5,8 @@ SWRAIDLEVEL {{ hetzner_raid_level }} BOOTLOADER grub HOSTNAME {{ hetzner_hostname }} PART /boot ext3 1024M -PART lvm {{ hetzner_vg_name }} all + +PART lvm {{ hetzner_vg_name }} all {% if hetzner_crypt_password is defined %} crypt{% endif %} LV {{ hetzner_vg_name }} root / xfs 50G LV {{ hetzner_vg_name }} swap swap swap 8G @@ -13,4 +14,13 @@ LV {{ hetzner_vg_name }} home /home xfs 10G LV {{ hetzner_vg_name }} var /var xfs 50G LV {{ hetzner_vg_name }} libvirt /var/lib/libvirt/images xfs {{ hetzner_size_of_libvirt_images }} +{# + we are setting an dummy password here, because this one is unsafe + This file remains on the server and could have been read by the + installimage, we're setting this one here and change it afterwards +#} +{% if hetzner_crypt_password is defined %} +CRYPTPASSWORD {{ hetzner_crypt_bootstrap_password }} +{% endif %} + IMAGE {{ hetzner_image }} diff --git a/ansible/roles/provision-hetzner/templates/post-install b/ansible/roles/provision-hetzner/templates/post-install new file mode 100644 index 00000000..9fae0d09 --- /dev/null +++ b/ansible/roles/provision-hetzner/templates/post-install @@ -0,0 +1,68 @@ +#!/bin/bash + +NETWORK_STRING='ip={{ public_ip }}::{{ hetzner_crypt_network_gatewayv4 }}:255.255.255.255:{{ hetzner_hostname }}:{{ hetzner_crypt_network_interface }}:none rd.route={{ hetzner_crypt_network_gatewayv4 }}\/32:{{ hetzner_crypt_network_gatewayv4 }}:{{ hetzner_crypt_network_interface }}' + +dnf copr enable uriesk/dracut-crypt-ssh -y +dnf install -y epel-release +dnf install -y dracut-crypt-ssh + +echo "generating hostkeys" +ssh-keygen -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key +ssh-keygen -t ecdsa -N '' -f /etc/ssh/ssh_host_ecdsa_key +ssh-keygen -t ed25519 -N '' -f /etc/ssh/ssh_host_ed25519_key + +echo adjusting kernel cmdline for early network +sed -i '/^GRUB_CMDLINE_LINUX="/ {s/\//g;s/"$/ rd.neednet=1 '"$NETWORK_STRING"'"/;}' /etc/default/grub + +echo rebuilding grub.cfg +grub2-mkconfig --output /boot/grub2/grub.cfg + +echo creating dracut/crypt-ssh.conf +cat > /etc/dracut.conf.d/crypt-ssh.conf << EOT +# NOTE: The defaults in this file MUST be carefully read and understood before using this module! +# The defaults may NOT be appropriate for your site. Carefully review and understand your threat model +# (ie, what attack scenarios you are protecting yourself against) and adjust accordingly! +# + +# The port to run the ssh daemon on +# Default: 222 +dropbear_port="{{ hetzner_crypt_network_ssh_port }}" + +# Where to get the RSA and/or ECDSA keys for dropbear, options are: +# GENERATE: generate a new one for each initrd run, the public key will be printed during the dracut build process +# and on boot +# SYSTEM: use (convert) the host key from the host system's SSH daemon. This will make the initrd ssh indistinguishable +# from the running system - this may be a security risk, depending on your threat model, but simplifies +# your client-side ssh configuration +# /path/to/openssh_key: an absolute path to a host key, in OpenSSH format as generated by ssh-keygen. +# A public key with '.pub' ending must be present too. +# +# It is recommend that you use the system one, or supply your own. If using the system key, be aware that an attacker +# that can access your initrd could use the host key to impersonate the running system. This could allow them to attempt +# an MITM attack. +# +# Default: GENERATE +# dropbear_rsa_key="GENERATE" +# dropbear_ecdsa_key="GENERATE" +# dropbear_ed25519_key="GENERATE" +#dropbear_rsa_key="SYSTEM" +#dropbear_ecdsa_key="SYSTEM" +#dropbear_ed25519_key="SYSTEM" + +# Location of the list of authorized public keys that can log into the initrd ssh daemon +# Defaults to the authorized_keys list for root. It may be advantageous to use a different authorized_keys list +# so that users/machines that can unlock the machine are not necessarily given full root access after boot. +# Note that root access to the initrd does give an attacker means to provide themselves with root access after boot, +# especially if they hold the encryption keys to the root drive - choose carefully! +# +# Default: /root/.ssh/authorized_keys +dropbear_acl="/root/.ssh/authorized_keys" + +# Users wishing to unlock LUKS volumes remotely using the 'unlock' helper will need cryptsetup available in the initramfs. +# Uncomment the below line to make sure that the application is available when needed. +# +install_items+=" /sbin/cryptsetup " +EOT + +echo rebuilding initramfs +dracut -v --force --kver="$(ls /lib/modules)" \ No newline at end of file diff --git a/cluster-example.yml b/cluster-example.yml index fd949c5e..4c81630c 100644 --- a/cluster-example.yml +++ b/cluster-example.yml @@ -5,6 +5,12 @@ hetzner_webservice_password: # see docs hetzner_hostname: "changeme" hetzner_ip: "changeme" +## the following block needs to be set for installation with cryptroot +#hetzner_crypt_password: +#hetzner_crypt_network_gatewayv4: +#hetzner_crypt_network_interface: enp7s0 +#hetzner_crypt_network_ssh_port: 22 + cluster_name: ocp4 public_domain: example.com