From 334b0710c7c9f1755efe10747a652fa55d50846d Mon Sep 17 00:00:00 2001 From: Antti Leinonen Date: Thu, 14 Mar 2024 13:20:29 +0200 Subject: [PATCH] Vdi installation script (#1250) * Add bin/download-applications-linux * Add vdi-setup * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Update * Revert "Update" This reverts commit 15767e8fbffbd291a8b2fc8d79220593569fe4ce. * Add instructions on how to set up VDI * Update the section about contacting helpdesk in the vdi guide --------- Co-authored-by: Henrik Nygren --- bin/download-applications-linux | 87 +++++++++++++++++ bin/vdi-setup | 164 ++++++++++++++++++++++++++++++++ docs/vdi-setup.md | 79 +++++++++++++++ 3 files changed, 330 insertions(+) create mode 100755 bin/download-applications-linux create mode 100755 bin/vdi-setup create mode 100644 docs/vdi-setup.md diff --git a/bin/download-applications-linux b/bin/download-applications-linux new file mode 100755 index 000000000000..fcc4a0b5f7b5 --- /dev/null +++ b/bin/download-applications-linux @@ -0,0 +1,87 @@ +#!/bin/bash +set -euo pipefail + +# Define the list of URLs and checksums to download +downloads=( + "https://storage.googleapis.com/skaffold/releases/v2.4.1/skaffold-linux-amd64 skaffold 5a5f147c6262263b6f30790673c89d63f44732c33d7eb472eb99248d883650eb" + "https://dl.k8s.io/release/v1.27.1/bin/linux/amd64/kubectl kubectl 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d" + "https://github.com/kubernetes/minikube/releases/download/v1.30.1/minikube-linux-amd64 minikube e53d9e8c31f4c5f683182f5323d3527aa0725f713945c6d081cf71aa548ab388" + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.0.3/kustomize_v5.0.3_linux_amd64.tar.gz kustomize b4e376166eeeab348cdce889dc3664ddcc929174ed103346af056c3e1dc95db7" + "https://github.com/rhysd/actionlint/releases/download/v1.6.24/actionlint_1.6.24_linux_amd64.tar.gz actionlint fbc889db89c64e9f390835ae3f3661aa1a1cdebc7738476032f72379741ebb5f" + "https://github.com/stern/stern/releases/download/v1.25.0/stern_1.25.0_linux_amd64.tar.gz stern 1a89589da2694fadcec2e002ca6e38a3a5eab2096bbdd1c46a551b9f416bc75a" + "https://github.com/ahmetb/kubectx/releases/download/v0.9.4/kubens_v0.9.4_linux_x86_64.tar.gz kubens cc376ced21b441efe4d61685830e0ff3ade194c303216d1ca08b72073c556d44" + "https://github.com/ahmetb/kubectx/releases/download/v0.9.4/kubectx_v0.9.4_linux_x86_64.tar.gz kubectx 9317271d6dd7c771c844d9d3b37a7e56c5d74c6e53366aa230ced3fb69d128bc" +) + +# Define the directory to download the binaries to +download_dir="$HOME/bin/secret-project" + +# Create the download directory if it doesn't exist +mkdir -p "$download_dir" + +# Download each binary from the list of URLs +for download in "${downloads[@]}"; do + # Extract the URL and checksum from the download string + url=$(echo "$download" | awk '{print $1}') + filename=$(echo "$download" | awk '{print $2}') + checksum=$(echo "$download" | awk '{print $3}') + destination="$download_dir/$filename" + # If file already downloaded and checksum matches, skip + if [ -f "$destination" ] && echo "$checksum $download_dir/$filename" | sha256sum --check --status; then + echo "$filename already exists in $download_dir" + continue + fi + + echo "Downloading $filename from $url to $destination" + + # If the url is a tar.gz, download it to a temporary file and extract it to a temporary directory + if [[ "$url" == *.tar.gz ]]; then + tmpfile=$(mktemp) + tmpdir=$(mktemp -d) + curl -Lss "$url" -o "$tmpfile" + tar -xzf "$tmpfile" -C "$tmpdir" + # Find the binary in the temporary directory using $filename as a pattern + binary=$(find "$tmpdir" -name "$filename") + # Move the binary to the download directory + mv "$binary" "$destination" + # Remove the temporary file and directory + rm "$tmpfile" + rm -rf "$tmpdir" + + else + # Download the binary to the download directory + curl -Lss "$url" -o "$destination" + fi + + # Verify the checksum of the downloaded file + if ! echo "$checksum $download_dir/$filename" | sha256sum --check --status; then + echo "Checksum verification failed for $filename" + # Expected and was + echo "Expected the checksum to be $checksum but was $(sha256sum "$destination")" + # Remove the file if the checksum doesn't match + rm "$destination" + exit 1 + fi + + +done +echo "Downloaded all binaries successfully" +chmod +x "$download_dir"/* +ls -l "$download_dir" + +# Add the download directory to the PATH +# Detect shell +shell_profile="$HOME/.profile" + +# Detect whether the PATH has already been added +if grep -q "$download_dir" "$shell_profile"; then + echo "$download_dir already exists in $shell_profile" + exit 0 +fi +echo "export PATH=$download_dir:\$PATH" >> "$shell_profile" +echo "Added $download_dir to $shell_profile" +echo "export PATH=$HOME/.cargo/bin:\$PATH" >> "$shell_profile" +echo "Added $HOME/.cargo/bin to PATH as requested by cargo" + +cargo install sqlx-cli --no-default-features --features rustls,postgres +cargo install oxipng diff --git a/bin/vdi-setup b/bin/vdi-setup new file mode 100755 index 000000000000..93a5c1a1fbdc --- /dev/null +++ b/bin/vdi-setup @@ -0,0 +1,164 @@ +#!/bin/bash +set -euo pipefail +banner="########### MOOC-VDI setup ###########" + +# Check if /data exists +if [ ! -d /data ]; then + echo "ERROR: /data doesn't exist. Are you sure this is a VDI and are you sure it has been configured properly?" + exit 1 +fi + +echo "/data exists" + +original_user="$(logname)" +original_home="/home/$original_user" + + +# Run as root +if [ "$EUID" -ne 0 ]; then + echo "ERROR: Please run as root" + exit 1 +fi + +echo "Running as root" + +# Create folder /data/$original_user +if [ ! -d "/data/$original_user" ]; then + mkdir "/data/$original_user" +fi + +echo "/data/$original_user exists" + +# Create .profile +if [ ! -f "/data/$original_user/.profile" ]; then + touch "/data/$original_user/.profile" +fi + +# Create /data/$original_user/Code/secret-project-331 +if [ ! -d "/data/$original_user/Code/secret-project-331" ]; then + mkdir -p "/data/$original_user/Code/secret-project-331" +fi + +echo "/data/$original_user/Code/secret-project-331 exists" + +bashrc_addition=$(cat <<-END + +$banner +if [ "\$(hostname)" = "$(hostname)" ] ; then + echo "Switching home dir to a local disk for performance." + export HOME=/data/$original_user + source /data/$original_user/.profile + cd "\$HOME/Code/secret-project-331" + export SHELL=/usr/bin/fish + [ -x \$SHELL ] && exec \$SHELL "\$@" +fi +$banner + +END +) + +echo "$bashrc_addition" > /tmp/bashrc_addition +chmod a+r /tmp/bashrc_addition + +script_to_run_as_original_user=$(cat <<-END +#!/bin/bash +set -euo pipefail +bashrc_addition=\$(cat /tmp/bashrc_addition) +echo "bashrc_addition: \$bashrc_addition" +echo "Running as $original_user" +# If .bashrc doesn't exist, create it +if [ ! -f "$original_home/.bashrc" ]; then + touch "$original_home/.bashrc" +fi + +# check if banner in .bashrc +if grep -q "$banner" "$original_home/.bashrc"; then + echo "Vdi snippet alredy exists in $original_home/.bashrc" +else + echo "Vdi snippet not found in $original_home/.bashrc" + echo "Adding banner to $original_home/.bashrc" + echo "\$bashrc_addition" >> "$original_home/.bashrc" +fi + +END +) + + + +echo "--------------------------------------------------------" +echo "$script_to_run_as_original_user" +echo "--------------------------------------------------------" + +echo "Running script" + +sudo -u "$original_user" bash -c "$script_to_run_as_original_user" + +rm /tmp/bashrc_addition + +export HOME="/data/$original_user" + +# Installing packages +# If nvm not installed, install it +if ! command -v nvm; then + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash + echo "Installed nvm" +fi + +# Install postgres-cli from apt +sudo apt-get update +sudo apt-get install -y postgresql-client bc jq rsync moreutils libssl-dev pkg-config + +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update + +# Install docker +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# add current user to docker group +sudo usermod -aG docker "$original_user" + +# If rust not installed, install it +if ! command -v rustc; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + echo "Installed rust" +fi + +fish -c "curl -sL https://raw.githubusercontent.com/jorgebucaran/fisher/main/functions/fisher.fish | source && fisher install jorgebucaran/fisher" +fish -c "fisher install jorgebucaran/nvm.fish; and fisher install edc/bass" + +echo "Fish setup complete" + +# Change docker data dir /data/original_user/docker +systemctl stop docker +if [ ! -d "/data/$original_user/docker" ]; then + mkdir "/data/$original_user/docker" +fi +# Update docker config +new_docker_config=$(cat <<-END +{ + "data-root": "/data/$original_user/docker" +} +END +) +echo "$new_docker_config" > /etc/docker/daemon.json +systemctl start docker + +# Clone the repository +repository="https://github.com/rage/secret-project-331.git" +clone_path="/data/$original_user/Code/secret-project-331" +git clone "$repository" "$clone_path" +echo "Cloned the secret project repository to $clone_path" + +chown -R "$original_user" "/data/$original_user" +echo "Fixed permissions in /data/$original_user" diff --git a/docs/vdi-setup.md b/docs/vdi-setup.md new file mode 100644 index 000000000000..8c1094fbccc9 --- /dev/null +++ b/docs/vdi-setup.md @@ -0,0 +1,79 @@ +# VDI setup + +## Ordering a VDI desktop + +1. Go to [https://onify.it.helsinki.fi/](https://onify.it.helsinki.fi/) and select `Order a virtual desktop`. +2. Select `Basic Linux`. +3. To get proper resources and an external disk that is required for the environment setup, the university's helpdesk needs to be emailed about it. The MOOC-center has an email thread regarding this, so you can ask in the Rage Research-slack's #secret-project-331 channel for help to email to that thread to get the resources for your virtual desktop. + +## Install and set up VMware Horizon Client + +1. If you don't have the VMware Horizon Client (university computers should have it by default), download it from [here](https://customerconnect.vmware.com/en/downloads/info/slug/desktop_end_user_computing/vmware_horizon_clients/horizon_8). +2. Install the software. +3. Open VMware Horizon Client. +4. Double click the `New Server`-button. +5. Enter your university credentials. The dropdown below the credentials-fields should default to `ATKK`, but if it's something else, switch it to `ATKK`. +6. Select your virtual desktop, probably named `vdi-` or `mooc-`. + +**NB!** You can also access the virtual desktop at [vdi.helsinki.fi](https://vdi.helsinki.fi) without installing the VMware Horizon Client, but this isn't recommended as the connection is a lot worse and the website-version has more limited features. + +## Getting sudo + +Fill this [form](https://elomake.helsinki.fi/lomakkeet/42471/lomake.html) to request admin rights to your virtual desktop. You can state as the reason for requesting administrative priviledges that developing MOOC-center's software is impossible without admin rights. + +## Setting up secret project in the virtual desktop + +### Setting up git and GitHub + +git should be installed by default, you can check by running `git --version`. If it is missing for some reason, you can install git [here](https://git-scm.com/downloads). + +Set your identity with + +- `git config --global user.name "John Doe"` +- `git config --global user.email johndoe@example.com` + + - **NB!** If you'd like to keep your email private, see GitHub's guide [here](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-email-preferences/setting-your-commit-email-address) on how to use a `noreply`-email address provided by GitHub. + +- Optional: Install GitHub CLI for caching your credentials (so you don't have to enter your personal access token for every git command) by copy-pasting the following to your terminal: + +```bash +type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y) +curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ +&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ +&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ +&& sudo apt update \ +&& sudo apt install gh -y +``` + +- Cache your credentials with GitHub CLI with `gh auth login`. + - Follow the prompts, be sure to select HTTPS. + - If you select login via browser, you need to install firefox with `snap install firefox`. + +### Setting up the project space + +1. Clone the repository with https (or ssh if you'd prefer, GitHub recommends https nowadays) with `git clone https://github.com/rage/secret-project-331.git`. + +2. Run `sudo bin/vdi-setup` script from the repository to install the required dependencies and set up the repository in the /data -disk. + +3. Run `source "$HOME/.profile"` to apply changes made to the PATH-variable. + +- If you get an error here, try running the command in a bash shell instead of the fish shell. You can enter a bash shell by running the command `bash`, and after running the source-command you can exit back to the fish shell with ctrl+d. + +4. Install the latest version of Node with `nvm install lts`. + +5. Run `newgrp docker` to apply changes made to the group `docker`. + +6. Run `bin/download-applications-linux` script to install the required tools. + +7. Restart the terminal. + +8. Delete the repository you manually cloned in step 1. It shouldn't be needed anymore, as you should use the repository that was cloned to /data with the vdi-setup -script during step 2. + +### Setting up the development environment + +1. Open a terminal and go to `/data/username/Code/secret-project-331`. + +2. Install google cloud cli with `sudo snap install google-cloud-cli --classic` and then log into it with `gcloud auth login`. + +3. Follow the instructions from the project's `doc/Development.md` to set up the environment. + - **NB!** most of the tools and dependencies are already installed during setting up the project space, so you can skip ahead to the section _Running the development environment_