From 3abbea152f12ed26d15e01f582bbcccc432cbedd Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 1 Aug 2024 16:54:47 -0400 Subject: [PATCH] Statefull Rocky 9.4 Base OS Confluent/Slurm Edition for Linux Recipe to support using Confluent as a system manager and provisioner. When setting up an ohpc cluster. Signed-off-by: tkucherera --- components/admin/docs/SPECS/docs.spec | 8 + .../common/add_confluent_hosts_intro.tex | 120 ++++++ .../common/add_to_compute_confluent_intro.tex | 61 +++ .../common/confluent_compute_bos_intro.tex | 4 + .../install/common/confluent_customize.tex | 35 ++ .../common/confluent_init_os_images_rocky.tex | 59 +++ .../install/common/confluent_setup.tex | 32 ++ .../common/confluent_slurm_test_job.tex | 118 ++++++ .../install/common/enable_confluent_repo.tex | 13 + .../install/common/enable_local_ohpc_repo.tex | 2 +- .../enable_local_ohpc_repo_confluent.tex | 30 ++ .../finalize_confluent_provisioning.tex | 16 + .../install/common/import_confluent_files.tex | 16 + .../common/import_confluent_files_slurm.tex | 23 ++ .../install_provisioning_confluent_intro.tex | 18 + .../common/lustre-client-centos-confluent.tex | 27 ++ .../common/lustre-client-post-confluent.tex | 34 ++ docs/recipes/install/common/memlimits.tex | 2 +- .../install/common/memlimits_confluent.tex | 23 ++ docs/recipes/install/common/ohpc-doc.sty | 2 + .../common/reset_computes_confluent.tex | 44 +++ .../install/common/slurm_pam_confluent.tex | 14 + .../common/slurm_startup_confluent.tex | 75 ++++ .../install/common/upgrade_confluent.tex | 127 +++++++ .../install/rocky9/input.local.template | 18 +- .../rocky9/x86_64/confluent/slurm/Makefile | 23 ++ .../rocky9/x86_64/confluent/slurm/common | 1 + .../rocky9/x86_64/confluent/slurm/manifest | 1 + .../x86_64/confluent/slurm/manifest.tex | 1 + .../rocky9/x86_64/confluent/slurm/steps.tex | 349 ++++++++++++++++++ 30 files changed, 1293 insertions(+), 3 deletions(-) create mode 100644 docs/recipes/install/common/add_confluent_hosts_intro.tex create mode 100644 docs/recipes/install/common/add_to_compute_confluent_intro.tex create mode 100644 docs/recipes/install/common/confluent_compute_bos_intro.tex create mode 100644 docs/recipes/install/common/confluent_customize.tex create mode 100644 docs/recipes/install/common/confluent_init_os_images_rocky.tex create mode 100644 docs/recipes/install/common/confluent_setup.tex create mode 100644 docs/recipes/install/common/confluent_slurm_test_job.tex create mode 100644 docs/recipes/install/common/enable_confluent_repo.tex create mode 100644 docs/recipes/install/common/enable_local_ohpc_repo_confluent.tex create mode 100644 docs/recipes/install/common/finalize_confluent_provisioning.tex create mode 100644 docs/recipes/install/common/import_confluent_files.tex create mode 100644 docs/recipes/install/common/import_confluent_files_slurm.tex create mode 100644 docs/recipes/install/common/install_provisioning_confluent_intro.tex create mode 100644 docs/recipes/install/common/lustre-client-centos-confluent.tex create mode 100644 docs/recipes/install/common/lustre-client-post-confluent.tex create mode 100644 docs/recipes/install/common/memlimits_confluent.tex create mode 100644 docs/recipes/install/common/reset_computes_confluent.tex create mode 100644 docs/recipes/install/common/slurm_pam_confluent.tex create mode 100644 docs/recipes/install/common/slurm_startup_confluent.tex create mode 100644 docs/recipes/install/common/upgrade_confluent.tex create mode 100644 docs/recipes/install/rocky9/x86_64/confluent/slurm/Makefile create mode 120000 docs/recipes/install/rocky9/x86_64/confluent/slurm/common create mode 120000 docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest create mode 120000 docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest.tex create mode 100644 docs/recipes/install/rocky9/x86_64/confluent/slurm/steps.tex diff --git a/components/admin/docs/SPECS/docs.spec b/components/admin/docs/SPECS/docs.spec index 4b356320f5..895c52272b 100644 --- a/components/admin/docs/SPECS/docs.spec +++ b/components/admin/docs/SPECS/docs.spec @@ -103,6 +103,9 @@ make ; %{parser} steps.tex > recipe.sh ; popd pushd docs/recipes/install/rocky9/x86_64/xcat/slurm make ; %{parser} steps.tex > recipe.sh ; popd +pushd docs/recipes/install/rocky9/x86_64/confluent/slurm +make ; %{parser} steps.tex > recipe.sh ; popd + pushd docs/recipes/install/rocky9/x86_64/xcat_stateful/slurm make ; %{parser} steps.tex > recipe.sh ; popd @@ -175,6 +178,11 @@ install -m 0755 -p -D docs/recipes/install/%{lpath}/recipe.sh %{buildroot}/%{OHP install -m 0644 -p -D docs/recipes/install/%{lpath}/steps.pdf %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/Install_guide.pdf install -m 0755 -p -D docs/recipes/install/%{lpath}/recipe.sh %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/recipe.sh +%define lpath rocky9/x86_64/confluent/slurm +install -m 0644 -p -D docs/recipes/install/%{lpath}/steps.pdf %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/Install_guide.pdf +install -m 0755 -p -D docs/recipes/install/%{lpath}/recipe.sh %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/recipe.sh + + %define lpath rocky9/x86_64/xcat/slurm install -m 0644 -p -D docs/recipes/install/%{lpath}/steps.pdf %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/Install_guide.pdf install -m 0755 -p -D docs/recipes/install/%{lpath}/recipe.sh %{buildroot}/%{OHPC_PUB}/doc/recipes/%{lpath}/recipe.sh diff --git a/docs/recipes/install/common/add_confluent_hosts_intro.tex b/docs/recipes/install/common/add_confluent_hosts_intro.tex new file mode 100644 index 0000000000..bc436d165a --- /dev/null +++ b/docs/recipes/install/common/add_confluent_hosts_intro.tex @@ -0,0 +1,120 @@ +%\subsubsection{Register nodes for provisioning} + +\noindent Next, we add {\em compute} nodes and define their properties as +attributes in \Confluent{} database. +These hosts are grouped logically into a group named {\em +compute} to facilitate group-level commands used later in the recipe. The compute +group has to be defined first before we can add any nodes to the group using the +{\texttt nodegroup define} command. Note the +use of variable names for the desired compute hostnames, node IPs, MAC +addresses, and BMC login credentials, which should be modified to accommodate +local settings and hardware. To enable serial console access via \Confluent{}, +{\texttt console.method} +property is also defined. + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Add hosts to cluster \ref{sec:confluent_add_nodes} +\begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily,] +#define the compute group +[sms](*\#*) nodegroupdefine compute + +# Define nodes as objects in confluent database +[sms](*\#*) for ((i=0; i<$num_computes; i++)) ; do +nodedefine ${c_name[$i]} groups=everything,compute hardwaremanagement.manager=${c_bmc[$i]} \ +secret.hardwaremanagementuser=${bmc_username} secret.hardwaremanagementpassword=${bmc_password} \ +net.hwaddr=${c_mac[$i]} net.ipv4_address=${c_ip[$i]} + done +\end{lstlisting} +% end_ohpc_run + +\begin{center} + \begin{tcolorbox}[] + \small +Defining nodes one-by-one, as done above, is only efficient +for a small number of nodes. For larger node counts, +\Confluent{} provides capabilities for automated detection and +configuration. +Consult the +\href{https://hpc.lenovo.com/users/documentation/confluentdisco.html}{\color{blue}\Confluent{} +Hardware Discovery \& Define Node Guide}. +\end{tcolorbox} +\end{center} + + +%\clearpage +If enabling {\em optional} IPoIB functionality (e.g. to support Lustre over \InfiniBand{}), additional +settings are required to define the IPoIB network with \Confluent{} and specify +desired IP settings for each compute. This can be accomplished as follows for +the {\em ib0} interface: + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Setup IPoIB networking +% ohpc_command if [[ ${enable_ipoib} -eq 1 ]];then +% ohpc_indent 5 +\begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily] +# Register desired IPoIB IPs per compute +[sms](*\#*) for ((i=0; i<$num_computes; i++)) ; do + nodeattrib ${c_name[i]} net.ib0.ipv4_address=${c_ipoib[i]}/${ipoib_netmask} + done +\end{lstlisting} +% ohpc_indent 0 +% ohpc_command fi +% end_ohpc_run + +%\clearpage +confluent2hosts can be used to help generate /etc/hosts entries for a noderange. +It can read from the confluent db, using -a. In this mode, each net.value.attribute +group is pulled together into hosts lines. ipv4 and ipv6 address +fields are associated with the corresponding hostname attributes. + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Generating lines to append to local /etc/hosts +\begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily] +# Add nodes to /etc/hosts +[sms](*\#*) confluent2hosts -a compute +\end{lstlisting} +% end_ohpc_run + +%\clearpage +With the desired compute nodes and domain identified, the remaining steps in the +provisioning configuration process are to define the provisioning mode and +image for the {\em compute} group and use \Confluent{} commands to complete +configuration for network services like DNS and DHCP. These tasks are +accomplished as follows: + +%\clearpage +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Complete networking setup, associate provisioning image +\begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily,literate={BOSSHORT}{\baseosshort{}}1 {IMAGE}{\installimage{}}1] +# Associate desired provisioning image for computes +[sms](*\#*) nodedeploy -n compute -p rocky-9.4-x86_64-default +\end{lstlisting} + +%%% If the Lustre client was enabled for computes in \S\ref{sec:lustre_client}, you +%%% should be able to mount the file system post-boot using the fstab entry +%%% (e.g. via ``\texttt{mount /mnt/lustre}''). Alternatively, if +%%% you prefer to have the file system mounted automatically at boot time, a simple +%%% postscript can be created and registered with \xCAT{} for this purpose as follows. +%%% +%%% % begin_ohpc_run +%%% % ohpc_validation_newline +%%% % ohpc_validation_comment Optionally create xCAT postscript to mount Lustre client +%%% % ohpc_command if [ ${enable_lustre_client} -eq 1 ];then +%%% % ohpc_indent 5 +%%% \begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily,literate={BOSVER}{\baseos{}}1] +%%% # Optionally create postscript to mount Lustre client at boot +%%% [sms](*\#*) echo '#!/bin/bash' > /install/postscripts/lustre-client +%%% [sms](*\#*) echo 'mount /mnt/lustre' >> /install/postscripts/lustre-client +%%% [sms](*\#*) chmod 755 /install/postscripts/lustre-client +%%% # Register script for computes +%%% [sms](*\#*) chdef compute -p postscripts=lustre-client +%%% \end{lstlisting} +%%% % ohpc_indent 0 +%%% % ohpc_command fi +%%% % end_ohpc_run +%%% + diff --git a/docs/recipes/install/common/add_to_compute_confluent_intro.tex b/docs/recipes/install/common/add_to_compute_confluent_intro.tex new file mode 100644 index 0000000000..a09ad033c5 --- /dev/null +++ b/docs/recipes/install/common/add_to_compute_confluent_intro.tex @@ -0,0 +1,61 @@ +% -*- mode: latex; fill-column: 120; -*- + +The next step is adding \OHPC{} components to the {\em compute} nodes that at this +point are running basic OSes. This process will leverage two \Confluent{}-provided +commands: \texttt{nodeshell} to run \texttt{\pkgmgr{}} installer on all the +nodes in parallel and \texttt{nodersync} to distribute configuration files from the +SMS to the {\em compute} nodes. + +\noindent To do this, repositories on the {\em compute} nodes need to be configured +properly. + +\Confluent{} has automatically setup an OS repository on the SMS and configured the +nodes to use it, but it has also enabled online OS repositories. + + +\noindent Next, we alse add the OHPC repo to the compute nodes \S\ref{sec:enable_repo} + +% begin_ohpc_run +% ohpc_comment_header Setup nodes repositories and Install OHPC components \ref{sec:add_components} +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Add OpenHPC repo +[sms](*\#*) (*\chrootinstall*) http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm +\end{lstlisting} +% end_ohpc_run + +The {\em compute} nodes also need access to the EPEL repository, a required +dependency for \OHPC{} packages. + +% begin_ohpc_run +% ohpc_comment_header Configure access to EPEL repo +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Add epel repo +[sms](*\#*) (*\chrootinstall*) epel-release + +\end{lstlisting} +% end_ohpc_run + + +\noindent Additionally, a workaround is needed for \OHPC{} documentation files, +which are installed into a read-only NFS share /opt/ohpc/pub. Any package +attempting to write to that directory will fail to install. The following +prevents that by directing \texttt{rpm} not to install documentation files on +the {\em compute} nodes: + +% begin_ohpc_run +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +[sms](*\#*) nodeshell compute echo -e %_excludedocs 1 \>\> ~/.rpmmacros +\end{lstlisting} +% end_ohpc_run + +\noindent Now \OHPC{} and other cluster-related software components can be +installed on the nodes. The first step is to install a base compute package: +% begin_ohpc_run +% ohpc_comment_header Add OpenHPC base components to compute image +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Install compute node base meta-package +[sms](*\#*) (*\chrootinstall*) ohpc-base-compute +\end{lstlisting} +% end_ohpc_run + +\noindent Next, we can include additional components: diff --git a/docs/recipes/install/common/confluent_compute_bos_intro.tex b/docs/recipes/install/common/confluent_compute_bos_intro.tex new file mode 100644 index 0000000000..882858537c --- /dev/null +++ b/docs/recipes/install/common/confluent_compute_bos_intro.tex @@ -0,0 +1,4 @@ +Installation is accomplished in two steps: First, a generic OS +image is installed on {\em compute} nodes and then, once the nodes are up +and running, \OHPC{} components are added to both the SMS and the nodes at the +same time. \ No newline at end of file diff --git a/docs/recipes/install/common/confluent_customize.tex b/docs/recipes/install/common/confluent_customize.tex new file mode 100644 index 0000000000..f4866a8668 --- /dev/null +++ b/docs/recipes/install/common/confluent_customize.tex @@ -0,0 +1,35 @@ +Here we set up \NFS{} mounting of a +\$HOME file system and the public \OHPC{} install path (\texttt{/opt/ohpc/pub}) +that will be hosted by the {\em master} host in this example configuration. + +\vspace*{0.15cm} +% begin_ohpc_run +% ohpc_comment_header Customize system configuration \ref{sec:master_customization} +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Disable /tftpboot and /install export entries +[sms](*\#*) perl -pi -e "s|/tftpboot|#/tftpboot|" /etc/exports +[sms](*\#*) perl -pi -e "s|/install|#/install|" /etc/exports + +# Export /home and OpenHPC public packages from master server +[sms](*\#*) echo "/home *(rw,no_subtree_check,fsid=10,no_root_squash)" >> /etc/exports +[sms](*\#*) echo "/opt/ohpc/pub *(ro,no_subtree_check,fsid=11)" >> /etc/exports +[sms](*\#*) exportfs -a +[sms](*\#*) systemctl restart nfs-server +[sms](*\#*) systemctl enable nfs-server + + +# Create NFS client mounts of /home and /opt/ohpc/pub on compute hosts +[sms](*\#*) nodeshell compute echo \ + "\""${sms_ip}:/home /home nfs nfsvers=3,nodev,nosuid 0 0"\"" \>\> /etc/fstab +[sms](*\#*) nodeshell compute echo \ + "\""${sms_ip}:/opt/ohpc/pub /opt/ohpc/pub nfs nfsvers=3,nodev 0 0"\"" \>\> /etc/fstab +[sms](*\#*) nodeshell compute systemctl restart nfs + +# Mount NFS shares +[sms](*\#*) nodeshell compute mount /home +[sms](*\#*) nodeshell compute mkdir -p /opt/ohpc/pub +[sms](*\#*) nodeshell compute mount /opt/ohpc/pub + +\end{lstlisting} +% end_ohpc_run + diff --git a/docs/recipes/install/common/confluent_init_os_images_rocky.tex b/docs/recipes/install/common/confluent_init_os_images_rocky.tex new file mode 100644 index 0000000000..654db83656 --- /dev/null +++ b/docs/recipes/install/common/confluent_init_os_images_rocky.tex @@ -0,0 +1,59 @@ +% -*- mode: latex; fill-column: 120; -*- + +With the provisioning services enabled, the next step is to define +a system image that can subsequently be +used to provision one or more {\em compute} nodes. The following subsections highlight this process. + +\subsubsection{Build initial BOS image} \label{sec:assemble_bos} +The following steps illustrate the process to build a minimal, default image for use with \Confluent{}. To begin, you will +first need to have a local copy of the ISO image available for the underlying OS. In this recipe, the relevant ISO image +is \texttt{Rocky-9.4-x86\_64-dvd.iso} (available from the Rocky +\href{https://rockylinux.org/download/}{\color{blue}download} page). +We initialize the image +creation process using the \texttt{osdeploy} command assuming that the necessary ISO image is available locally in +\texttt{\$\{iso\_path\}} as follows: + +The \texttt{osdeploy initialize} command is used to prepare a confluent server to deploy deploy operating systems. +For first time setup, run osdeploy initialize interactively to be walked through the various options using: +\texttt{osdeploy initialize -i} + +% begin_ohpc_run +% ohpc_comment_header Initialize OS images for use with Confluent \ref{sec:assemble_bos} +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,keepspaces,literate={BOSVER}{\baseos{}}1] +[sms](*\#*) osdeploy initialize -${initialize_options} +[sms](*\#*) osdeploy import ${iso_path} + +\end{lstlisting} +% end_ohpc_run + +\noindent Once completed, OS image should be available for use within \Confluent{}. These can be queried via: + +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,keepspaces,literate={BOSVER}{\baseos{}}1] +# Query available images +[sms](*\#*) osdeploy list +Distributions: + rocky-8.5-x86_64 + rocky-9.4-x86_64 +Profiles: + rhel-9.4-x86_64-default + rocky-8.5-x86_64-default +\end{lstlisting} + +If needing to copy files from the sms node to the compute nodes during deployment, this can be done by +modifying the syncfiles file that is created when \texttt{osdeploy import} command is run. For an environment +that has no DNS server and needs to have /etc/hosts file synced amongst all the nodes, the following command +should be run. + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Sync the hosts file in cluster +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,keepspaces,literate={BOSVER}{\baseos{}}1] + [sms](*\#*) echo "/etc/hosts -> /etc/hosts" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default/syncfiles + +\end{lstlisting} +% end_ohpc_run + + +%The \texttt{CHROOT} environment variable highlights the path and is used by +%subsequent commands to augment the basic installation. + diff --git a/docs/recipes/install/common/confluent_setup.tex b/docs/recipes/install/common/confluent_setup.tex new file mode 100644 index 0000000000..7a518f4c42 --- /dev/null +++ b/docs/recipes/install/common/confluent_setup.tex @@ -0,0 +1,32 @@ +At this point, all of the packages necessary to use \Confluent{} on the {\em master} +host should be installed. Next, we enable support for local provisioning using +a second private interface (refer to Figure~\ref{fig:physical_arch}) + +% begin_ohpc_run +% ohpc_comment_header Complete basic Confluent setup for master node \ref{sec:setup_confluent} +%\begin{verbatim} +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,keepspaces] +# Enable internal interface for provisioning +[sms](*\#*) ip link set dev ${sms_eth_internal} up +[sms](*\#*) ip address add ${sms_ip}/${internal_netmask} broadcast + dev ${sms_eth_internal} + +\end{lstlisting} +%\end{verbatim} +% end_ohpc_run + + +\noindent \Confluent{} requires a network domain name specification for system-wide name +resolution. This value can be set to match your local DNS schema or given a +unique identifier such as `local`. A default group called everything is +automatically added to every node. It provides a method to indicate global settings. +Attributes may all be specified on the command line, and an example set could be: + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Define local domainname, deployment protocol and dns +\begin{lstlisting}[language=bash,keywords={},upquote=true,basicstyle=\footnotesize\ttfamily,literate={BOSVER}{\baseos{}}1] +[sms](*\#*) nodegroupattrib everything deployment.useinsecureprotocols=${deployment_protocols} dns.domain=${dns_domain} +[sms](*\#*) nodegroupattrib everything dns.servers=${dns_servers} net.ipv4_gateway=${ipv4_gateway} +\end{lstlisting} + +\noindent We will also define \ No newline at end of file diff --git a/docs/recipes/install/common/confluent_slurm_test_job.tex b/docs/recipes/install/common/confluent_slurm_test_job.tex new file mode 100644 index 0000000000..952414df91 --- /dev/null +++ b/docs/recipes/install/common/confluent_slurm_test_job.tex @@ -0,0 +1,118 @@ +With the resource manager enabled for production usage, users should now be +able to run jobs. To demonstrate this, we will add a `test` user on the {\em master} +host that can be used to run an example job. + +% begin_ohpc_run +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) useradd -m test +\end{lstlisting} +% end_ohpc_run + +Next, the user's credentials need to be distributed across the cluster. +\Confluent{}'s \texttt{nodeappy} has a merge functionality that adds new entries into +credential files on {\em compute} nodes: + +% begin_ohpc_run +\begin{lstlisting}[language=bash,keywords={}] +# Create a sync file for pushing user credentials to the nodes +[sms](*\#*) echo "/etc/passwd -> /etc/passwd" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default/syncfiles +[sms](*\#*) echo "/etc/group -> /etc/group" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default/syncfiles + +# Use Confluent to distribute credentials to nodes +[sms](*\#*) nodeapply -F compute +\end{lstlisting} +% end_ohpc_run + + +~\\ +\input{common/prun} + +\iftoggle{isSLES_ww_slurm_x86}{\clearpage} +%\iftoggle{isxCAT}{\clearpage} + +\subsection{Interactive execution} +To use the newly created ``test'' account to compile and execute the +application {\em interactively} through the resource manager, execute the +following (note the use of \texttt{prun} for parallel job launch which summarizes +the underlying native job launch mechanism being used): + +\begin{lstlisting}[language=bash,keywords={}] +# Switch to "test" user +[sms](*\#*) su - test + +# Compile MPI "hello world" example +[test@sms ~]$ mpicc -O3 /opt/ohpc/pub/examples/mpi/hello.c + +# Submit interactive job request and use prun to launch executable +[test@sms ~]$ salloc -n 8 -N 2 + +[test@c1 ~]$ prun ./a.out + +[prun] Master compute host = c1 +[prun] Resource manager = slurm +[prun] Launch cmd = mpiexec.hydra -bootstrap slurm ./a.out + + Hello, world (8 procs total) + --> Process # 0 of 8 is alive. -> c1 + --> Process # 4 of 8 is alive. -> c2 + --> Process # 1 of 8 is alive. -> c1 + --> Process # 5 of 8 is alive. -> c2 + --> Process # 2 of 8 is alive. -> c1 + --> Process # 6 of 8 is alive. -> c2 + --> Process # 3 of 8 is alive. -> c1 + --> Process # 7 of 8 is alive. -> c2 +\end{lstlisting} + +\begin{center} +\begin{tcolorbox}[] +The following table provides approximate command equivalences between SLURM and +OpenPBS: + +\vspace*{0.15cm} +\input common/rms_equivalence_table +\end{tcolorbox} +\end{center} +\nottoggle{isCentOS}{\clearpage} + +\iftoggle{isCentOS}{\clearpage} + +\subsection{Batch execution} + +For batch execution, \OHPC{} provides a simple job script for reference (also +housed in the \path{/opt/ohpc/pub/examples} directory. This example script can +be used as a starting point for submitting batch jobs to the resource manager +and the example below illustrates use of the script to submit a batch job for +execution using the same executable referenced in the previous interactive example. + +\begin{lstlisting}[language=bash,keywords={}] +# Copy example job script +[test@sms ~]$ cp /opt/ohpc/pub/examples/slurm/job.mpi . + +# Examine contents (and edit to set desired job sizing characteristics) +[test@sms ~]$ cat job.mpi +#!/bin/bash + +#SBATCH -J test # Job name +#SBATCH -o job.%j.out # Name of stdout output file (%j expands to %jobId) +#SBATCH -N 2 # Total number of nodes requested +#SBATCH -n 16 # Total number of mpi tasks #requested +#SBATCH -t 01:30:00 # Run time (hh:mm:ss) - 1.5 hours + +# Launch MPI-based executable + +prun ./a.out + +# Submit job for batch execution +[test@sms ~]$ sbatch job.mpi +Submitted batch job 339 +\end{lstlisting} + +\begin{center} +\begin{tcolorbox}[] +\small +The use of the \texttt{\%j} option in the example batch job script shown is a convenient +way to track application output on an individual job basis. The \texttt{\%j} token +is replaced with the \SLURM{} job allocation number once assigned (job~\#339 in +this example). +\end{tcolorbox} +\end{center} \ No newline at end of file diff --git a/docs/recipes/install/common/enable_confluent_repo.tex b/docs/recipes/install/common/enable_confluent_repo.tex new file mode 100644 index 0000000000..80fa4e9743 --- /dev/null +++ b/docs/recipes/install/common/enable_confluent_repo.tex @@ -0,0 +1,13 @@ +To begin, enable use of the public \Confluent{} repository by adding it to the local list +of available package repositories. This also requires network access from +your {\em master} server to the internet, or alternatively, that +the repository be mirrored locally. In this case, we use the network. + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_comment_header Enable Confluent repositories \ref{sec:enable_confluent} +\begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{8.0}{10}\ttfamily, + literate={VER}{\OHPCVerTree{}}1 {OSREPO}{\OSTree{}}1 {TAG}{\OSTag{}}1 {ARCH}{\arch{}}1 {-}{-}1] +[sms](*\#*) (*\install*) https://hpc.lenovo.com/yum/latest/el9/x86_64/lenovo-hpc-yum-1-1.x86_64.rpm +\end{lstlisting} +% end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/enable_local_ohpc_repo.tex b/docs/recipes/install/common/enable_local_ohpc_repo.tex index 4aa4050baf..f9c2c7804a 100644 --- a/docs/recipes/install/common/enable_local_ohpc_repo.tex +++ b/docs/recipes/install/common/enable_local_ohpc_repo.tex @@ -10,7 +10,7 @@ % begin_ohpc_run % ohpc_validation_newline -% ohpc_comment_header Download OHPC repo and create local mirror \ref{sec:enable_xcat} +% ohpc_comment_header Download OHPC repo and create local mirror \ref{sec:enable_repo} \begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{7.6}{10}\ttfamily, literate={VER}{\OHPCVerTree{}}1 {OSTREE}{\OSTree{}}1 {TAG}{\OSTag{}}1 {ARCH}{\arch{}}1 {-}{-}1 {VERLONG}{\OHPCVersion{}}1] diff --git a/docs/recipes/install/common/enable_local_ohpc_repo_confluent.tex b/docs/recipes/install/common/enable_local_ohpc_repo_confluent.tex new file mode 100644 index 0000000000..19c15417f2 --- /dev/null +++ b/docs/recipes/install/common/enable_local_ohpc_repo_confluent.tex @@ -0,0 +1,30 @@ +To begin, enable use of the \OHPC{} repository by adding it to the local list +of available package repositories. Note that this requires network access from +your {\em master} server to the \OHPC{} repository, or alternatively, that +the \OHPC{} repository be mirrored locally. In cases where network external +connectivity is available, \OHPC{} provides an \texttt{ohpc-release} package +that includes GPG keys for package signing and enabling the repository. The +example which follows illustrates installation of the \texttt{ohpc-release} +package directly from the \OHPC{} build server. + +% begin_ohpc_run +% ohpc_validation_comment Add OpenHPC components to compute instance +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Add OpenHPC repo +[sms](*\#*) (*\install*) http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm +\end{lstlisting} +% end_ohpc_run + +\begin{center} +\begin{tcolorbox}[] +\small Many sites may find it useful or necessary to maintain a local copy of the +\OHPC{} repositories. To facilitate this need, standalone tar +archives are provided -- one containing a repository of binary packages as well as any +available updates, and one containing a repository of source RPMS. The tar files +also contain a simple bash script to configure the package manager to use the +local repository after download. To use, simply unpack the tarball where you +would like to host the local repository and execute the \texttt{make\_repo.sh} script. +Tar files for this release can be found at \href{http://repos.openhpc.community/dist/\OHPCVersion} + {\color{blue}{http://repos.openhpc.community/dist/\OHPCVersion}} +\end{tcolorbox} +\end{center} diff --git a/docs/recipes/install/common/finalize_confluent_provisioning.tex b/docs/recipes/install/common/finalize_confluent_provisioning.tex new file mode 100644 index 0000000000..e266332017 --- /dev/null +++ b/docs/recipes/install/common/finalize_confluent_provisioning.tex @@ -0,0 +1,16 @@ +\subsection{Finalizing provisioning configuration} \label{sec:assemble_bootstrap} + +To finalize the \Confluent{} provisioning configuration, this section first highlights +packing of the image from the chroot environment followed by the +registration of desired compute nodes. To assemble the final compute image, run +\texttt{packimage} as follows: + +% begin_ohpc_run +% ohpc_comment_header Finalize Confluent provisioning configuration \ref{sec:assemble_bootstrap} +% ohpc_validation_newline +% ohpc_validation_comment Pack compute image +\begin{lstlisting}[language=bash,upquote=true,keywords={},basicstyle=\fontencoding{T1}\fontsize{8.0}{10}\ttfamily, + literate={-}{-}1 {BOSSHORT}{\baseosshort{}}1 {ARCH}{\arch{}}1] +[sms](*\#*) packimage BOSSHORT-ARCH-netboot-compute +\end{lstlisting} +% end_ohpc_run diff --git a/docs/recipes/install/common/import_confluent_files.tex b/docs/recipes/install/common/import_confluent_files.tex new file mode 100644 index 0000000000..6360179d9d --- /dev/null +++ b/docs/recipes/install/common/import_confluent_files.tex @@ -0,0 +1,16 @@ +The \Confluent{} system includes functionality to synchronize files located on the +SMS server for distribution to managed hosts. This is one way to +distribute user credentials to {\em compute} nodes (alternatively, you may +prefer to use a central authentication service like LDAP). To import local file-based +credentials, issue the following to enable the {\em synclist} feature and +register user credential files: + +% begin_ohpc_run +% ohpc_comment_header Import files \ref{sec:file_import} +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true,literate={BOSSHORT}{\baseosshort{}}1] +# Add desired credential files to synclist +[sms](*\#*) echo "/etc/passwd -> /etc/passwd" > /var/lib/confluent/public/os/rocky-9.4-x86_64-default +[sms](*\#*) echo "/etc/group -> /etc/group" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default +[sms](*\#*) echo "/etc/shadow -> /etc/shadow" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default +\end{lstlisting} +% \end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/import_confluent_files_slurm.tex b/docs/recipes/install/common/import_confluent_files_slurm.tex new file mode 100644 index 0000000000..daf47be923 --- /dev/null +++ b/docs/recipes/install/common/import_confluent_files_slurm.tex @@ -0,0 +1,23 @@ +\noindent Similarly, to import the +%global Slurm configuration file and the +cryptographic +key +%and associated file permissions +that is required by the {\em munge} +authentication library to be available on every host in the resource management +pool, issue the following: + +% begin_ohpc_run +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +[sms](*\#*) echo "/etc/munge/munge.key -> /etc/munge/munge.key" >> /var/lib/confluent/public/os/rocky-9.4-x86_64-default +\end{lstlisting} +% \end_ohpc_run + +\begin{center} +\begin{tcolorbox}[] +\small +The ``\texttt{nodeapply compute -F}'' command can be used to distribute changes made to any +defined synchronization files on the SMS host. Users wishing to automate this process may +want to consider adding a crontab entry to perform this action at defined intervals. +\end{tcolorbox} +\end{center} \ No newline at end of file diff --git a/docs/recipes/install/common/install_provisioning_confluent_intro.tex b/docs/recipes/install/common/install_provisioning_confluent_intro.tex new file mode 100644 index 0000000000..9b1e7a5e68 --- /dev/null +++ b/docs/recipes/install/common/install_provisioning_confluent_intro.tex @@ -0,0 +1,18 @@ +% -*- mode: latex; fill-column: 120; -*- +With \Confluent{} repository enabled, issue the following install the provisioning +service on {\em master} node + +% begin_ohpc_run +% ohpc_comment_header Add baseline OpenHPC and provisioning services \ref{sec:add_provisioning} +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) (*\install*) lenovo-confluent +[sms](*\#*) (*\install*) tftp-server + +# enable Confluent and its tools for use in current shell +[sms](*\#*) systemctl enable confluent --now +[sms](*\#*) systemctl enable httpd --now +[sms](*\#*) systemctl enable tftp.socket --now +[sms](*\#*) source /etc/profile.d/confluent_env.sh +\end{lstlisting} +% ohpc_validation_newline +% end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/lustre-client-centos-confluent.tex b/docs/recipes/install/common/lustre-client-centos-confluent.tex new file mode 100644 index 0000000000..a72ef95590 --- /dev/null +++ b/docs/recipes/install/common/lustre-client-centos-confluent.tex @@ -0,0 +1,27 @@ +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Enable Optional packages +% ohpc_validation_newline +% ohpc_command if [[ ${enable_lustre_client} -eq 1 ]];then +% ohpc_indent 5 + +% ohpc_validation_comment Install Lustre client on master +\begin{lstlisting}[language=bash,keywords={},upquote=true] + # Add Lustre client software to master host + [sms](*\#*) (*\install*) lustre-client-ohpc + \end{lstlisting} + % end_ohpc_run + + % begin_ohpc_run + % ohpc_validation_comment Enable lustre on compute nodes + \begin{lstlisting}[language=bash,keywords={},upquote=true] + # Enable lustre on compute nodes + [sms](*\#*) (*\chrootinstall*) lustre-client-ohpc + + # Create mount point and file system mount + [sms](*\#*) nodeshell compute mkdir /mnt/lustre + [sms](*\#*) nodeshell compute echo \ + "\""${mgs_fs_name} /mnt/lustre lustre defaults,_netdev,localflock,retry=2 0 0"\"" \>\> /etc/fstab + \end{lstlisting} + % end_ohpc_run + \ No newline at end of file diff --git a/docs/recipes/install/common/lustre-client-post-confluent.tex b/docs/recipes/install/common/lustre-client-post-confluent.tex new file mode 100644 index 0000000000..f555e9d376 --- /dev/null +++ b/docs/recipes/install/common/lustre-client-post-confluent.tex @@ -0,0 +1,34 @@ +\input{common/localflock-tooltip} + +The default underlying network type used by \Lustre{} is {\em tcp}. If your +external \Lustre{} file system is to be mounted using a network type other than +{\em tcp}, additional configuration files are necessary to identify the desired +network type. The example below illustrates creation of modprobe configuration files +instructing \Lustre{} to use an \InfiniBand{} network with the \textbf{o2ib} LNET driver +attached to \texttt{ib0}. Note that these modifications are made to both the +{\em master} host and {\em compute} nodes. + +%x\clearpage +% begin_ohpc_run +% ohpc_validation_comment Enable o2ib for Lustre +\begin{lstlisting}[language=bash,keywords={},upquote=true] +[sms](*\#*) echo "options lnet networks=o2ib(ib0)" >> /etc/modprobe.d/lustre.conf +[sms](*\#*) nodeshell compute echo "\""options lnet networks=o2ib\(ib0\)"\"" \>\> /etc/modprobe.d/lustre.conf +\end{lstlisting} +% end_ohpc_run + +With the \Lustre{} configuration complete, the client can be mounted on the {\em master} +and {\em compute} hosts as follows: +% begin_ohpc_run +% ohpc_validation_comment mount Lustre client on master and compute +\begin{lstlisting}[language=bash,keywords={},upquote=true] +[sms](*\#*) mkdir /mnt/lustre +[sms](*\#*) mount -t lustre -o localflock ${mgs_fs_name} /mnt/lustre + +# Mount on compute nodes +[sms](*\#*) nodeshell compute mount /mnt/lustre +\end{lstlisting} +% ohpc_indent 0 +% ohpc_command fi +% ohpc_validation_newline +% end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/memlimits.tex b/docs/recipes/install/common/memlimits.tex index 3beef4e745..5dfef223ef 100644 --- a/docs/recipes/install/common/memlimits.tex +++ b/docs/recipes/install/common/memlimits.tex @@ -2,7 +2,7 @@ generally necessary to increase the locked memory settings for system users. This can be accomplished by updating the \texttt{/etc/security/limits.conf} file and this should be performed within -the {{\em compute} image and on all job submission hosts. In this recipe, jobs +the {\em compute} image and on all job submission hosts. In this recipe, jobs are submitted from the {\em master} host, and the following commands can be used to update the maximum locked memory settings on both the master host and the compute image: diff --git a/docs/recipes/install/common/memlimits_confluent.tex b/docs/recipes/install/common/memlimits_confluent.tex new file mode 100644 index 0000000000..40f380f5ba --- /dev/null +++ b/docs/recipes/install/common/memlimits_confluent.tex @@ -0,0 +1,23 @@ +In order to utilize \InfiniBand{} or Omni-Path as the underlying high speed interconnect, it is +generally necessary to increase the locked memory settings for system +users. This can be accomplished by updating +the \texttt{/etc/security/limits.conf} file and this should be performed +on all job submission hosts. In this recipe, jobs +are submitted from the {\em master} host, and the following commands can be +used to update the maximum locked memory settings on both the master host and +compute nodes: + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Update memlock settings +\begin{lstlisting}[language=bash,keywords={},upquote=true] +# Update memlock settings on master +[sms](*\#*) perl -pi -e 's/# End of file/\* soft memlock unlimited\n$&/s' /etc/security/limits.conf +[sms](*\#*) perl -pi -e 's/# End of file/\* hard memlock unlimited\n$&/s' /etc/security/limits.conf +# Update memlock settings on compute +[sms](*\#*) nodeshell compute perl -pi -e "'s/# End of file/\* soft memlock unlimited\n$&/s' \ + /etc/security/limits.conf" +[sms](*\#*) nodeshell compute perl -pi -e "'s/# End of file/\* hard memlock unlimited\n$&/s' \ + /etc/security/limits.conf" +\end{lstlisting} +% end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/ohpc-doc.sty b/docs/recipes/install/common/ohpc-doc.sty index f683b292ae..f2b508a78f 100644 --- a/docs/recipes/install/common/ohpc-doc.sty +++ b/docs/recipes/install/common/ohpc-doc.sty @@ -145,3 +145,5 @@ \newcommand{\SLURM}{Slurm} \newcommand{\Warewulf}{Warewulf} \newcommand{\xCAT}{xCAT} +\newcommand{\Confluent}{Confluent} + diff --git a/docs/recipes/install/common/reset_computes_confluent.tex b/docs/recipes/install/common/reset_computes_confluent.tex new file mode 100644 index 0000000000..218fea1043 --- /dev/null +++ b/docs/recipes/install/common/reset_computes_confluent.tex @@ -0,0 +1,44 @@ +Prior to booting the {\em compute} hosts, we configure them to use PXE as their +next boot mode. After the initial PXE, ensuing boots will return to using the default boot device +specified in the BIOS. + +% begin_ohpc_run +% ohpc_comment_header Set nodes to netboot \ref{sec:boot_computes} +\begin{lstlisting}[language=bash,keywords={},upquote=true] +[sms](*\#*) nodesetboot compute network +\end{lstlisting} +% end_ohpc_run + +At this point, the {\em master} server should be able to boot the newly defined +compute nodes. This is done by using the \texttt{nodepower} \Confluent{} command +leveraging IPMI protocol set up during the the {\em compute} node definition +in \S~\ref{sec:confluent_add_nodes}. The following power cycles each of the +desired hosts. + + +% begin_ohpc_run +% ohpc_comment_header Boot compute nodes +\begin{lstlisting}[language=bash,keywords={},upquote=true] +[sms](*\#*) nodepower compute boot +\end{lstlisting} +% end_ohpc_run + +Once kicked off, the boot process should take about 5-10 +minutes (depending on BIOS post times). You can monitor the +provisioning by using the \texttt{nodeconsole} command, which displays serial console +for a selected node. Note that the escape sequence +is \texttt{CTRL-e c .} typed sequentially. + +Successful provisioning can be verified by a parallel command on the compute +nodes. The \Confluent{}-provided +\texttt{nodeshell} command, which uses \Confluent{} node names and groups. +For example, to run a command on +the newly imaged compute hosts using \texttt{nodeshell}, execute the following: + +\begin{lstlisting}[language=bash] +[sms](*\#*) nodeshell compute uptime +c1: 12:56:50 up 14 min, 0 users, load average: 0.00, 0.01, 0.04 +c2: 12:56:50 up 13 min, 0 users, load average: 0.00, 0.02, 0.05 +c3: 12:56:50 up 14 min, 0 users, load average: 0.00, 0.02, 0.05 +c4: 12:56:50 up 14 min, 0 users, load average: 0.00, 0.01, 0.04 +\end{lstlisting} diff --git a/docs/recipes/install/common/slurm_pam_confluent.tex b/docs/recipes/install/common/slurm_pam_confluent.tex new file mode 100644 index 0000000000..2e39469a68 --- /dev/null +++ b/docs/recipes/install/common/slurm_pam_confluent.tex @@ -0,0 +1,14 @@ +An additional optional customization that is recommended is to +restrict \texttt{ssh} access on compute nodes to only allow access by users who +have an active job associated with the node. This can be enabled via the use of +a pluggable authentication module (PAM) provided as part of the \SLURM{} package +installs. To enable this feature on {\em compute} nodes, issue the +following: + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Enable slurm pam module +\begin{lstlisting}[language=bash,keywords={},upquote=true] +[sms](*\#*) nodeshell compute echo "\""account required pam_slurm.so"\"" \>\> /etc/pam.d/sshd +\end{lstlisting} +% end_ohpc_run \ No newline at end of file diff --git a/docs/recipes/install/common/slurm_startup_confluent.tex b/docs/recipes/install/common/slurm_startup_confluent.tex new file mode 100644 index 0000000000..857f2b85ac --- /dev/null +++ b/docs/recipes/install/common/slurm_startup_confluent.tex @@ -0,0 +1,75 @@ +In section \S\ref{sec:basic_install}, the \SLURM{} resource manager was +installed and configured on the {\em master} host. \SLURM{} clients were also +installed, but have not been configured yet. To do so, \SLURM{} and +\texttt{munge} configuration files need to be copied to the nodes. This can be +accomplished as follows: + +% begin_ohpc_run +% ohpc_comment_header Resource Manager Startup \ref{sec:rms_startup} +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) nodersync /etc/slurm/slurm.conf compute:/etc/slurm/slurm.conf +[sms](*\#*) nodersync /etc/munge/munge.key compute:/etc/munge/munge.key +\end{lstlisting} +% end_ohpc_run + + +With \SLURM{} configured, we can now startup the +resource manager services in preparation for running user jobs. Generally, this +is a two-step process that requires starting up the controller daemons on the {\em + master} host and the client daemons on each of the {\em compute} hosts. +%Since the {\em compute} hosts were booted into an image that included the SLURM client +%components, the daemons should already be running on the {\em compute} +%hosts. +Note that \SLURM{} leverages the use of the {\em munge} library to provide +authentication services and this daemon also needs to be running on all hosts +within the resource management pool. +%The munge daemons should already +%be running on the {\em compute} subsystem at this point, +The following commands can be used to startup the necessary services to support +resource management under \SLURM{}. + +%\iftoggle{isCentOS}{\clearpage} + +% begin_ohpc_run +\begin{lstlisting}[language=bash,keywords={}] +# Start munge and slurm controller on master host +[sms](*\#*) systemctl enable munge +[sms](*\#*) systemctl enable slurmctld +[sms](*\#*) systemctl start munge +[sms](*\#*) systemctl start slurmctld + +# Start slurm clients on compute hosts +[sms](*\#*) nodeshell compute systemctl enable munge +[sms](*\#*) nodeshell compute systemctl enable slurmd +[sms](*\#*) nodeshell compute systemctl start munge +[sms](*\#*) nodeshell compute systemctl start slurmd +\end{lstlisting} +% end_ohpc_run + +%%% In the default configuration, the {\em compute} hosts will be initialized in an +%%% {\em unknown} state. To place the hosts into production such that they are +%%% eligible to schedule user jobs, issue the following: + +%%% % begin_ohpc_run +%%% \begin{lstlisting}[language=bash] +%%% [sms](*\#*) scontrol update partition=normal state=idle +%%% \end{lstlisting} +%%% % end_ohpc_run + +After this, check status of the nodes within \SLURM{} by using the +\texttt{sinfo} command. All compute nodes should be in an {\em idle} state +(without asterisk). If the state is reported as {\em unknown}, the following +might help: + +%%% % begin_ohpc_run +\begin{lstlisting}[language=bash] +[sms](*\#*) scontrol update partition=normal state=idle +\end{lstlisting} +%%% % end_ohpc_run + +In case of additional \SLURM{} issues, ensure that the configuration file fits +your hardware and that it is identical across the nodes. Also, verify +that \SLURM{} user id is the same on the SMS and {\em compute} nodes. You may +also consult +\href{https://slurm.schedmd.com/troubleshoot.html}{\color{blue}\SLURM{} +Troubleshooting Guide}. diff --git a/docs/recipes/install/common/upgrade_confluent.tex b/docs/recipes/install/common/upgrade_confluent.tex new file mode 100644 index 0000000000..9b26399d7f --- /dev/null +++ b/docs/recipes/install/common/upgrade_confluent.tex @@ -0,0 +1,127 @@ +\subsection{Upgrading OpenHPC Packages} \label{appendix:upgrade} + + +As newer \OHPC{} releases are made available, users are encouraged to upgrade +their locally installed packages against the latest repository versions to +obtain access to bug fixes and newer component versions. This can be +accomplished with the underlying package manager as \OHPC{} packaging maintains +versioning state across releases. Also, package builds available from the +\OHPC{} repositories have ``\texttt{-ohpc}'' appended to their names so that +wild cards can be used as a simple way to obtain updates. The following general +procedure highlights a method for upgrading existing installations. +When upgrading from a minor release older than v\OHPCVerTree{}, you will first +need to update your local \OHPC{} repository configuration to point against the +v\OHPCVerTree{} release (or update your locally hosted mirror). Refer to +\S\ref{sec:enable_repo} for more details on enabling the latest +repository. In contrast, when upgrading between micro releases on the same +branch (e.g. from v\OHPCVerTree{} to \OHPCVerTree{}.2), there is no need to +adjust local package manager configurations when using the public repository as +rolling updates are pre-configured. + +The initial step, when using a local mirror as described in +\S\ref{sec:enable_repo}, is downloading a new tarball from \texttt{http://build.openhpc.community/dist}. +After this, move (or remove) the old repository directory, unpack +the new tarball and run the \texttt{make\_repo.sh} script. In the following, +substitute \texttt{x.x.x} with the desired version number. + +\begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{7.6}{10}\ttfamily, + literate={VER}{\OHPCVerTree{}}1 {OSREPO}{\OSTree{}}1 {TAG}{\OSTag{}}1 {ARCH}{\arch{}}1 {-}{-}1 + {VER2}{\OHPCVersion{}}1] +[sms](*\#*) wget http://repos.openhpc.community/dist/x.x.x/OpenHPC-x.x.x.OSREPO.ARCH.tar +[sms](*\#*) mv $ohpc_repo_dir "$ohpc_repo_dir"_old +[sms](*\#*) mkdir -p $ohpc_repo_dir +[sms](*\#*) tar xvf OpenHPC-x.x.x.OSREPO.ARCH.tar -C $ohpc_repo_dir +[sms](*\#*) $ohpc_repo_dir/make_repo.sh +\end{lstlisting} + +The {\em compute} hosts will pickup new packages automatically as long as the +repository location on the SMS stays the same. If you prefer to use a different +location, setup repositories on the nodes as described in +\S\ref{sec:enable_repo}. + +The actual update is performed as follows: +\begin{enumerate*} +\item (Optional) Ensure repo metadata is current (on head and compute nodes.) + Package managers will naturally do this on their own over time, + but if you are wanting to access updates immediately after a new release, + the following can be used to sync to the latest. + +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) (*\clean*) +[sms](*\#*) nodeshell compute (*\clean*) +\end{lstlisting} + +\item Upgrade master (SMS) node + +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) (*\upgrade*) "*-ohpc" +\end{lstlisting} + +\item Upgrade packages on compute nodes + +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) nodeshell compute (*\upgrade*) "*-ohpc" +\end{lstlisting} + +\nottoggle{isxCATstateful}{\item Rebuild image(s) + +\iftoggleverb{isWarewulf} +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) wwvnfs --chroot $CHROOT +\end{lstlisting} +\fi + +\iftoggleverb{isxCAT} +\begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{8.0}{10}\ttfamily, + literate={-}{-}1 {BOSVER}{\baseos{}}1 {ARCH}{\arch{}}1] +[sms](*\#*) packimage BOSVER-x86_64-netboot-compute +\end{lstlisting} +\fi +} + +\end{enumerate*} + +\noindent Note that to update running services such as a resource manager a +service restart (or a full reboot) is required. + +%%% \subsubsection{New component variants} +%%% +%%% As newer variants of key compiler/MPI stacks are released, \OHPC{} will +%%% periodically add toolchains enabling the latest variant. To stay consistent +%%% throughout the build hierarchy, minimize recompilation requirements for existing +%%% binaries, and allow for multiple variants to coexist, unique delimiters are +%%% used to distinguish RPM package names and module hierarchy. +%%% +%%% In the case of a fresh install, \OHPC{} recipes default to installation of the +%%% latest toolchains available in a given release branch. However, if upgrading a +%%% previously installed system, administrators can {\em opt-in} to enable new +%%% variants as they become available. To illustrate this point, consider the +%%% previous \OHPC{} 1.3.2 release as an example which contained an {``openmpi''} +%%% MPI variant providing OpenMPI 1.10.x along with runtimes and libraries compiled +%%% with this toolchain. That release also contained the {``llvm4''} compiler +%%% family which has been updated to {``llvm5''} in \OHPC{} 1.3.3. In the case +%%% where an admin would like to enable the newer {``openmpi3''} toolchain, +%%% installation of these additions is simplified with the use of \OHPC{}'s +%%% meta-packages (see Table~\ref{table:groups} in Appendix +%%% \ref{appendix:manifest}). The following example illustrates adding the +%%% complete ``openmpi3'' toolchain. Note that we leverage the convenience +%%% meta-packages containing MPI-dependent builds, and we also update the +%%% modules environment to make it the default. +%%% +%%% \begin{lstlisting}[language=bash,keywords={}] +%%% # Update default environment +%%% [sms](*\#*) (*\remove*) lmod-defaults-gnu7-openmpi-ohpc +%%% [sms](*\#*) (*\install*) lmod-defaults-gnu7-openmpi3-ohpc +%%% +%%% # Install OpenMPI 3.x-compiled meta-packages with dependencies +%%% [sms](*\#*) (*\install*) ohpc-gnu7-perf-tools \ +%%% ohpc-gnu7-io-libs \ +%%% ohpc-gnu7-python-libs \ +%%% ohpc-gnu7-runtimes \ +%%% ohpc-gnu7-openmpi3-parallel-libs +%%% +%%% # Install LLVM/Clang 5.x +%%% [sms](*\#*) (*\install*) llvm5-compilers-ohpc +%%% \end{lstlisting} +%%% +%%% diff --git a/docs/recipes/install/rocky9/input.local.template b/docs/recipes/install/rocky9/input.local.template index acb4bfffc6..aee3b38af5 100644 --- a/docs/recipes/install/rocky9/input.local.template +++ b/docs/recipes/install/rocky9/input.local.template @@ -21,6 +21,9 @@ sms_eth_internal="${sms_eth_internal:-eth1}" # Subnet netmask for internal cluster network internal_netmask="${internal_netmask:-255.255.0.0}" +# ipv4 gateway +ipv4_gateway="${ipv4_gateway:-172.16.0.2}" + # Local ntp server for time synchronization ntp_server="${ntp_server:-0.centos.pool.ntp.org}" @@ -31,12 +34,25 @@ bmc_password="${bmc_password:-unknown}" # Additional time to wait for compute nodes to provision (seconds) provision_wait="${provision_wait:-180}" -# Local domainname for cluster (xCAT recipe only) +# DNS Local domainname for cluster (xCAT and Confluent recipe only) +dns_servers="${dns_servers:-172.30.0.254}" domain_name="${domain_name:-local}" # Path to copy of OS ISO image (xCAT recipe only) iso_path="${iso_path:-}" +# Osdeployment Initialize Options (Confluent recipe only) +initialize_options="${initialize_options:-usklpta}" +deployment_protocols="${deployment_protocols:-firmware}" + + + +# Path to local repository web roots (Confluent reciRocky-9.4-x86_64-dvd.isope only) +ohpc_repo_dir_confluent="${ohpc_repo_dir:-/var/lib/confluent/public/ohpc}" +ohpc_repo_remote_dir="${ohpc_repo_remote_dir:-/confluent-public/ohpc}" +epel_repo_dir_confluent="${epel_repo_dir:-/var/lib/confluent/public/epel}" +epel_repo_remote_dir="${epel_repo_remote_dir:-/confluent-public/epel}" + # Path to local repository web roots (xCAT recipe only) ohpc_repo_dir="${ohpc_repo_dir:-/install/ohpc}" epel_repo_dir="${epel_repo_dir:-/install/epel}" diff --git a/docs/recipes/install/rocky9/x86_64/confluent/slurm/Makefile b/docs/recipes/install/rocky9/x86_64/confluent/slurm/Makefile new file mode 100644 index 0000000000..8483f7035a --- /dev/null +++ b/docs/recipes/install/rocky9/x86_64/confluent/slurm/Makefile @@ -0,0 +1,23 @@ +PAPER := steps.pdf +TEX_SUFS := .aux .log .nav .out .snm .toc .vrb .fdb_latexmk .bbl .blg .fls +GITID := $(shell git describe 2> /dev/null) + +#$(info $$GITID is [${GITID}]) + + +%.pdf:%.tex steps.tex manifest.tex $(wildcard common/*.tex common/ohpc-doc.sty common/base_edition/*.tex common/figures/*.pdf manifest/*.tex) +ifdef GITID + common/vc +else + test -s vc.tex || { echo "vc.tex file does not exist (and this is not a git repo)!"; exit 1; } +endif + latexmk -pdf $< + +all: $(PAPER) + +clean: + $(RM) $(foreach suf, ${TEX_SUFS}, $(PAPER:.pdf=${suf})) +ifdef GITID + $(RM) vc.tex +endif + $(RM) $(PAPER) diff --git a/docs/recipes/install/rocky9/x86_64/confluent/slurm/common b/docs/recipes/install/rocky9/x86_64/confluent/slurm/common new file mode 120000 index 0000000000..3da9bd8324 --- /dev/null +++ b/docs/recipes/install/rocky9/x86_64/confluent/slurm/common @@ -0,0 +1 @@ +../../../../common \ No newline at end of file diff --git a/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest b/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest new file mode 120000 index 0000000000..29057155d1 --- /dev/null +++ b/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest @@ -0,0 +1 @@ +../../warewulf/slurm/manifest \ No newline at end of file diff --git a/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest.tex b/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest.tex new file mode 120000 index 0000000000..9417c314c9 --- /dev/null +++ b/docs/recipes/install/rocky9/x86_64/confluent/slurm/manifest.tex @@ -0,0 +1 @@ +../../warewulf/slurm/manifest.tex \ No newline at end of file diff --git a/docs/recipes/install/rocky9/x86_64/confluent/slurm/steps.tex b/docs/recipes/install/rocky9/x86_64/confluent/slurm/steps.tex new file mode 100644 index 0000000000..11513afbb3 --- /dev/null +++ b/docs/recipes/install/rocky9/x86_64/confluent/slurm/steps.tex @@ -0,0 +1,349 @@ +\documentclass[letterpaper]{article} +\usepackage{common/ohpc-doc} +\setcounter{secnumdepth}{5} +\setcounter{tocdepth}{5} + +% Include git variables +\input{vc.tex} + +% Define Base OS and other local macros +\newcommand{\baseOS}{Rocky 9.4} +\newcommand{\OSRepo}{Rocky\_9.4} +\newcommand{\OSTree}{EL\_9} +\newcommand{\OSTag}{el9} +\newcommand{\baseos}{rocky9.4} +\newcommand{\baseosshort}{rocky9} +\newcommand{\provisioner}{confluent} +\newcommand{\provheader}{\provisioner{}} +\newcommand{\rms}{SLURM} +\newcommand{\rmsshort}{slurm} +\newcommand{\arch}{x86\_64} + +% Define package manager commands +\newcommand{\pkgmgr}{dnf} +\newcommand{\addrepo}{wget -P /etc/yum.repos.d} +\newcommand{\chrootaddrepo}{wget -P \$CHROOT/etc/yum.repos.d} +\newcommand{\clean}{dnf clean expire-cache} +\newcommand{\chrootclean}{dnf --installroot=\$CHROOT clean expire-cache} +\newcommand{\install}{dnf -y install} +\newcommand{\chrootinstall}{nodeshell compute dnf -y install} +\newcommand{\groupinstall}{dnf -y groupinstall} +\newcommand{\groupchrootinstall}{nodeshell compute dnf -y groupinstall} +\newcommand{\remove}{dnf -y remove} +\newcommand{\upgrade}{dnf -y upgrade} +\newcommand{\chrootupgrade}{dnf -y --installroot=\$CHROOT upgrade} +\newcommand{\tftppkg}{syslinux-tftpboot} +\newcommand{\beegfsrepo}{https://www.beegfs.io/release/beegfs\_7.2.1/dists/beegfs-rhel8.repo} + +% boolean for os-specific formatting +\toggletrue{isCentOS} +\toggletrue{isCentOS_ww_slurm_x86} +\toggletrue{isSLURM} +\toggletrue{isx86} +\toggletrue{isCentOS_x86} + +\begin{document} +\graphicspath{{common/figures/}} +\thispagestyle{empty} + +% Title Page +\input{common/title} +% Disclaimer +\input{common/legal} + +\newpage +\tableofcontents +\newpage + +% Introduction -------------------------------------------------- + +\section{Introduction} \label{sec:introduction} +\input{common/install_header} +\input{common/intro} \\ + +\input{common/base_edition/edition} +\input{common/audience} +\input{common/requirements} +\input{common/inputs} + + + +% Base Operating System -------------------------------------------- + +\section{Install Base Operating System (BOS)} +\input{common/bos} + +%\clearpage +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Disable firewall +\begin{lstlisting}[language=bash,keywords={}] +[sms](*\#*) systemctl disable firewalld +[sms](*\#*) systemctl stop firewalld +\end{lstlisting} +% end_ohpc_run + +% ------------------------------------------------------------------ + +\section{Install \Confluent{} and Provision Nodes with BOS} \label{sec:provision_compute_bos} +\input{common/confluent_compute_bos_intro} + +\subsection{Enable \Confluent{} repository for local use} \label{sec:enable_confluent} +\input{common/enable_confluent_repo} + +\subsection{Add provisioning services on {\em master} node} \label{sec:add_provisioning} +\input{common/install_provisioning_confluent_intro} + +\vspace*{-0.15cm} +\subsection{Complete basic \Confluent{} setup for {\em master} node} \label{sec:setup_confluent} +\input{common/confluent_setup} + +\subsection{Define {\em compute} image for provisioning} +\input{common/confluent_init_os_images_rocky} + +\vspace*{0.9cm} +\subsection{Add compute nodes into \Confluent{} database} \label{sec:confluent_add_nodes} +\input{common/add_confluent_hosts_intro} + +%\vspace*{-0.25cm} +\subsection{Boot compute nodes} \label{sec:boot_computes} +\input{common/reset_computes_confluent} + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Check if provisioning is in progress +% ohpc_validation_newline +% ohpc_command while true; do +% ohpc_command deployment_pending=false +% ohpc_command while read line; do +% ohpc_command dp=$(echo $line | cut -d ":" -f 2) +% ohpc_command if [ "$dp" != " completed" ]; +% ohpc_command then +% ohpc_command deployment_pending=true +% ohpc_command fi +% ohpc_command done <<< `nodedeploy compute` +% ohpc_command if $deployment_pending; +% ohpc_command then +% ohpc_command echo "deployment still pending" +% ohpc_command deployment_pending=false +% ohpc_command sleep 60 +% ohpc_command else +% ohpc_command break +% ohpc_command fi +% ohpc_command done +% end_ohpc_run + + +\section{Install \OHPC{} Components} \label{sec:basic_install} +\input{common/install_ohpc_components_intro} + +\subsection{Enable \OHPC{} repository for local use} \label{sec:enable_repo} +\input{common/enable_local_ohpc_repo_confluent} + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Verify OpenHPC repository has been enabled before proceeding +% ohpc_validation_newline +% ohpc_command dnf repolist | grep -q OpenHPC +% ohpc_command if [ $? -ne 0 ];then +% ohpc_command echo "Error: OpenHPC repository must be enabled locally" +% ohpc_command exit 1 +% ohpc_command fi +% end_ohpc_run + +\input{common/rocky_repos} + + +% begin_ohpc_run +\begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{8.0}{10}\ttfamily,literate={ARCH}{\arch{}}1 {-}{-}1] +[sms](*\#*) (*\install*) epel-release + +# Enable crb on sms +[sms](*\#*) /usr/bin/crb enable +\end{lstlisting} +% end_ohpc_run + +Now \OHPC{} packages can be installed. To add the base package on the SMS +issue the following +% begin_ohpc_run +\begin{lstlisting}[language=bash,keywords={},basicstyle=\fontencoding{T1}\fontsize{8.0}{10}\ttfamily,literate={ARCH}{\arch{}}1 {-}{-}1] +[sms](*\#*) (*\install*) ohpc-base +\end{lstlisting} +% end_ohpc_run + +\input{common/automation} + +\subsection{Setup time synchronization service on {\em master} node} \label{sec:add_ntp} +\input{common/time} + +%\input{common/enable_pxe} + + +\subsection{Add resource management services on {\em master} node} \label{sec:add_rm} +\input{common/install_slurm} + +\subsection{Optionally add \InfiniBand{} support services on {\em master} node} \label{sec:add_ofed} +\input{common/ibsupport_sms_centos} + +\subsection{Optionally add \OmniPath{} support services on {\em master} node} \label{sec:add_opa} +\input{common/opasupport_sms_centos} + +\vspace*{0.2cm} +\subsubsection{Add \OHPC{} components} \label{sec:add_components} +\input{common/add_to_compute_confluent_intro} + +%\newpage +% begin_ohpc_run +% ohpc_validation_comment Add OpenHPC components to compute instance +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Add Slurm client support meta-package +[sms](*\#*) (*\chrootinstall*) ohpc-slurm-client + +# Add Network Time Protocol (NTP) support +[sms](*\#*) (*\chrootinstall*) ntp + +# Add kernel drivers +[sms](*\#*) (*\chrootinstall*) kernel + +# Enable crb +[sms](*\#*) nodeshell compute /usr/bin/crb enable + +# Include nfs-utils +[sms](*\#*) (*\chrootinstall*) nfs-utils + +# Include modules user environment +[sms](*\#*) (*\chrootinstall*) lmod-ohpc +\end{lstlisting} + + +% end_ohpc_run + +% ohpc_comment_header Optionally add InfiniBand support services in compute node image \ref{sec:add_components} +% ohpc_command if [[ ${enable_ib} -eq 1 ]];then +% ohpc_indent 5 +\begin{lstlisting}[language=bash,literate={-}{-}1,keywords={},upquote=true] +# Optionally add IB support and enable +[sms](*\#*) (*\groupchrootinstall*) "InfiniBand Support" +\end{lstlisting} +% ohpc_indent 0 +% ohpc_command fi +% end_ohpc_run + +\vspace*{-0.25cm} +\subsubsection{Customize system configuration} \label{sec:master_customization} +\input{common/confluent_customize} + +% Additional commands when additional computes are requested + +% begin_ohpc_run +% ohpc_validation_newline +% ohpc_validation_comment Update basic slurm configuration if additional computes defined +% ohpc_validation_comment This is performed on the SMS, nodes will pick it up config file is copied there later +% ohpc_command if [ ${num_computes} -gt 4 ];then +% ohpc_command perl -pi -e "s/^NodeName=(\S+)/NodeName=${compute_prefix}[1-${num_computes}]/" /etc/slurm/slurm.conf +% ohpc_command perl -pi -e "s/^PartitionName=normal Nodes=(\S+)/PartitionName=normal Nodes=${compute_prefix}[1-${num_computes}]/" /etc/slurm/slurm.conf +% ohpc_command fi +% end_ohpc_run + +%\clearpage +\subsubsection{Additional Customization ({\em optional})} \label{sec:addl_customizations} +\input{common/compute_customizations_intro} + +\paragraph{Increase locked memory limits} +\input{common/memlimits_confluent} + +\paragraph{Enable ssh control via resource manager} +\input{common/slurm_pam_confluent} + +\paragraph{Add \Lustre{} client} \label{sec:lustre_client} +\input{common/lustre-client} +\input{common/lustre-client-centos-confluent} +\input{common/lustre-client-post-confluent} + +\vspace*{0.4cm} + +\paragraph{Add \clustershell{}} +\input{common/clustershell} + +\paragraph{Add \genders{}} +\input{common/genders} + +\paragraph{Add Magpie} +\input{common/magpie} + +\paragraph{Add \conman{}} \label{sec:add_conman} +\input{common/conman} + +\paragraph{Add \nhc{}} \label{sec:add_nhc} +\input{common/nhc} +\input{common/nhc_slurm} + +%\subsubsection{Identify files for synchronization} \label{sec:file_import} +%\input{common/import_confluent_files} +%\input{common/import_confluent_files_slurm} + +%%%\subsubsection{Optional kernel arguments} \label{sec:optional_kargs} +%%%\input{common/conman_post} + +\section{Install \OHPC{} Development Components} +\input{common/dev_intro} + +%\vspace*{-0.15cm} +%\clearpage +\subsection{Development Tools} \label{sec:install_dev_tools} +\input{common/dev_tools} + +\vspace*{-0.15cm} +\subsection{Compilers} \label{sec:install_compilers} +\input{common/compilers} + +%\clearpage +\subsection{MPI Stacks} \label{sec:mpi} +\input{common/mpi_slurm} + +\subsection{Performance Tools} \label{sec:install_perf_tools} +\input{common/perf_tools} + +\subsection{Setup default development environment} +\input{common/default_dev} + +%\vspace*{0.2cm} +\subsection{3rd Party Libraries and Tools} \label{sec:3rdparty} +\input{common/third_party_libs_intro} + +\input{common/third_party_libs} +\input{common/third_party_mpi_libs_x86} + +\vspace*{.6cm} +\subsection{Optional Development Tool Builds} \label{sec:3rdparty_intel} +\input{common/oneapi_enabled_builds_slurm} + +\section{Resource Manager Startup} \label{sec:rms_startup} +\input{common/slurm_startup_confluent} + +\section{Run a Test Job} \label{sec:test_job} +\input{common/confluent_slurm_test_job} + +\clearpage +\appendix +{\bf \LARGE \centerline{Appendices}} \vspace*{0.2cm} + +\addcontentsline{toc}{section}{Appendices} +\renewcommand{\thesubsection}{\Alph{subsection}} + +\input{common/automation_appendix} +\input{common/upgrade_confluent} +\input{common/test_suite} +\input{common/customization_appendix_centos} +\input{manifest} +\input{common/signature} + + +\end{document} + + + + + + +