From 40fd469237f5d492da45bcbed11a83633e8b8760 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 14:52:41 -0700 Subject: [PATCH 01/13] Added .gitignore --- .gitignore | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..287a2f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + From 93644e10dbbd5eb0ffb08a97f0ab2502b6051574 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 14:56:23 -0700 Subject: [PATCH 02/13] Requiring python3.9; which is the highest Python version on `klone` that is still receiving support and patches --- pyproject.toml | 3 ++- src/hyakvnc.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 254679a..1498c6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,10 @@ version = "2.0" description = "Create and manage VNC sessions on HYAK Klone." authors = [ {name = "Hansem Ro", email = "hansem7@uw.edu"}, + {name = "Altan Orhon", email = "altan@uw.edu"}, ] dependencies = [] -requires-python = ">=3.6" +requires-python = ">=3.9" license = {text = "MIT"} readme = "README.md" diff --git a/src/hyakvnc.py b/src/hyakvnc.py index afdcdfa..92d6ca3 100755 --- a/src/hyakvnc.py +++ b/src/hyakvnc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python39 # SPDX-License-Identifier: MIT # Copyright (c) 2023 Hansem Ro From 0c6faef19940bba38d2ae4490b0fd1ccf5bc281c Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 15:15:36 -0700 Subject: [PATCH 03/13] Added pre-commit --- .pre-commit-config.yaml | 15 +++++++++++++++ README.md | 9 +++++++++ pyproject.toml | 11 +++++++---- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 .pre-commit-config.yaml mode change 100755 => 100644 README.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f425407 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.7.0 + hooks: + - id: black + language_version: python3.9 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/README.md b/README.md old mode 100755 new mode 100644 index dfc52ab..25c1c87 --- a/README.md +++ b/README.md @@ -46,6 +46,15 @@ python3 -m pip install --user . If successful, then `hyakvnc` should be installed to `~/.local/bin/`. +#### Optional dependencies for development + +The optional dependency group`[dev]` in `pyroject.toml` includes dependencies useful for development, including [pre-commit](https://pre-commit.com/) hooks that run in order to commit to the `git` repository. +These apply various checks, including running the `black` code formatter before the commit takes place. + +To ensure `pre-commit` is installed, run: +```bash +pip install '.[precommit]' +`` ### General usage `hyakvnc` is command-line tool that only works while on the login node. diff --git a/pyproject.toml b/pyproject.toml index 1498c6a..9a79675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,17 @@ name = "hyakvnc" version = "2.0" description = "Create and manage VNC sessions on HYAK Klone." authors = [ - {name = "Hansem Ro", email = "hansem7@uw.edu"}, - {name = "Altan Orhon", email = "altan@uw.edu"}, + { name = "Hansem Ro", email = "hansem7@uw.edu" }, + { name = "Altan Orhon", email = "altan@uw.edu" }, ] dependencies = [] requires-python = ">=3.9" -license = {text = "MIT"} +license = { text = "MIT" } readme = "README.md" +[project.optional-dependencies] +dev = ["pre-commit"] + [build-system] requires = ["pdm-pep517"] build-backend = "pdm.pep517.api" @@ -22,4 +25,4 @@ where = ["src"] hyakvnc = "hyakvnc:main" [project.urls] -homepage = "" +homepage = "https://github.com/uw-psych/hyakvnc" From 90f7e5f3c9429a43d0c9ee504ca531c956de3cf3 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 15:17:29 -0700 Subject: [PATCH 04/13] Added GitHub action for Black --- .github/workflows/black.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000..8d00eec --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,12 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + with: + src: "./src" From 71c05f457a981a953f29fe99ea336f0dd3061498 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 15:54:53 -0700 Subject: [PATCH 05/13] Corrected README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 25c1c87..39394f7 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ To ensure `pre-commit` is installed, run: ```bash pip install '.[precommit]' `` + ### General usage `hyakvnc` is command-line tool that only works while on the login node. From acefebcafdfac481457acb4f11e5577df5f854c4 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 16:04:14 -0700 Subject: [PATCH 06/13] Added flake8 --- .flake8 | 3 +++ .pre-commit-config.yaml | 4 ++++ pyproject.toml | 5 +++++ 3 files changed, 12 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..9e80a2e --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +extend-ignore = E203 +max-line-length = 100 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f425407..e8b9d67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,10 @@ repos: hooks: - id: black language_version: python3.9 + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 9a79675..7159f0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,3 +26,8 @@ hyakvnc = "hyakvnc:main" [project.urls] homepage = "https://github.com/uw-psych/hyakvnc" + +[tool.black] +line-length = 100 +target-version = ['py39'] +include = '\.pyi?$' From 4d0e720257fd092f3f40f4692da7f5552f52f519 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 16:12:16 -0700 Subject: [PATCH 07/13] Removed flake8 pre-commit until compliant --- .pre-commit-config.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8b9d67..f425407 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,10 +4,6 @@ repos: hooks: - id: black language_version: python3.9 - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: From d74369d57e5478dd2b502322fb6c38171d183e08 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 16:16:03 -0700 Subject: [PATCH 08/13] Formatted --- src/hyakvnc.py | 510 ++++++++++++++++++++++++++++++------------------- 1 file changed, 309 insertions(+), 201 deletions(-) diff --git a/src/hyakvnc.py b/src/hyakvnc.py index 92d6ca3..67ecf01 100755 --- a/src/hyakvnc.py +++ b/src/hyakvnc.py @@ -119,15 +119,15 @@ # - xfce4 # - tigervnc with vncserver -import argparse # for argument handling -import logging # for debug logging -import time # for sleep -import signal # for signal handling +import argparse # for argument handling +import logging # for debug logging +import time # for sleep +import signal # for signal handling import glob import pwd -import os # for path, file/dir checking, hostname -import subprocess # for running shell commands -import re # for regex +import os # for path, file/dir checking, hostname +import subprocess # for running shell commands +import re # for regex # tasks: # - [x] user arguments to control hours @@ -184,6 +184,7 @@ if APPTAINER_BINDPATH is None: APPTAINER_BINDPATH = "/tmp,$HOME,$PWD,/gscratch,/opt,/:/hyak_root,/sw,/mmfs1" + class Node: """ The Node class has the following initial data: bool: debug, string: name. @@ -198,7 +199,7 @@ def __init__(self, name, sing_container, xstartup, debug=False): self.sing_container = os.path.abspath(sing_container) self.xstartup = os.path.abspath(xstartup) - def get_sing_exec(self, args=''): + def get_sing_exec(self, args=""): """ Added before command to execute inside an apptainer (singularity) container. @@ -209,6 +210,7 @@ def get_sing_exec(self, args=''): """ return f"{APPTAINER_BIN} exec {args} -B {APPTAINER_BINDPATH} {self.sing_container}" + class SubNode(Node): """ The SubNode class specifies a node requested via Slurm (also known as work @@ -245,7 +247,7 @@ def print_props(self): if self.debug: logging.debug(msg) - def run_command(self, command:str, timeout=None): + def run_command(self, command: str, timeout=None): """ Run command (with arguments) on subnode @@ -282,11 +284,11 @@ def list_pids(self): if "PID" in line: pass elif re.match("[0-9]+", line): - pid = int(line.split(' ', 1)[0]) + pid = int(line.split(" ", 1)[0]) ret.append(pid) return ret - def check_pid(self, pid:int): + def check_pid(self, pid: int): """ Returns True if given pid is active in job_id and False otherwise. """ @@ -301,7 +303,7 @@ def get_vnc_pid(self, hostname, display_number): hostname = self.hostname if display_number is None: display_number = self.vnc_display_number - assert(hostname is not None) + assert hostname is not None if display_number is not None: filepaths = glob.glob(os.path.expanduser(f"~/.vnc/{hostname}*:{display_number}.pid")) for path in filepaths: @@ -316,8 +318,8 @@ def check_vnc(self): """ Returns True if VNC session is active and False otherwise. """ - assert(self.name is not None) - assert(self.job_id is not None) + assert self.name is not None + assert self.job_id is not None pid = self.get_vnc_pid(self.hostname, self.vnc_display_number) if pid is None: pid = self.get_vnc_pid(self.name, self.vnc_display_number) @@ -327,7 +329,7 @@ def check_vnc(self): logging.debug(f"check_vnc: Checking VNC PID {pid}") return self.check_pid(pid) - def start_vnc(self, display_number=None, extra_args='', timeout=20): + def start_vnc(self, display_number=None, extra_args="", timeout=20): """ Starts VNC session @@ -349,19 +351,22 @@ def start_vnc(self, display_number=None, extra_args='', timeout=20): # get display number and port number while proc.poll() is None: - line = str(proc.stdout.readline(), 'utf-8').strip() + line = str(proc.stdout.readline(), "utf-8").strip() if line is not None: if self.debug: logging.debug(f"start_vnc: {line}") if "desktop" in line: # match against the following pattern: - #New 'n3000.hyak.local:1 (hansem7)' desktop at :1 on machine n3000.hyak.local - #New 'n3000.hyak.local:6 (hansem7)' desktop is n3000.hyak.local:6 - pattern = re.compile(""" + # New 'n3000.hyak.local:1 (hansem7)' desktop at :1 on machine n3000.hyak.local + # New 'n3000.hyak.local:6 (hansem7)' desktop is n3000.hyak.local:6 + pattern = re.compile( + """ (New\s) (\'([^:]+:(?P[0-9]+))\s([^\s]+)\s) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = re.match(pattern, line) assert match is not None self.vnc_display_number = int(match.group("display_number")) @@ -370,12 +375,12 @@ def start_vnc(self, display_number=None, extra_args='', timeout=20): logging.debug(f"Obtained display number: {self.vnc_display_number}") logging.debug(f"Obtained VNC port: {self.vnc_port}") else: - print('\x1b[1;32m' + "Success" + '\x1b[0m') + print("\x1b[1;32m" + "Success" + "\x1b[0m") return True if self.debug: logging.error("Failed to start vnc session (Timeout/?)") else: - print('\x1b[1;31m' + "Timed out" + '\x1b[0m') + print("\x1b[1;31m" + "Timed out" + "\x1b[0m") return False def list_vnc(self): @@ -385,9 +390,9 @@ def list_vnc(self): active = list() stale = list() cmd = f"{self.get_sing_exec()} vncserver -list" - #TigerVNC server sessions: + # TigerVNC server sessions: # - #X DISPLAY # PROCESS ID + # X DISPLAY # PROCESS ID #:1 7280 (stale) #:12 29 (stale) #:2 83704 (stale) @@ -405,9 +410,9 @@ def list_vnc(self): stale.append(display_number) else: active.append(display_number) - return (active,stale) + return (active, stale) - def __remove_files__(self, filepaths:list): + def __remove_files__(self, filepaths: list): """ Removes files on subnode and returns True on success and False otherwise. @@ -421,7 +426,7 @@ def __remove_files__(self, filepaths:list): cmd = f"{cmd} &> /dev/null" if self.debug: logging.debug(f"Calling ssh {self.hostname} {cmd}") - return subprocess.call(['ssh', self.hostname, cmd]) == 0 + return subprocess.call(["ssh", self.hostname, cmd]) == 0 def __listdir__(self, dirpath): """ @@ -429,10 +434,13 @@ def __listdir__(self, dirpath): """ ret = list() cmd = f"test -d {dirpath} && ls -al {dirpath} | tail -n+4" - pattern = re.compile(""" + pattern = re.compile( + """ ([^\s]+\s+){8} (?P.*) - """, re.VERBOSE) + """, + re.VERBOSE, + ) proc = self.run_command(cmd) while proc.poll() is None: line = str(proc.stdout.readline(), "utf-8").strip() @@ -447,7 +455,7 @@ def kill_vnc(self, display_number=None): Kill specified VNC session with given display number or all VNC sessions. """ if display_number is None: - active,stale = self.list_vnc() + active, stale = self.list_vnc() for entry in active: if self.debug: logging.debug(f"kill_vnc: active entry: {entry}") @@ -485,10 +493,10 @@ def kill_vnc(self, display_number=None): while proc.poll() is None: line = str(proc.stdout.readline(), "utf-8").strip() # Failed attempt: - #Can't kill '29': Operation not permitted - #Killing Xtigervnc process ID 29... + # Can't kill '29': Operation not permitted + # Killing Xtigervnc process ID 29... # On successful attempt: - #Killing Xtigervnc process ID 29... success! + # Killing Xtigervnc process ID 29... success! if self.debug: logging.debug(f"kill_vnc: {line}") if "success" in line: @@ -508,6 +516,7 @@ def kill_vnc(self, display_number=None): socket_file = f"/tmp/.X11-unix/{display_number}" self.__remove_files__([socket_file]) + class LoginNode(Node): """ The LoginNode class specifies Hyak login node for its Slurm and SSH @@ -527,7 +536,7 @@ def find_nodes(self, job_name="vnc"): command = f"squeue | grep {os.getlogin()} | grep {job_name}" proc = self.run_command(command) while True: - line = str(proc.stdout.readline(), 'utf-8') + line = str(proc.stdout.readline(), "utf-8") if self.debug: logging.debug(f"find_nodes: {line}") if not line: @@ -541,13 +550,16 @@ def find_nodes(self, job_name="vnc"): # 870400 compute-h vnc hansem7 PD 0:00 1 (Resources) # or the following if a node failed to be acquired and needs to be killed # 984669 compute-h vnc hansem7 PD 0:00 1 (QOSGrpCpuLimit) - pattern = re.compile(""" + pattern = re.compile( + """ (\s+) (?P[0-9]+) (\s+[^ ]+){6} (\s+) (?P[^\s]+) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = pattern.match(line) assert match is not None name = match.group("subnode_name") @@ -573,7 +585,7 @@ def find_nodes(self, job_name="vnc"): elif self.debug: msg = f"Found active subnode {name} with job ID {job_id}" logging.debug(msg) - tmp = SubNode(name, job_id, '', '', self.debug) + tmp = SubNode(name, job_id, "", "", self.debug) ret.add(tmp) return None @@ -590,7 +602,7 @@ def set_vnc_password(self): cmd = f"{self.get_sing_exec()} vncpasswd" self.call_command(cmd) - def call_command(self, command:str): + def call_command(self, command: str): """ Call command (with arguments) on login node (to allow user interaction). @@ -627,9 +639,21 @@ def run_command(self, command): if isinstance(command, list): return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) elif isinstance(command, str): - return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - def reserve_node(self, res_time=3, timeout=10, cpus=8, gpus="0", mem="16G", partition="compute-hugemem", account="ece", job_name="vnc"): + return subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + + def reserve_node( + self, + res_time=3, + timeout=10, + cpus=8, + gpus="0", + mem="16G", + partition="compute-hugemem", + account="ece", + job_name="vnc", + ): """ Reserves a node and waits until the node has been acquired. @@ -646,15 +670,24 @@ def reserve_node(self, res_time=3, timeout=10, cpus=8, gpus="0", mem="16G", part Returns SubNode object if it has been acquired successfully and None otherwise. """ - cmd = ["timeout", str(timeout), "salloc", - "-J", job_name, - "--no-shell", - "-p", partition, - "-A", account, - "-t", f"{res_time}:00:00", - "--mem=" + mem, - "--gpus=" + gpus, - "-c", str(cpus)] + cmd = [ + "timeout", + str(timeout), + "salloc", + "-J", + job_name, + "--no-shell", + "-p", + partition, + "-A", + account, + "-t", + f"{res_time}:00:00", + "--mem=" + mem, + "--gpus=" + gpus, + "-c", + str(cpus), + ] proc = self.run_command(cmd) alloc_stat = False @@ -678,34 +711,42 @@ def __reserve_node_irq_handler__(signalNumber, frame): signal.signal(signal.SIGINT, __reserve_node_irq_handler__) signal.signal(signal.SIGTSTP, __reserve_node_irq_handler__) - print(f"Allocating node with {cpus} CPU(s), {gpus.split(':').pop()} GPU(s), and {mem} RAM for {res_time} hours...") + print( + f"Allocating node with {cpus} CPU(s), {gpus.split(':').pop()} GPU(s), and {mem} RAM for {res_time} hours..." + ) while proc.poll() is None and not alloc_stat: print("...") - line = str(proc.stdout.readline(), 'utf-8').strip() + line = str(proc.stdout.readline(), "utf-8").strip() if self.debug: msg = f"reserve_node: {line}" logging.debug(msg) if "Pending" in line or "Granted" in line: # match against pattern: - #salloc: Pending job allocation 864875 - #salloc: Granted job allocation 864875 - pattern = re.compile(""" + # salloc: Pending job allocation 864875 + # salloc: Granted job allocation 864875 + pattern = re.compile( + """ (salloc:\s) ((Granted)|(Pending)) (\sjob\sallocation\s) (?P[0-9]+) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = pattern.match(line) if match is not None: subnode_job_id = match.group("job_id") elif "are ready for job" in line: # match against pattern: - #salloc: Nodes n3000 are ready for job - pattern = re.compile(""" + # salloc: Nodes n3000 are ready for job + pattern = re.compile( + """ (salloc:\sNodes\s) (?P[ngz][0-9]{4}) (\sare\sready\sfor\sjob) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = pattern.match(line) if match is not None: subnode_name = match.group("node_name") @@ -728,7 +769,9 @@ def __reserve_node_irq_handler__(signalNumber, frame): tmp_nodes = self.find_nodes(job_name) for tmp_node in tmp_nodes: if self.debug: - logging.debug(f"reserve_node: fallback: Checking {tmp_node.name} with Job ID {tmp_node.job_id}") + logging.debug( + f"reserve_node: fallback: Checking {tmp_node.name} with Job ID {tmp_node.job_id}" + ) if tmp_node.job_id == subnode_job_id: if self.debug: logging.debug(f"reserve_node: fallback: Match found") @@ -749,7 +792,7 @@ def __reserve_node_irq_handler__(signalNumber, frame): self.subnode = SubNode(subnode_name, subnode_job_id, self.sing_container, self.xstartup) return self.subnode - def cancel_job(self, job_id:int): + def cancel_job(self, job_id: int): """ Cancel specified job ID @@ -761,9 +804,9 @@ def cancel_job(self, job_id:int): if self.debug: logging.debug(msg) proc = self.run_command(["scancel", str(job_id)]) - print(str(proc.communicate()[0], 'utf-8')) + print(str(proc.communicate()[0], "utf-8")) - def check_port(self, port:int): + def check_port(self, port: int): """ Returns True if port is unused and False if used. """ @@ -772,7 +815,7 @@ def check_port(self, port:int): cmd = f"netstat -ant | grep LISTEN | grep {port}" proc = self.run_command(cmd) while proc.poll() is None: - line = str(proc.stdout.readline(), 'utf-8').strip() + line = str(proc.stdout.readline(), "utf-8").strip() if self.debug: logging.debug(f"netstat line: {line}") if str(port) in line: @@ -784,13 +827,13 @@ def get_port(self): Returns unused port number if found and None if not found. """ # 300 is arbitrary limit - for i in range(0,300): + for i in range(0, 300): port = BASE_VNC_PORT + i if self.check_port(port): return port return None - def create_port_forward(self, login_port:int, subnode_port:int): + def create_port_forward(self, login_port: int, subnode_port: int): """ Port forward between login node and subnode @@ -827,13 +870,13 @@ def create_port_forward(self, login_port:int, subnode_port:int): msg = f"Successfully created port forward" logging.info(msg) else: - print('\x1b[1;32m' + "Success" + '\x1b[0m') + print("\x1b[1;32m" + "Success" + "\x1b[0m") return True if self.debug: msg = f"Error: Failed to create port forward" logging.error(msg) else: - print('\x1b[1;31m' + "Failed" + '\x1b[0m') + print("\x1b[1;31m" + "Failed" + "\x1b[0m") return False def get_port_forwards(self, nodes=None): @@ -865,23 +908,26 @@ def get_port_forwards(self, nodes=None): cmd = f"ps x | grep ssh | grep {node.name}" proc = self.run_command(cmd) while proc.poll() is None: - line = str(proc.stdout.readline(), 'utf-8').strip() + line = str(proc.stdout.readline(), "utf-8").strip() if cmd not in line: # Match against pattern: - #1974577 ? Ss 0:20 ssh -N -f -L 5902:127.0.0.1:5902 n3065.hyak.local - pattern = re.compile(""" + # 1974577 ? Ss 0:20 ssh -N -f -L 5902:127.0.0.1:5902 n3065.hyak.local + pattern = re.compile( + """ ([^\s]+(\s)+){4} (ssh\s-N\s-f\s-L\s(?P[0-9]+):127.0.0.1:(?P[0-9]+)) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = re.match(pattern, line) if match is not None: ln_port = int(match.group("ln_port")) sn_port = int(match.group("sn_port")) - port_map.update({sn_port:ln_port}) - node_port_map.update({node.name:port_map}) + port_map.update({sn_port: ln_port}) + node_port_map.update({node.name: port_map}) return node_port_map - def get_job_port_forward(self, job_id:int, node_name:str, node_port_map:dict): + def get_job_port_forward(self, job_id: int, node_name: str, node_port_map: dict): """ Returns tuple containing LoginNodePort and SubNodePort for given job ID and node_name. Returns None on failure. @@ -889,11 +935,13 @@ def get_job_port_forward(self, job_id:int, node_name:str, node_port_map:dict): if self.get_time_left(job_id) is not None: port_map = node_port_map[node_name] if port_map is not None: - subnode = SubNode(node_name, job_id, self.sing_container, '', self.debug) + subnode = SubNode(node_name, job_id, self.sing_container, "", self.debug) for vnc_port in port_map.keys(): display_number = vnc_port - BASE_VNC_PORT if self.debug: - logging.debug(f"get_job_port_forward: Checking job {job_id} vnc_port {vnc_port}") + logging.debug( + f"get_job_port_forward: Checking job {job_id} vnc_port {vnc_port}" + ) # get PID from VNC pid file pid = subnode.get_vnc_pid(subnode.name, display_number) if pid is None: @@ -902,11 +950,13 @@ def get_job_port_forward(self, job_id:int, node_name:str, node_port_map:dict): # if PID is active, then we have a hit for a specific job if pid is not None and subnode.check_pid(pid): if self.debug: - logging.debug(f"get_job_port_forward: {job_id} has vnc_port {vnc_port} and login node port {port_map[vnc_port]}") - return (vnc_port,port_map[vnc_port]) + logging.debug( + f"get_job_port_forward: {job_id} has vnc_port {vnc_port} and login node port {port_map[vnc_port]}" + ) + return (vnc_port, port_map[vnc_port]) return None - def get_time_left(self, job_id:int, job_name="vnc"): + def get_time_left(self, job_id: int, job_name="vnc"): """ Returns the time remaining for given job ID or None if the job is not present. @@ -914,8 +964,8 @@ def get_time_left(self, job_id:int, job_name="vnc"): cmd = f'squeue -o "%L %.18i %.8j %.8u %R" | grep {os.getlogin()} | grep {job_name} | grep {job_id}' proc = self.run_command(cmd) if proc.poll() is None: - line = str(proc.stdout.readline(), 'utf-8') - return line.split(' ', 1)[0] + line = str(proc.stdout.readline(), "utf-8") + return line.split(" ", 1)[0] return None def print_props(self): @@ -932,7 +982,7 @@ def print_props(self): if item == "subnode" and props[item] is not None: props[item].print_props() - def print_status(self, job_name:str, node_set=None, node_port_map=None): + def print_status(self, job_name: str, node_set=None, node_port_map=None): """ Print details of each active VNC job in node_set. VNC port and display number should be in node_port_map. @@ -971,27 +1021,36 @@ def repair_ln_sn_port_forwards(self, node_set=None, node_port_map=None): if node_port_map and node_port_map[node.name]: print(f"{node.name} with job ID {node.job_id} already has valid port forward") else: - subnode = SubNode(node.name, node.job_id, self.sing_container, self.xstartup, self.debug) + subnode = SubNode( + node.name, node.job_id, self.sing_container, self.xstartup, self.debug + ) subnode_pids = subnode.list_pids() # search for vnc process proc = subnode.run_command("ps x | grep vnc") while proc.poll() is None: - line = str(proc.stdout.readline(), 'utf-8').strip() - pid = int(line.split(' ', 1)[0]) + line = str(proc.stdout.readline(), "utf-8").strip() + pid = int(line.split(" ", 1)[0]) # match found if pid in subnode_pids: if self.debug: - logging.debug(f"repair_ln_sn_port_forwards: VNC PID {pid} found for job ID {node.job_id}") - pattern = re.compile(""" + logging.debug( + f"repair_ln_sn_port_forwards: VNC PID {pid} found for job ID {node.job_id}" + ) + pattern = re.compile( + """ (vnc\s+:) (?P\d+) - """, re.VERBOSE) + """, + re.VERBOSE, + ) match = re.search(pattern, line) assert match is not None vnc_port = BASE_VNC_PORT + int(match.group("display_number")) u2h_port = self.get_port() if self.debug: - logging.debug(f"repair_ln_sn_port_forwards: LoginNode({u2h_port})<->JobID({vnc_port})") + logging.debug( + f"repair_ln_sn_port_forwards: LoginNode({u2h_port})<->JobID({vnc_port})" + ) if u2h_port is None: print(f"Error: cannot find available/unused port") continue @@ -999,6 +1058,7 @@ def repair_ln_sn_port_forwards(self, node_set=None, node_port_map=None): self.subnode = subnode self.create_port_forward(u2h_port, vnc_port) + def check_auth_keys(): """ Returns True if a public key (~/.ssh/*.pub) exists in @@ -1011,121 +1071,158 @@ def check_auth_keys(): return True return False + def create_parser(): parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") # general arguments - parser.add_argument('-d', '--debug', - dest='debug', - action='store_true', - help='Enable debug logging') - parser.add_argument('-v', '--version', - dest='print_version', - action='store_true', - help='Print program version and exit') - parser.add_argument('-J', - dest='job_name', - metavar='', - help='Slurm job name', - default='hyakvnc', - type=str) + parser.add_argument( + "-d", "--debug", dest="debug", action="store_true", help="Enable debug logging" + ) + parser.add_argument( + "-v", + "--version", + dest="print_version", + action="store_true", + help="Print program version and exit", + ) + parser.add_argument( + "-J", + dest="job_name", + metavar="", + help="Slurm job name", + default="hyakvnc", + type=str, + ) # create command - parser_create = subparsers.add_parser('create', - help='Create VNC session') - parser_create.add_argument('-p', '--partition', - dest='partition', - metavar='', - help='Slurm partition', - required=True, - type=str) - parser_create.add_argument('-A', '--account', - dest='account', - metavar='', - help='Slurm account', - required=True, - type=str) - parser_create.add_argument('--timeout', - dest='timeout', - metavar='', - help='[default: 120] Slurm node allocation and VNC startup timeout length (in seconds)', - default=120, - type=int) - parser_create.add_argument('--port', - dest='u2h_port', - metavar='', - help='User<->Hyak Port override', - type=int) - parser_create.add_argument('-t', '--time', - dest='time', - metavar='', - help='Subnode reservation time (in hours)', - required=True, - type=int) - parser_create.add_argument('-c', '--cpus', - dest='cpus', - metavar='', - help='Subnode cpu count', - required=True, - type=int) - parser_create.add_argument('-G', '--gpus', - dest='gpus', - metavar='[type:]', - help='Subnode gpu count', - default='0', - type=str) - parser_create.add_argument('--mem', - dest='mem', - metavar='', - help='Subnode memory amount with units', - required=True, - type=str) - parser_create.add_argument('--container', - dest='sing_container', - metavar='', - help='Path to VNC Apptainer/Singularity Container (.sif)', - required=True, - type=str) - parser_create.add_argument('--xstartup', - dest='xstartup', - metavar='', - help='Path to xstartup script', - required=True, - type=str) + parser_create = subparsers.add_parser("create", help="Create VNC session") + parser_create.add_argument( + "-p", + "--partition", + dest="partition", + metavar="", + help="Slurm partition", + required=True, + type=str, + ) + parser_create.add_argument( + "-A", + "--account", + dest="account", + metavar="", + help="Slurm account", + required=True, + type=str, + ) + parser_create.add_argument( + "--timeout", + dest="timeout", + metavar="", + help="[default: 120] Slurm node allocation and VNC startup timeout length (in seconds)", + default=120, + type=int, + ) + parser_create.add_argument( + "--port", + dest="u2h_port", + metavar="", + help="User<->Hyak Port override", + type=int, + ) + parser_create.add_argument( + "-t", + "--time", + dest="time", + metavar="", + help="Subnode reservation time (in hours)", + required=True, + type=int, + ) + parser_create.add_argument( + "-c", + "--cpus", + dest="cpus", + metavar="", + help="Subnode cpu count", + required=True, + type=int, + ) + parser_create.add_argument( + "-G", + "--gpus", + dest="gpus", + metavar="[type:]", + help="Subnode gpu count", + default="0", + type=str, + ) + parser_create.add_argument( + "--mem", + dest="mem", + metavar="", + help="Subnode memory amount with units", + required=True, + type=str, + ) + parser_create.add_argument( + "--container", + dest="sing_container", + metavar="", + help="Path to VNC Apptainer/Singularity Container (.sif)", + required=True, + type=str, + ) + parser_create.add_argument( + "--xstartup", + dest="xstartup", + metavar="", + help="Path to xstartup script", + required=True, + type=str, + ) # status command - parser_status = subparsers.add_parser('status', - help='Print details of all VNC jobs with given job name and exit') + parser_status = subparsers.add_parser( + "status", help="Print details of all VNC jobs with given job name and exit" + ) # kill command - parser_kill = subparsers.add_parser('kill', - help='Kill specified job') - parser_kill.add_argument('job_id', - metavar='', - help='Kill specified VNC session, cancel its VNC job, and exit', - type=int) + parser_kill = subparsers.add_parser("kill", help="Kill specified job") + parser_kill.add_argument( + "job_id", + metavar="", + help="Kill specified VNC session, cancel its VNC job, and exit", + type=int, + ) # kill-all command - parser_kill_all = subparsers.add_parser('kill-all', - help='Cancel all VNC jobs with given job name and exit') + parser_kill_all = subparsers.add_parser( + "kill-all", help="Cancel all VNC jobs with given job name and exit" + ) # set-passwd command - parser_set_passwd = subparsers.add_parser('set-passwd', - help='Prompts for new VNC password and exit') - parser_set_passwd.add_argument('--container', - dest='sing_container', - metavar='', - help='Path to VNC Apptainer Container (.sif)', - required=True, - type=str) + parser_set_passwd = subparsers.add_parser( + "set-passwd", help="Prompts for new VNC password and exit" + ) + parser_set_passwd.add_argument( + "--container", + dest="sing_container", + metavar="", + help="Path to VNC Apptainer Container (.sif)", + required=True, + type=str, + ) # repair command - parser_repair = subparsers.add_parser('repair', - help='Repair all missing/broken LoginNode<->SubNode port forwards, and then exit') + parser_repair = subparsers.add_parser( + "repair", help="Repair all missing/broken LoginNode<->SubNode port forwards, and then exit" + ) return parser + def main(): parser = create_parser() args = parser.parse_args() @@ -1223,8 +1320,7 @@ def main(): if args.debug: logging.error(msg) - - if args.command == 'create': + if args.command == "create": assert os.path.exists(args.sing_container) assert os.path.exists(args.xstartup) @@ -1232,7 +1328,7 @@ def main(): hyak = LoginNode(hostname, args.sing_container, args.xstartup, args.debug) # check memory format - assert(re.match("[0-9]+[KMGT]", args.mem)) + assert re.match("[0-9]+[KMGT]", args.mem) # set VNC password at user's request or if missing if not hyak.check_vnc_password(): @@ -1242,7 +1338,16 @@ def main(): hyak.set_vnc_password() # reserve node - subnode = hyak.reserve_node(args.time, args.timeout, args.cpus, args.gpus, args.mem, args.partition, args.account, args.job_name) + subnode = hyak.reserve_node( + args.time, + args.timeout, + args.cpus, + args.gpus, + args.mem, + args.partition, + args.account, + args.job_name, + ) if subnode is None: exit(1) @@ -1266,11 +1371,11 @@ def __irq_handler__(signalNumber, frame): signal.signal(signal.SIGINT, __irq_handler__) signal.signal(signal.SIGTSTP, __irq_handler__) - gpu_count = int(args.gpus.split(':').pop()) - sing_exec_args = '' + gpu_count = int(args.gpus.split(":").pop()) + sing_exec_args = "" if gpu_count > 0: # Use `--nv` apptainer argument to bind CUDA driver and library - sing_exec_args = '--nv' + sing_exec_args = "--nv" # start vnc if not subnode.start_vnc(extra_args=sing_exec_args, timeout=args.timeout): @@ -1309,11 +1414,11 @@ def __irq_handler__(signalNumber, frame): logging.debug(msg) print(f"then connect to VNC session at localhost:{hyak.u2h_port}") print("=====================") - elif args.command == 'set-passwd': + elif args.command == "set-passwd": assert os.path.exists(args.sing_container) # create login node object - hyak = LoginNode(hostname, args.sing_container, '', args.debug) + hyak = LoginNode(hostname, args.sing_container, "", args.debug) if args.debug: logging.info("Setting new VNC password...") @@ -1321,7 +1426,7 @@ def __irq_handler__(signalNumber, frame): hyak.set_vnc_password() elif args.command is not None: # create login node object - hyak = LoginNode(hostname, '', '', args.debug) + hyak = LoginNode(hostname, "", "", args.debug) # check for existing subnodes with same job name node_set = hyak.find_nodes(args.job_name) @@ -1329,12 +1434,12 @@ def __irq_handler__(signalNumber, frame): # get port forwards (and display numbers) node_port_map = hyak.get_port_forwards(node_set) - if args.command == 'repair': + if args.command == "repair": # repair broken port forwards hyak.repair_ln_sn_port_forwards(node_set, node_port_map) - elif args.command == 'status': + elif args.command == "status": hyak.print_status(args.job_name, node_set, node_port_map) - elif args.command == 'kill': + elif args.command == "kill": # kill single VNC job with same job name msg = f"Attempting to kill {args.job_id}" print(msg) @@ -1348,7 +1453,9 @@ def __irq_handler__(signalNumber, frame): logging.info("Found kill target") logging.info(f"\tVNC display number: {node.vnc_display_number}") # kill vnc session - port_forward = hyak.get_job_port_forward(node.job_id, node.name, node_port_map) + port_forward = hyak.get_job_port_forward( + node.job_id, node.name, node_port_map + ) if port_forward: node.kill_vnc(port_forward[0] - BASE_VNC_PORT) # cancel job @@ -1359,7 +1466,7 @@ def __irq_handler__(signalNumber, frame): if args.debug: logging.error(msg) exit(1) - elif args.command == 'kill-all': + elif args.command == "kill-all": # kill all VNC jobs with same job name msg = f"Killing all VNC sessions with job name {args.job_name}..." print(msg) @@ -1373,5 +1480,6 @@ def __irq_handler__(signalNumber, frame): hyak.cancel_job(node.job_id) exit(0) + if __name__ == "__main__": main() From b74ad2ddd7d720afd5a4e0335349913ecd0c57f3 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 6 Sep 2023 16:42:19 -0700 Subject: [PATCH 09/13] Typo in shebang --- src/hyakvnc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyakvnc.py b/src/hyakvnc.py index 67ecf01..e3c8693 100755 --- a/src/hyakvnc.py +++ b/src/hyakvnc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python39 +#!/usr/bin/env python3.9 # SPDX-License-Identifier: MIT # Copyright (c) 2023 Hansem Ro From 28e4fc9c7f5bcc5ccda23d4cbdfa4638f3bebca5 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Thu, 7 Sep 2023 09:22:40 -0700 Subject: [PATCH 10/13] Don't hardcode Python version; rely on pip to require Python >= 3.9 at install --- .gitignore | 2 ++ src/hyakvnc.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 287a2f0..8d8160f 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +# WHL files +*.whl diff --git a/src/hyakvnc.py b/src/hyakvnc.py index e3c8693..5a39c4c 100755 --- a/src/hyakvnc.py +++ b/src/hyakvnc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.9 +#!/usr/bin/env python3 # SPDX-License-Identifier: MIT # Copyright (c) 2023 Hansem Ro From 4b0908891186ff04572a533b090b39ac986cfb03 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Thu, 7 Sep 2023 09:24:16 -0700 Subject: [PATCH 11/13] Exclude `*.whl` from gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8d8160f..fbffe1a 100644 --- a/.gitignore +++ b/.gitignore @@ -155,10 +155,10 @@ cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# be foud at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -# WHL files +# User-specific stuff *.whl From 094e44b06afb1715cae0ad79d68c2f4020e8bc65 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Thu, 7 Sep 2023 12:34:34 -0700 Subject: [PATCH 12/13] Update README --- README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 39394f7..c51b56d 100644 --- a/README.md +++ b/README.md @@ -23,25 +23,38 @@ Before running `hyakvnc`, you'll need the following: - Install additional tools and libraries to the container as required by programs running within the VNC session. - xstartup script used to launch a desktop environment +- A Python interpreter (version **3.9** or higher) ### Building -Update pip: +`hyakvnc` is a Python package that can be installed with `pip`. The minimum Python version required is **3.9**. You can check the version of the default Python 3 interpreter with: + +```bash +python3 -V +``` +As of 2021-09-30, the default Python 3 interpreter on Klone is version 3.6.8. Because `hyakvnc` requires version 3.9 or higher, it is necessary to specify the path to a Python 3.9 or newer interpreter when installing `hyakvnc`. You can list the Python 3 interpreters you have available with: + +```bash +compgen -c | grep '^python3\.[[:digit:]]$' +``` + +At this time, `klone` supports Python 3.9, which can be run with the command `python3.9`. The following instructions are written with `python3.9` in mind. If you use another version, such as `python3.11`, you will need to substitute `python3.9` with, e.g., `python3.11` in the instructions. + ```bash -python3 -m pip install --upgrade --user pip +python3.9 -m pip install --upgrade --user pip ``` Build and install the package: ```bash -python3 -m pip install --user git+https://github.com/uw-psych/hyakvnc +python3.9 -m pip install --user git+https://github.com/uw-psych/hyakvnc ``` Or, clone the repo and install the package locally: ```bash git clone https://github.com/uw-psych/hyakvnc -python3 -m pip install --user . +python3.9 -m pip install --user . ``` If successful, then `hyakvnc` should be installed to `~/.local/bin/`. @@ -51,10 +64,11 @@ If successful, then `hyakvnc` should be installed to `~/.local/bin/`. The optional dependency group`[dev]` in `pyroject.toml` includes dependencies useful for development, including [pre-commit](https://pre-commit.com/) hooks that run in order to commit to the `git` repository. These apply various checks, including running the `black` code formatter before the commit takes place. -To ensure `pre-commit` is installed, run: +To ensure `pre-commit` and other development packages are installed, run: + ```bash -pip install '.[precommit]' -`` +python3.9 -m pip install --user '.[dev]' +``` ### General usage From b08885c241cb9b95cfde4ad82938b80e6d4b004c Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Thu, 7 Sep 2023 14:32:12 -0700 Subject: [PATCH 13/13] Update README --- README.md | 88 ++++++++++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index c51b56d..21a1de4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -hyakvnc -======= +# hyakvnc Create and manage VNC Slurm jobs on UW HYAK Klone cluster. @@ -15,14 +14,14 @@ time. ### Prerequisites Before running `hyakvnc`, you'll need the following: + - SSH client - VNC client/viewer - - TigerVNC viewer is recommended for all platforms + - TigerVNC viewer is recommended for all platforms - HYAK Klone access with compute resources - VNC Apptainer with TigerVNC server and a desktop environment - - Install additional tools and libraries to the container as required by - programs running within the VNC session. -- xstartup script used to launch a desktop environment + - Install additional tools and libraries to the container as required by programs running within the VNC session. +- `xstartup` script used to launch a desktop environment - A Python interpreter (version **3.9** or higher) ### Building @@ -32,13 +31,14 @@ Before running `hyakvnc`, you'll need the following: ```bash python3 -V ``` + As of 2021-09-30, the default Python 3 interpreter on Klone is version 3.6.8. Because `hyakvnc` requires version 3.9 or higher, it is necessary to specify the path to a Python 3.9 or newer interpreter when installing `hyakvnc`. You can list the Python 3 interpreters you have available with: ```bash compgen -c | grep '^python3\.[[:digit:]]$' ``` -At this time, `klone` supports Python 3.9, which can be run with the command `python3.9`. The following instructions are written with `python3.9` in mind. If you use another version, such as `python3.11`, you will need to substitute `python3.9` with, e.g., `python3.11` in the instructions. +At this time, Klone supports Python 3.9, which can be run with the command `python3.9`. The following instructions are written with `python3.9` in mind. If you use another version, such as `python3.11`, you will need to substitute `python3.9` with, e.g., `python3.11` in the instructions. ```bash python3.9 -m pip install --upgrade --user pip @@ -61,7 +61,7 @@ If successful, then `hyakvnc` should be installed to `~/.local/bin/`. #### Optional dependencies for development -The optional dependency group`[dev]` in `pyroject.toml` includes dependencies useful for development, including [pre-commit](https://pre-commit.com/) hooks that run in order to commit to the `git` repository. +The optional dependency group `[dev]` in `pyroject.toml` includes dependencies useful for development, including [pre-commit](https://pre-commit.com/) hooks that run in order to commit to the `git` repository. These apply various checks, including running the `black` code formatter before the commit takes place. To ensure `pre-commit` and other development packages are installed, run: @@ -76,57 +76,53 @@ python3.9 -m pip install --user '.[dev]' ### Creating a VNC session -1. Start a VNC session with the `hyakvnc create` command followed by arguments - to specify the Slurm account and partition, compute resource needs, - reservation time, and paths to a VNC apptainer and xstartup script. +1. Start a VNC session with the `hyakvnc create` command followed by arguments to specify the Slurm account and partition, compute resource needs, reservation time, and paths to a VNC apptainer and xstartup script. -```bash -# example: Starting VNC session on `ece` compute resources for 5 hours on a -# node with 16 cores and 32GB of memory -hyakvnc create -A ece -p compute-hugemem \ - -t 5 -c 16 --mem 32G \ - --container /path/to/container.sif \ - --xstartup /path/to/xstartup -``` + ```bash + # example: Starting VNC session on `ece` compute resources for 5 hours on a + # node with 16 cores and 32GB of memory + hyakvnc create -A ece -p compute-hugemem \ + -t 5 -c 16 --mem 32G \ + --container /path/to/container.sif \ + --xstartup /path/to/xstartup + ``` 2. If successful, `hyakvnc` should print a unique port forward command: -``` -... -===================== -Run the following in a new terminal window: - ssh -N -f -L AAAA:127.0.0.1:BBBB hansem7@klone.hyak.uw.edu -then connect to VNC session at localhost:AAAA -===================== -``` + ```text + ... + ===================== + Run the following in a new terminal window: + ssh -N -f -L AAAA:127.0.0.1:BBBB hansem7@klone.hyak.uw.edu + then connect to VNC session at localhost:AAAA + ===================== + ``` -Copy this port forward command for the following step. + Copy this port forward command for the following step. -3. Set up port forward between your computer and HYAK login node. On your - machine, in a new terminal window, run the the copied command. +3. Set up port forward between your computer and HYAK login node. On your machine, in a new terminal window, run the the copied command. -Following the example, run: + Following the example, run: -```bash -ssh -N -f -L AAAA:127.0.0.1:BBBB hansem7@klone.hyak.uw.edu -``` + ```bash + ssh -N -f -L AAAA:127.0.0.1:BBBB hansem7@klone.hyak.uw.edu + ``` -Alternatively, for PuTTY users, navigate to -PuTTY Configuration->Connection->SSH->Tunnels, then set: - - source port to `AAAA` - - destination to `127.0.0.1:BBBB` + Alternatively, for PuTTY users, navigate to `PuTTY Configuration->Connection->SSH->Tunnels`, then set: + - source port to `AAAA` + - destination to `127.0.0.1:BBBB` -Press `Add`, then connect to Klone as normal. Keep this window open as it -maintains a connection to the VNC session running on Klone. + Press `Add`, then connect to Klone as normal. Keep this window open as it + maintains a connection to the VNC session running on Klone. 4. Connect to the VNC session at instructed address (in this example: `localhost:AAAA`) 5. To close the VNC session, run the following -```bash -hyakvnc kill-all -``` + ```bash + hyakvnc kill-all + ``` ### Checking active VNC sessions @@ -159,7 +155,7 @@ re-established with new port mappings via the `hyakvnc repair` command. Before repairing: -``` +```text Active hyakvnc jobs: Job ID: NNNNNNNN SubNode: n3050 @@ -169,7 +165,7 @@ Active hyakvnc jobs: After repairing port forwarding: -``` +```text Active hyakvnc jobs: Job ID: NNNNNNNN SubNode: n3050 @@ -186,7 +182,7 @@ which may require the user to set up new port forward(s). ### Override Apptainer bind paths -If present, `hyakvnc` will use `APPTAINER_BINDPATH` or `SINGULARITY_BINDPATH` to +If present, `hyakvnc` will use the [environment variables](https://tldp.org/LDP/Bash-Beginners-Guide/html/sect_03_02.html) `APPTAINER_BINDPATH` or `SINGULARITY_BINDPATH` to determine how paths are mounted in the VNC container environment. If neither is defined, `hyakvnc` will use its default bindpath.