diff --git a/Makefile.am b/Makefile.am index 8c945688..321b2720 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,6 @@ ACLOCAL_AMFLAGS = -I config -SUBDIRS = src etc t +SUBDIRS = src etc doc t EXTRA_DIST= \ config/tap-driver.sh \ diff --git a/configure.ac b/configure.ac index 7cb74966..c73ee7fe 100644 --- a/configure.ac +++ b/configure.ac @@ -57,6 +57,38 @@ fi # checks for packages PKG_CHECK_MODULES([JANSSON], [jansson >= 2.10], [], []) +AS_IF([test "x$enable_docs" != "xno"], [ + AM_CHECK_PYMOD(sphinx, + [Version(sphinx.__version__) >= Version ('1.6.7')], + [sphinx=true], + [sphinx=false; AC_MSG_WARN([could not find sphinx to generate docs, version 1.6.7+ required])] + ) + AM_CHECK_PYMOD(docutils, + [Version(docutils.__version__) >= Version ('0.11.0')], + [docutils=true], + [docutils=false; AC_MSG_WARN([could not find docutils to generate docs, version 0.11.0+ required])] + ) +]) +# If --enable-docs=yes, but no doc generator found, +# then error immediately: +# +AS_IF([test "x$enable_docs" = "xyes" -a "x$sphinx" = "xfalse"],[ + AC_MSG_ERROR([--enable-docs used but no document generator found!]) +]) +AS_IF([test "x$enable_docs" = "xyes" -a "x$docutils" = "xfalse"],[ + AC_MSG_ERROR([--enable-docs used but docutils not found!]) +]) +AM_CONDITIONAL([ENABLE_DOCS], [test "x$sphinx" = "xtrue" -a "x$docutils" = "xtrue"]) +AC_CHECK_PROG(ASPELL,[aspell],[aspell]) + +AS_IF([test "x$enable_docs" != "xno"], [ + if test "x$sphinx" = "xfalse"; then + AC_MSG_WARN([Python Sphinx not found. Manual pages will not be generated.]) + elif test "x$docutils" = "xfalse"; then + AC_MSG_WARN([Python Docutils not found. Manual pages will not be generated.]) + fi +]) + # # Project directories # @@ -119,6 +151,8 @@ AC_CONFIG_FILES([Makefile src/plugins/Makefile etc/Makefile etc/flux-accounting.service + doc/Makefile + doc/test/Makefile t/Makefile ]) AC_OUTPUT diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 00000000..2691a335 --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,83 @@ +.NOTPARALLEL: + +SUBDIRS = . test + +MAN_FILES = $(MAN5_FILES) + +MAN5_FILES = $(MAN5_FILES_PRIMARY) + +MAN5_FILES_PRIMARY = \ + man5/flux-config-accounting.5 + +RST_FILES = \ + $(MAN5_FILES_PRIMARY:.5=.rst) + +if ENABLE_DOCS +man_MANS = $(MAN5_FILES) +endif + +SUFFIXES = .rst .5 + +sphinx_man = $(sphinx_man_$(V)) +sphinx_man_ = $(sphinx_man_$(AM_DEFAULT_VERBOSITY)) +sphinx_man_0 = @echo " BUILD manpages"; + +sphinx_html = $(sphinx_html_$(V)) +sphinx_html_ = $(sphinx_html_$(AM_DEFAULT_VERBOSITY)) +sphinx_html_0 = @echo " BUILD html"; + +sphinx_verbose_flags = $(sphinx_verbose_flags_$(V)) +sphinx_verbose_flags_ = $(sphinx_verbose_flags_$(AM_DEFAULT_VERBOSITY)) +sphinx_verbose_flags_0 = +sphinx_verbose_flags_1 = -v +sphinx_verbose_flags_2 = -vv + +STDERR_DEVNULL = $(stderr_devnull_$(V)) +stderr_devnull_ = $(stderr_devnull_$(AM_DEFAULT_VERBOSITY)) +stderr_devnull_0 = >/dev/null 2>&1 + +$(MAN_FILES): manpages.py conf.py $(RST_FILES) + $(sphinx_man) \ + SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \ + -m sphinx $(sphinx_verbose_flags) -b man $(srcdir) ./man \ + $(STDERR_DEVNULL) + @echo " MV manpages"; \ + for sec in 5; do \ + $(MKDIR_P) man$$sec && \ + mv -f $(abs_builddir)/man/*.$$sec man$$sec/; \ + done + +.PHONY: html +html: conf.py $(RST_FILES) + $(sphinx_html) \ + SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \ + -m sphinx $(sphinx_verbose_flags) -b html $(srcdir) ./html \ + $(STDERR_DEVNULL) + +EXTRA_DIST = \ + conf.py \ + manpages.py \ + index.rst \ + domainrefs.py \ + requirements.txt \ + guide/accounting-guide.rst \ + $(RST_FILES) \ + man5/index.rst + +CLEANFILES = \ + $(MAN_FILES) + +clean-local: + -rm -rf man python/autogenerated + +distclean-local: + -rm -rf doctrees + +AM_LDFLAGS = \ + $(CODE_COVERAGE_LIBS) \ + -no-install + +AM_CPPFLAGS = \ + -I$(top_srcdir) \ + -I$(top_srcdir)/src/include \ + -I$(top_builddir)/src/common/libflux diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 00000000..00739726 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,150 @@ +############################################################### +# Copyright 2024 Lawrence Livermore National Security, LLC +# (c.f. AUTHORS, NOTICE.LLNS, COPYING) +# +# This file is part of the Flux resource manager framework. +# For details, see https://github.com/flux-framework. +# +# SPDX-License-Identifier: LGPL-3.0 +############################################################### + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +# add `manpages` directory to sys.path +import pathlib + +sys.path.append(str(pathlib.Path(__file__).absolute().parent)) + +from manpages import man_pages +import docutils.nodes + +# -- Project information ----------------------------------------------------- + +project = "flux-accounting" +copyright = """Copyright 2024 Lawrence Livermore National Security, LLC and Flux developers. + +SPDX-License-Identifier: LGPL-3.0""" + +# -- General configuration --------------------------------------------------- + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +master_doc = "index" +source_suffix = ".rst" + +extensions = ["sphinx.ext.intersphinx", "sphinx.ext.napoleon", "domainrefs"] + +domainrefs = { + "core:man5": { + "text": "%s(5)", + "url": "https://flux-framework.readthedocs.io/projects/flux-core/en/latest/man5/%s.html", + }, +} + +# Disable "smartquotes" to avoid things such as turning long-options +# "--" into en-dash in html output, which won't make much sense for +# manpages. +smartquotes = False + +# -- Setup for Sphinx API Docs ----------------------------------------------- + +# Workaround since sphinx does not automatically run apidoc before a build +# Copied from https://github.com/readthedocs/readthedocs.org/issues/1139 + +script_dir = os.path.normpath(os.path.dirname(__file__)) +py_bindings_dir = os.path.normpath(os.path.join(script_dir, "../src/bindings/python/")) + +# Make sure that the python bindings are in PYTHONPATH for autodoc +sys.path.insert(0, py_bindings_dir) + +# run api doc +def run_apidoc(_): + # Move import inside so that `gen-cmdhelp.py` can exec this file in LGTM.com + # without sphinx installed + # pylint: disable=import-outside-toplevel + from sphinx.ext.apidoc import main + + try: + # Check if running under `make` + build_dir = os.path.normpath(os.environ.get("SPHINX_BUILDDIR")) + except: + build_dir = script_dir + output_path = os.path.join(build_dir, "python", "autogenerated") + exclusions = [ + os.path.join(py_bindings_dir, "setup.py"), + ] + main(["-e", "-f", "-M", "-T", "-o", output_path, py_bindings_dir] + exclusions) + + +def man_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + section = int(name[-1]) + page = None + for man in man_pages: + if man[1] == text and man[4] == section: + page = man[0] + break + if page == None: + page = "man7/flux-undocumented" + section = 7 + + node = docutils.nodes.reference( + rawsource=rawtext, + text=f"{text}({section})", + refuri=f"../{page}.html", + **options, + ) + return [node], [] + + +# launch setup +def setup(app): + app.connect("builder-inited", run_apidoc) + for section in [5]: + app.add_role(f"man{section}", man_role) + + +napoleon_google_docstring = True + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# -- Options for Intersphinx ------------------------------------------------- + +intersphinx_mapping = { + "core": ( + "https://flux-framework.readthedocs.io/projects/flux-core/en/latest/", + None, + ), + "rfc": ( + "https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/", + None, + ), +} diff --git a/doc/domainrefs.py b/doc/domainrefs.py new file mode 100644 index 00000000..a86afe95 --- /dev/null +++ b/doc/domainrefs.py @@ -0,0 +1,65 @@ +""" +Originally from: + + https://github.com/mitogen-hq/mitogen/blob/master/docs/domainrefs.py + +Copyright 2021, the Mitogen authors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" +import functools +import re + +import docutils.nodes +import docutils.utils + + +CUSTOM_RE = re.compile("(.*) <(.*)>") + + +def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]): + template = "https://docs.ansible.com/ansible/latest/modules/%s_module.html" + + match = CUSTOM_RE.match(text) + if match: # "custom text " + title = match.group(1) + text = match.group(2) + elif text.startswith("~"): # brief + text = text[1:] + title = config.get("brief", "%s") % (docutils.utils.unescape(text),) + else: + title = config.get("text", "%s") % (docutils.utils.unescape(text),) + + node = docutils.nodes.reference( + rawsource=rawtext, text=title, refuri=config["url"] % (text,), **options + ) + + return [node], [] + + +def setup(app): + for name, info in app.config._raw_config["domainrefs"].items(): + app.add_role(name, functools.partial(role, info)) diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 00000000..5e2d4b29 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,17 @@ +Documentation for flux-accounting +================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + +.. _man-pages: + +flux-accounting Manual Pages +============================ + +.. toctree:: + :maxdepth: 2 + + man5/index diff --git a/doc/man5/index.rst b/doc/man5/index.rst new file mode 100644 index 00000000..6bc66fa6 --- /dev/null +++ b/doc/man5/index.rst @@ -0,0 +1,8 @@ +man5 +==== + +.. toctree:: + :caption: File formats and conventions + :maxdepth: 1 + + flux-config-accounting diff --git a/doc/manpages.py b/doc/manpages.py new file mode 100644 index 00000000..bcfbc573 --- /dev/null +++ b/doc/manpages.py @@ -0,0 +1,27 @@ +############################################################### +# Copyright 2022 Lawrence Livermore National Security, LLC +# (c.f. AUTHORS, NOTICE.LLNS, COPYING) +# +# This file is part of the Flux resource manager framework. +# For details, see https://github.com/flux-framework. +# +# SPDX-License-Identifier: LGPL-3.0 +############################################################### + +author = "This page is maintained by the Flux community." + +# Add man page entries with the following information: +# - Relative file path (without .rst extension) +# - Man page name +# - Man page description +# - Author (use [author]) +# - Manual section +man_pages = [ + ( + "man5/flux-config-accounting", + "flux-config-accounting", + "flux-accounting priority plugin configuration file", + [author], + 5, + ), +] diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..3563f716 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx<6.0.0 +sphinx-rtd-theme>=0.5.2 +docutils>=0.14,<0.18 +jinja2<3.1.0 diff --git a/doc/test/Makefile.am b/doc/test/Makefile.am new file mode 100644 index 00000000..3564a687 --- /dev/null +++ b/doc/test/Makefile.am @@ -0,0 +1,13 @@ +dist_noinst_SCRIPTS = spellcheck + +EXTRA_DIST = spell.en.pws + +AM_TESTS_ENVIRONMENT = \ + ASPELL=$(ASPELL) \ + pws_dict=$(abs_top_srcdir)/doc/test/spell.en.pws \ + man_base_dir=$(abs_top_srcdir)/doc + +LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \ + $(top_srcdir)/config/tap-driver.sh + +TESTS = spellcheck diff --git a/doc/test/spell.en.pws b/doc/test/spell.en.pws new file mode 100644 index 00000000..591a3014 --- /dev/null +++ b/doc/test/spell.en.pws @@ -0,0 +1,537 @@ +personal_ws-1.1 en 0 +maxdepth +literalinclude +toctree +rst +proc +localid +exitcode +getline +plugstack +setlevel +errn +dest +killall +sig +unsets +setenvf +environ +substring +asprintf +KVS +kvs +api +comms +dev +dir +doctype +dropcache +fromkvs +github +json +linkname +manpage +mkdir +multicast +namespace +readlink +tokvs +adoc +apisock +cmd +ZMTP +stderr +stdout +stdin +stdio +CurveCP +CurveZMQ +IPC +ZeroMQ +keygen +KEYGEN +MUNGE +TCP +UID +UUID +uuids +ok +crit +topo +msec +cmb +syslog +graphviz +prepended +multipart +Tpng +nodesets +keepalives +emerg +gettimeofday +tmpdir +tmp +DocBook +Livermore +LLC +Stylesheets +XSL +conf +google +zeromq +EOF +Init +LD +LWJ +LWJs +MPI +MVAPICH +PMI +Prepend +TotalView +WRECKRUN +builddir +cmdline +egrep +epidkey +hiWSLCPRzqFYHNubktbCQ +hostname +io +jobid +jobids +libpmi +lwj +mpi +mpicc +nnodes +noinput +ntasks +pmi +resrc +runtime +src +tpn +uuid +ary +baz +EPGM +modopts +nodeset +noexec +pre +slurm +srun +cryptographic +unlink +timestamped +libs +scalable +config +sbin +zpl +libexec +usr +setenv +getenv +repo +lua +cpath +DEST +logdest +modopt +noshell +plugins +sid +SIGKILL +tty +uri +usefile +secdir +boolean +argc +argv +const +modctl +MODPATH +SHA +fanout +NOHOST +sysexits +NODEID +SIGINT +SIGTERM +signo +subprocess +subprocesses +PID +ps +Livermore +LLC +adoc +doctype +github +manpage +printf +const +doctype +nonblock +recv +stderr +tmpdir +uri +bitmask +cb +EINVAL +EAGAIN +ENOSYS +ENOMEM +errno +strerror +ev +fw +init +io +offsetof +POLLERR +POLLIN +POLLOUT +pollevents +pollfd +requeue +revents +struct +typedef +msg +nodeid +pre +bsize +comms +initializers +matchtag +requeued +typemask +uint +requeues +RQ +libev +baz +unsubscribe +unsubscribes +zeromq +ZeroMQ +libflux +arg +destructor +libev +libev's +ZMQ +EB +buf +EPROTO +EWOULDBLOCK +fd +iobuf +recvfd +sendfd +matchtags +NORESPONSE +rpc +str +trpc +bool +topen +tevent +tsend +trecv +tsync +scalability +env +ENV +TIMEDFLUSH +HWMFLUSH +treduce +itemweight +unreduced +HWM +unweighted +WCOUNT +tinfo +hwloc +epgm +args +attr +ATTRFLAG +ENOENT +EPERM +READONLY +tbon +attrget +getattr +lsattr +setattr +buflimit +bufcount +dmesg +jitter +zmq +nowait +zsock +msgtype +cmp +fnmatch +wildcards +addvec +delvec +msghandler +SIGCHLD +signum +pid +rpid +rstatus +waitpid +rstat +prev +nlink +inotify +ipc +FILENAME +dst +taskid +taskids +errnum +sqlite +sophia +sha +blobref +blobrefs +lstopo +xml +filesystem +filename +PYTHONPATH +localhost +EEXIST +rsh +RCMD +pty +Cron +CRON +cron +lifecycle +rexec +subcommand +subcommands +stddev +RTT +wakeup +crontab +appname +maxlevel +wireup +hwm +recurse +treeobj +unlinked +getat +lookup +dirat +readlinkat +fmt +getf +jansson +jansson's +rpcf +decodef +len +respondf +encodef +UTF +whitespace +dirsize +subdirectories +adc +afd +bfef +da +ee +EFBIG +fc +maxblob +metadata +treeref +valref +ver +rundir +walltime +clearbit +getrusage +objname +rusage +setbit +rolemask +userid +selfpmi +UTC +addrole +delrole +strtoul +EHOSTUNREACH +ap +MSGID +NULL's +ORed +PRI +procid +progname +timestamp +va +vlog +wallclock +DIRREF +ec +FFFFFFFF +FFFFFFFFF +rootdir +IP +mcast +PUs +ETIMEDOUT +fn +destructors +ENOTDIR +EISDIR +READDIR +lookupat +symlink +nprocs +procs +txn +kvsdir +vpack +dirref +lookups +mh +namespaces +ENOTSUP +EOVERFLOW +itr +JOBIDS +ns +Embde +idset +AUTOGROW +findNext +findPrevious +vEB +lflux +resizing +HOSTLIST +hostlist +hostlists +nodelist +hostnames +unfulfills +MSGFLAG +errstr +valgrind +NODENAME +dumplog +getopt +params +sched +setopt +unsetenv +CUDA +GPU +GPUs +cpu +distrib +epilog +gpubind +gpus +fulfillments +enodata +getroot +eventlog +eventlogs +nocopy +srckey +dstkey +semver +versioning +JOBNAME +LGPL +SPDX +MATCHDEBUG +Ctrl +UNIQ +startup +errmsg +WAITCREATE +incref +decref +vv +vvv +jobspec +FSD +enqueues +cpus +gpu +idsets +systemd +toml +unordered +fsd +hms +username +submitter's +enqueue +nslots +alloc +datetime +formatter +ndecode +dothex +cwd +execve +initrc +INITRC +loglevel +overwrite +PTRACE +rankinfo +rc +rcpath +searchpath +sysconfdir +TRACEME +WIFEXTED +builtin +OMP +envfile +regex +encodings +dec +subkey +waitstatus +returncode +bulksubmit +basename +bcc +dirname +jps +xargs +sep +RPCs +jobtap +unavail +reprioritize +reprioritization +afterany +afterok +afternotok +parsedatetime +cancelall +raiseall +IPv4 +IPv6 diff --git a/doc/test/spellcheck b/doc/test/spellcheck new file mode 100755 index 00000000..d6310682 --- /dev/null +++ b/doc/test/spellcheck @@ -0,0 +1,51 @@ +#!/bin/bash + +if test $man_base_dir; then + set ${man_base_dir}/man*/*.rst +fi + +if test $# == 0; then + echo "Usage: spellcheck file [file...]" >&2 + exit 1 +fi + +dict=${pws_dict:-./spell.en.pws} +if ! test -r $dict; then + echo Dictionary $dict not found >&2 + exit 1 +fi +if egrep -q '^[[:space:]]*$' $dict; then + echo Dictionary $dict contains blank line >&2 + exit 1 +fi +if test -z "$ASPELL"; then + echo "1..0 # skip because aspell is not installed" + exit 0 +fi +if ! $ASPELL -n list /dev/null; then + echo "1..$# # skip because aspell dry run failed" + exit 0 +fi + +exit_val=0 +echo "1..$#" +count=1 +for f in $*; do + filename=$(basename $f) + dir=$(basename $(dirname $f)) + tmpfile=$(mktemp) + rc=$(cat $f | $ASPELL -p $dict --encoding='utf-8' -n list \ + | sort | uniq | tee $tmpfile | wc -l) + if test $rc == 0; then + echo "ok $count - spell check $dir/$filename" + else + echo "not ok $count - spell check $dir/$filename failed" + echo "---" + exit_val=1 + fi + cat $tmpfile | sed -e "s!^!$(basename $f): !" + rm -f $tmpfile + ((count++)) +done + +exit $exit_val