Allow the use of normal Markdown in README files

In this PR a stage is added to the documentation compilation to call the script `ci_tools/readme_to_doxygen.py` which converts from GitHub/GitLab markdown syntax to Doxygen syntax. The page tags are generated automatically from the paths and inserted into Markdown references. This makes both the docs and the READMEs legible. It also removes the need for `About.md` which duplicated `README.md`. Additionally sub-sections are now correctly marked which fixes the indexing on the pages generated by Doxygen.

ci_tools/readme_to_doxygen.py

@@ -0,0 +1,174 @@
+""" Script for translating Markdown files compatible with GitHub/GitLab to a format that can be understood by Doxygen.
+"""
+from argparse import ArgumentParser
+import os
+import re
+
+# '$', not preceded by '$' or followed by '$'
+single_math_tag = re.compile(r'(?<!\$)\$(?!\$)')
+# '$$', not preceded by '$' or followed by '$'
+multiline_math_tag = re.compile(r'(?<!\$)\$\$(?!\$)')
+reference_tag = re.compile(r'(\[[^\]]*\])(\([^\)]*\))')
+section_tag = re.compile(r'\n## ([^\n]*)\n')
+
+doxygen_tag_dict = {" ":"_", "+":"x", os.path.sep: '_'}
+
+def get_compatible_tag(tag):
+    """
+    Get a Doxygen tag which doesn't contain problematic symbols.
+
+    Construct a Doxygen tag from a string by replacing any problematic
+    symbols.
+
+    Parameters
+    ----------
+    tag : str
+        The string from which the tag should be constructed.
+
+    Returns
+    -------
+    str
+        A string that can be used as a Doxygen tag.
+    """
+    for k, v in doxygen_tag_dict.items():
+        tag = tag.replace(k, v)
+    return tag
+
+def format_equations(line, start_tag, end_tag, start_replace, end_replace):
+    """
+    Format equations with Doxygen style.
+
+    Use regexes to find equations. If they are not in code blocks (e.g. for
+    documentation explaining how to format equations) then replace Markdown
+    formatting with Doxygen style formatting.
+
+    Parameters
+    ----------
+    line : str
+        A line of markdown text.
+
+    start_tag : re.Pattern
+        A regex pattern matching the start of the equation.
+
+    end_tag : re.Pattern
+        A regex pattern matching the end of the equation.
+
+    start_replace : str
+        A Doxygen-compatible string which should be used to introduce the equation.
+
+    end_replace : str
+        A Doxygen-compatible string which should be used to close the equation.
+
+    Returns
+    -------
+    str
+        The updated line.
+    """
+    # Find the code blocks in the line
+    code_tags = [i for i,l in enumerate(line) if l=='`']
+    n_code_tags = len(code_tags)
+    # Ensure that all inline code blocks are closed
+    assert n_code_tags % 2 == 0
+    n_code_tags //= 2
+    code_blocks = [(code_tags[i], code_tags[i+1]) for i in range(n_code_tags)]
+    start_match = start_tag.search(line)
+    while start_match:
+        if not any(start < start_match.start() < end for start, end in code_blocks):
+            end_match = end_tag.search(line, start_match.end())
+            assert end_match is not None
+            replacement = start_replace + line[start_match.end():end_match.start()] + end_replace
+            line = line[:start_match.start()] + replacement + line[end_match.end():]
+            start_match = start_tag.search(line, start_match.start() + len(replacement))
+        else:
+            start_match = start_tag.search(line, start_match.end())
+
+    return line
+
+root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='Tool for translating Markdown files to a format that can be understood by Doxygen.')
+
+    parser.add_argument('input_file', type=str, help='The file to treated.')
+    parser.add_argument('output_file', type=str, help='The file where the results should be saved.')
+    args = parser.parse_args()
+
+    file = args.input_file
+    folder = os.path.dirname(file)
+
+    with open(file, 'r', encoding='utf-8') as f:
+        contents = f.readlines()
+
+    title = contents[0]
+
+    assert title.startswith('#')
+
+    title = title[1:].strip()
+
+    relpath = os.path.relpath(file, start=root_dir)
+
+    if file.endswith('README.md'):
+        reldir = os.path.dirname(relpath)
+        tag = get_compatible_tag(reldir)
+    else:
+        tag = get_compatible_tag(relpath[:-3])
+
+    # Add tag to page
+    if tag:
+        title = f'@page {tag} {title}\n'
+    else:
+        title = '@mainpage\n'
+
+    body = []
+    in_code = False
+    in_math_code = False
+    for line in contents[1:]:
+        stripped = line.strip()
+        if stripped.startswith('```'):
+            if in_code:
+                in_code = False
+            elif in_math_code:
+                body.append('@f]')
+                in_math_code = False
+                continue
+            elif stripped == '```math':
+                body.append('@f[')
+                in_math_code = True
+                continue
+            else:
+                in_code = True
+        elif not in_code:
+            if line.startswith('##') and line[2] != '#':
+                sec_title = line[2:].strip()
+                sec_tag = get_compatible_tag(sec_title)
+                line = f"\n@section {sec_tag} {sec_title}\n"
+            else:
+                # Replace inline math with Doxygen tag
+                line = format_equations(line, single_math_tag, single_math_tag, "@f$", "@f$")
+                line = format_equations(line, multiline_math_tag, multiline_math_tag, "@f[", "@f]")
+
+                # Look for references
+                match_found = reference_tag.search(line)
+                while match_found:
+                    path = match_found.group(2)[1:-1]
+                    # Replace references to READMEs with the page tag
+                    if path.endswith('README.md') and not os.path.isabs(path):
+                        path = os.path.normpath(os.path.join(folder, path))
+                        relpath = os.path.relpath(path, start=root_dir)
+                        reldir = os.path.dirname(relpath)
+                        subpage_tag = get_compatible_tag(reldir)
+                        doxygen_ref = "@subpage " + subpage_tag
+                        line = line[:match_found.start()] + doxygen_ref + line[match_found.end():]
+                        match_found = reference_tag.search(line, match_found.start() + len(doxygen_ref))
+                    else:
+                        match_found = reference_tag.search(line, match_found.end())
+
+        body.append(line)
+
+
+    directory = os.path.dirname(args.output_file)
+    os.makedirs(directory, exist_ok = True)
+
+    with open(args.output_file, 'w', encoding='utf-8') as f:
+        f.write(title)
+        f.write(''.join(body))

docs/Adding_docs.md

@@ -1,4 +1,4 @@
-@page adding_docs Adding Documentation
+# Adding Documentation
 
 There are two types of documentation:
 1.  Documentation describing code structures
@@ -91,20 +91,20 @@ public:
 
 Each folder containing code should also contain a `README.md` file providing an overview of the contents of the folder and any general information and references which may be helpful for understanding this code.
 
-The `README.md` file should begin with the line:
+The `README.md` file should begin with a title:
 ```markdown
-@page <tag> <title>
+# <title>
 ```
-Where `<tag>` is a one word key which identifies the page, and `<title>` is the title of the page. If the folder doesn't contain a `.private` file (indicating that the folder is not yet ready to be shared publically), the new page can be referenced from any enclosing page that may exist. For example if you create a new folder in `src/`, you should add the following line to the list of folders in `src/README.md`:
+If the folder doesn't contain a `.private` file (indicating that the folder is not yet ready to be shared publically), the new page can be referenced from any enclosing page that may exist. For example if you create a new folder in `src/`, you should add the following line to the list of folders in `src/README.md`:
 ```markdown
-- @subpage <tag>
+- [my_new_folder](./my_new_folder/README.md) : Short description of contents.
 ```
 
 ## Mathematical notation in documentation
 
-Mathematical notation can be used in Doxygen output. It should be blocked with `@f$` commands. E.g:
-@code
-@f$a \ne b@f$
-@endcode
+Mathematical notation can be used in Doxygen output. It should be blocked with `$` commands. E.g:
+```
+$a \ne b$
+```
 
 An equation can also be printed on its own line using `$$` commands.

docs/CMakeLists.txt

@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: MIT
 cmake_minimum_required(VERSION 3.15)
 
+FIND_PACKAGE(Python3 REQUIRED)
+message(STATUS "DOXYGEN")
+
 file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html")
 set(DOXYGEN_PROJECT_NAME "Gyselalib++")
 set(DOXYGEN_AUTOLINK_SUPPORT YES)
@@ -52,10 +55,37 @@ set(DOXYGEN_WARN_AS_ERROR NO)
 set(DOXYGEN_WARN_LOGFILE "doxygen.log")
 set(DOXYGEN_STRIP_FROM_PATH "")
 #set(DOXYGEN_IMAGE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/images")
-FILE(GLOB MarkdownSources "${CMAKE_CURRENT_SOURCE_DIR}/*.md")
+FILE(GLOB_RECURSE MarkdownSources
+	"${voicexx_SOURCE_DIR}/src/*README.md"
+	"${voicexx_SOURCE_DIR}/tests/*README.md"
+	"${voicexx_SOURCE_DIR}/simulations/*README.md"
+	"${voicexx_SOURCE_DIR}/vendor/sll/*README.md"
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.md")
+
+message(STATUS ${MarkdownSources})
+
+SET(TREATED_MD)
+
+# Generate Doxygen compatible Markdown files
+foreach(MD_FILE ${MarkdownSources} "${voicexx_SOURCE_DIR}/README.md")
+
+	file(RELATIVE_PATH relpath ${voicexx_SOURCE_DIR} ${MD_FILE})
+
+	message(STATUS "${Python3_EXECUTABLE} ${voicexx_SOURCE_DIR}/ci_tools/readme_to_doxygen.py ${MD_FILE} ${CMAKE_CURRENT_BINARY_DIR}/${relpath}")
+
+add_custom_command(
+        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${relpath}
+	COMMAND ${Python3_EXECUTABLE} ${voicexx_SOURCE_DIR}/ci_tools/readme_to_doxygen.py ${MD_FILE} ${CMAKE_CURRENT_BINARY_DIR}/${relpath}
+        DEPENDS ${MD_FILE} ${voicexx_SOURCE_DIR}/ci_tools/readme_to_doxygen.py
+        VERBATIM
+)
+
+SET(TREATED_MD ${TREATED_MD} ${CMAKE_CURRENT_BINARY_DIR}/${relpath})
+
+endforeach()
+
+message(STATUS ${TREATED_MD})
+
 doxygen_add_docs(doc
-	${MarkdownSources}
-        "${voicexx_SOURCE_DIR}/src"
-	"${voicexx_SOURCE_DIR}/vendor/sll/README.md"
-        "${voicexx_SOURCE_DIR}/vendor/sll/include"
+	${TREATED_MD}
         ALL)

docs/CODING_STANDARD.md

@@ -1,4 +1,4 @@
-@page conventions Coding Standards
+# Coding Standards
 
 ## C++ Features
 * We use C++17

docs/Using_git.md

@@ -1,4 +1,4 @@
-@page git Using Git
+# Using Git
 
 Git is an extremely important tool for handling versioning and concurrent developments on the same code. There are many resources available online to help you get to grips with this tool. Here we summarise a few key concepts which will be helpful when working with git and Gyselalibxx. It is designed to be a jumping-off point and a reference for reoccuring git questions, not a complete git reference or tutorial.
 

src/README.md

@@ -1,11 +1,11 @@
-@page src "Gyselalib++ contents"
+# Gyselalib++ contents
 
 The `src/` folder contains all the code necessary to build a gyrokinetic semi-Lagrangian simulation. It is broken up into the following sub-folders:
 
-<!-- - @subpage advection - Code describing semi-Lagrangian advection routines. -->
-- @subpage geometryRTheta - Code describing methods which are specific to a 2D curvilinear geometry containing a singular point.
-<!-- - @subpage geometryXVx - Code describing methods which are specific to a simulation with 1 spatial dimension and 1 velocity dimension. -->
-- @subpage interpolation - Code describing interpolation methods.
-<!-- - @subpage paraconfpp - Paraconf utility functions. -->
-<!-- - @subpage quadrature - Code describing different quadrature methods. -->
-<!-- - @subpage speciesinfo - Code used to describe the different species. -->
+<!-- - [advection](./advection/README.md) - Code describing semi-Lagrangian advection routines. -->
+- [geometryRTheta](./geometryRTheta/README.md) - Code describing methods which are specific to a 2D curvilinear geometry containing a singular point.
+<!-- - [geometryXVx](./geometryXVx/README.md) - Code describing methods which are specific to a simulation with 1 spatial dimension and 1 velocity dimension. -->
+- [interpolation](./interpolation/README.md) - Code describing interpolation methods.
+<!-- - [paraconfpp](./paraconfpp/README.md) - Paraconf utility functions. -->
+<!-- - [quadrature](./quadrature/README.md) - Code describing different quadrature methods. -->
+<!-- - [speciesinfo](./speciesinfo/README.md) - Code used to describe the different species. -->

src/geometryRTheta/README.md

@@ -1,10 +1,10 @@
- @page geometryRTheta Geometry (r, theta)
+# Geometry (r, theta)
 
 The `geoemtryRTheta` folder contains all the code describing methods which are specific to a 2D curvilinear geometry containing a singular point. It is broken up into the following sub-folders:
 
 
-- @subpage Spline_interpolator_polar - Code describing interpolation methods on 2D polar domain. 
+- [interpolation](./interpolation/README.md) - Code describing interpolation methods on 2D polar domain. 
 
-<!-- - @subpage Polar_Poisson_solver --> - Code describing the polar Poisson solver.
+<!-- - [poisson](./poisson/README.md) --> - Code describing the polar Poisson solver.
 
-<!-- - @subpage GeometryRTheta  --> - All the dimension tags used for a gyrokinetic semi-Lagrangian simulation in a curvilinear geoemtry.
+<!-- - [geometry](./geometry/README.md)  --> - All the dimension tags used for a gyrokinetic semi-Lagrangian simulation in a curvilinear geoemtry.

src/geometryRTheta/interpolation/README.md

@@ -1,8 +1,8 @@
-@page Spline_interpolator_polar Spline interpolator in polar coordinates
+# Spline interpolator in polar coordinates
 
-Use a function basis built from a product of two 1D spline bases in @f$ r @f$ and in @f$ \theta @f$. This interpolator does **not** use the polar spline basis with special treatment for the center. This means that care must be taken when using this interpolator, to ensure that numerically identical values are provided at @f$ r=0 @f$. If this rule is respected then the representation will be continuous.
+Use a function basis built from a product of two 1D spline bases in $r$ and in $\theta$. This interpolator does **not** use the polar spline basis with special treatment for the center. This means that care must be taken when using this interpolator, to ensure that numerically identical values are provided at $r=0$. If this rule is respected then the representation will be continuous.
 
-The interpolator is built from a product of b-splines if @f$ r @f$ and @f$ \theta @f$ rather than using a polar spline basis as the former is significantly less costly than the latter. Therefore the polar spline basis should only be used when additional smoothness is required.
+The interpolator is built from a product of b-splines if $r$ and $\theta$ rather than using a polar spline basis as the former is significantly less costly than the latter. Therefore the polar spline basis should only be used when additional smoothness is required.
 
 
 The `operator()` takes as input the values of the function we want to interpolate and the coordinates where we want to interpolate. It computes the coefficients on the 2D bsplines basis and evaluates the decomposition on the coordinates given as input. 

src/interpolation/README.md

@@ -1,10 +1,10 @@
-@page interpolation Interpolation Methods
+# Interpolation Methods
 
-Interpolation methods are any methods which, given values @f$f_j=f(x_j)@f$, allow the construction of an interpolating function @f$\phi(x)@f$ such that:
+Interpolation methods are any methods which, given values $f_j=f(x_j)$, allow the construction of an interpolating function $\phi(x)$ such that:
 
-@f$\phi(x_j)=f_j=f(x_j)@f$
+$\phi(x_j)=f_j=f(x_j)$
 
-and return the value of that interpolating function @f$\phi(x)@f$ at the desired coordinates.
+and return the value of that interpolating function $\phi(x)$ at the desired coordinates.
 
 The general interface for this method is defined via IInterpolator.
 
@@ -13,9 +13,9 @@ The interpolation methods implemented are:
 
 ## Spline Interpolation
 
-Interpolation by a spline interpolating function is implemented in the class SplineInterpolator. In this case the interpolating function @f$\phi(x)@f$ is a spline. The basis splines are passed as a template parameter, it is these that determine the order of the spline interpolation.
+Interpolation by a spline interpolating function is implemented in the class SplineInterpolator. In this case the interpolating function $\phi(x)$ is a spline. The basis splines are passed as a template parameter, it is these that determine the order of the spline interpolation.
 
-In order for the interpolation to function correctly the values @f$f_j=f(x_j)@f$ provided must be located at the points @f$x_j@f$ identified as the spline interpolation points.
+In order for the interpolation to function correctly the values $f_j=f(x_j)$ provided must be located at the points $x_j$ identified as the spline interpolation points.
 
 The spline interpolation method is based entirely on the SplineBuilder and SplineEvaluator classes which are found in @ref sll.