Merge branch 'master' into feature/group-invite-link

.github/workflows/release.yml

@@ -9,10 +9,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js 12.x
+      - name: Use Node.js 16.x
         uses: actions/setup-node@v1
         with:
-          node-version: '12'
+          node-version: '16'
       - name: Build web bundle
         working-directory: mwdb/web
         run: |
@@ -95,4 +95,4 @@ jobs:
             certpl/mwdb-web-tests:latest
           cache-from: |
             type=registry,ref=certpl/mwdb-web-tests:buildcache
-          push: true
+          push: true

.readthedocs.yml

@@ -0,0 +1,22 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"

deploy/docker/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 RUN python3 -m venv /app/venv
 RUN /app/venv/bin/pip --no-cache-dir install wheel
 
-RUN apk add --no-cache libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo postgresql-dev
+RUN apk add --no-cache libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo postgresql-dev libfuzzy2-dev
 
 COPY requirements.txt /app
 RUN /app/venv/bin/pip --no-cache-dir install -r /app/requirements.txt
@@ -18,7 +18,7 @@ FROM python:3.8-alpine
 
 LABEL maintainer="[email protected]"
 
-RUN apk add --no-cache postgresql-client postgresql-dev libmagic
+RUN apk add --no-cache postgresql-client postgresql-dev libmagic libfuzzy2-dev
 
 # Copy backend files
 COPY --from=build /app/venv /app/venv

dev/bump_version.json

@@ -4,7 +4,6 @@
         "mwdb/version.py": "app_version = \"$VERSION\"",
         "mwdb/web/package.json": "\"name\": \"mwdb-web\",\\s+\"version\": \"$VERSION\"",
         "mwdb/web/package-lock.json": "\"name\": \"mwdb-web\",\\s+\"version\": \"$VERSION\"",
-        "mwdb/web/src/commons/package.json": "\"version\": \"$VERSION\"",
         "docs/conf.py": "release = '$VERSION'",
         "docker-compose.yml": "image: certpl/mwdb(?:-web)?\\:v$VERSION"
     },

docker-compose-e2e.yml

@@ -93,7 +93,7 @@ services:
     ports:
       - "127.0.0.1:8025:8025"
   karton-system:
-    image: certpl/karton-system:v5.0.0
+    image: certpl/karton-system:v5.1.0
     depends_on:
       - redis
       - minio
@@ -102,14 +102,14 @@ services:
     entrypoint: karton-system
     command: --setup-bucket
   karton-classifier:
-    image: certpl/karton-classifier:v1.4.0
+    image: certpl/karton-classifier:v2.0.0
     depends_on:
       - redis
       - minio
     volumes:
       - "./dev/karton.ini:/etc/karton/karton.ini"
   karton-dashboard:
-    image: certpl/karton-dashboard:v1.4.0
+    image: certpl/karton-dashboard:v1.5.0
     depends_on:
       - redis
       - minio
@@ -118,7 +118,7 @@ services:
     ports:
       - "127.0.0.1:8030:5000"
   karton-mwdb-reporter:
-    image: certpl/karton-mwdb-reporter:v1.2.0
+    image: certpl/karton-mwdb-reporter:v1.3.0
     depends_on:
       - redis
       - minio

docker-compose.yml

@@ -9,7 +9,7 @@ services:
     depends_on:
       - postgres
       - redis
-    image: certpl/mwdb:v2.9.0
+    image: certpl/mwdb:v2.10.3
     restart: on-failure
     env_file:
       # NOTE: use gen_vars.sh in order to generate this file
@@ -22,7 +22,7 @@ services:
     build:
       context: .
       dockerfile: deploy/docker/Dockerfile-web
-    image: certpl/mwdb-web:v2.9.0
+    image: certpl/mwdb-web:v2.10.3
     ports:
       - "80:80"
     restart: on-failure

docker/gunicorn.conf.py

@@ -3,5 +3,6 @@
 wsgi_app = "mwdb.app:app"
 bind = "0.0.0.0:8080"
 user = "nobody"
+preload_app = bool(int(os.getenv("PRELOAD_APP", "0")))
 reload = bool(int(os.getenv("HOT_RELOAD", "0")))
 workers = int(os.getenv("GUNICORN_WORKERS", "4"))

docker/mwdb.ini

@@ -9,6 +9,6 @@ local_plugins_autodiscover = 1
 request_timeout = 20000
 file_upload_timeout = 60000
 statement_timeout = 15000
-
+use_x_forwarded_for = 1
 
 [mwdb_limiter]

docs/conf.py

@@ -21,7 +21,7 @@
 author = 'CERT Polska'
 
 # The full version, including alpha/beta/rc tags
-release = '2.9.0'
+release = '2.10.3'
 
 
 # -- General configuration ---------------------------------------------------

docs/integration-guide.rst

@@ -220,4 +220,235 @@ List of available hooks and events triggering these hooks.
   * relation between parent and child objects was added
   * relation between parent and child objects was removed
   * attribute was assigned to object
-  * attribute was removed from object
+  * attribute was removed from object
+
+Creating web plugins
+--------------------
+
+MWDB Core comes with powerful web plugin engine which allows to extend almost any component within MWDB UI. For a long
+time it was an undocumented feature used mainly by mwdb.cert.pl service and built on top of Create React App hacks and
+overrides.
+
+Starting from v2.9.0 release, we're using joined powers of `Vite <https://vitejs.dev/>`_ and `Rollup <https://rollupjs.org/>`_ to make it a real thing.
+
+Web plugins: getting started
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Frontend plugins are a very different animal from Python backend plugins and you may need a bit more knowledge
+about build-time mechanisms.
+
+Let's go to the ``docker/plugins`` directory within ``mwdb-core`` repository and extend our ``hello_world`` plugin:
+
+.. code-block::
+
+    docker
+    └── plugins
+        └── hello_world
+            └── __init__.py
+   +        └── index.jsx
+   +        └── package.json
+
+First is ``package.json`` that contains short specification of our plugin. Name must contain ``@mwdb-web/plugin-`` prefix.
+
+.. code-block:: json
+
+   {
+       "name": "@mwdb-web/plugin-hello-world",
+       "version": "0.0.1",
+       "main": "./index.jsx"
+   }
+
+Finally we can write simple plugin that adds new ``Hello world`` page. Let's check ``index.jsx` contents:
+
+.. code-block:: jsx
+
+    // Imports from React and Font Awesome libraries
+    import React from 'react';
+    import { Route, Link } from 'react-router-dom';
+    import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+    import { faHandHoldingHeart } from "@fortawesome/free-solid-svg-icons";
+
+    // Import from MWDB Core commons and components
+    import { View } from "@mwdb-web/commons/ui";
+    import { AboutView } from "@mwdb-web/components/Views/AboutView";
+
+    function HelloWorld() {
+        return (
+            <View>
+                <h1>Hello world!</h1>
+                Nice to see you!
+                <hr />
+                <About/>
+            </View>
+        )
+    }
+
+    export default () => ({
+        routes: [
+            <Route path='hello' element={<HelloWorld />} />
+        ],
+        navbarAfter: [
+            () => (
+                <li className="nav-item">
+                    <Link className="nav-link" to={"/hello"}>
+                        <FontAwesomeIcon
+                            className="navbar-icon"
+                            icon={faHandHoldingHeart}
+                        />
+                        Hello there!
+                    </Link>
+                </li>
+            )
+        ],
+    })
+
+After setting up all of the things, run ``docker-compose -f docker-compose-dev.yml build`` and ``docker-compose -f docker-compose-dev.yml up``
+to run the application. If everything is OK, you should see the results like below:
+
+<show the result>
+
+But what actually happened in that ``index.jsx`` file? there are lots of things going there!
+
+Let's focus on most important ones:
+
+* Line starting with ``export default`` is actually an entrypoint of our plugin. It exports callback that is called
+  after plugin is loaded.
+
+* Entrypoint callback is expected to return an object that contains specification of extensions provided by plugin.
+
+* Our plugin contains two extensions:
+  * ``routes`` that implement React Router routes to be included in web application
+  * ``navbarAfter`` being a list of React component functions that will be rendered after ``navbar``
+
+* Plugins adds new navbar button ``Hello there!`` and ``/hello`` route rendering ``HelloWorld`` component.
+  Our new component uses ``View`` from ``@mwdb-web/commons/ui`` which is common wrapper for main views used
+  within application. In addition, it renders ``About`` view imported from ``@mwdb-web/components`` just under our gretting.
+
+But where is actual list of possible extensions defined? They're defined in core application code and can be found
+by references to few methods and wrappers from ``common/plugins`` :
+
+* ``fromPlugins`` collects specific type of extension from all loaded plugins and returns a list of them. For example: new ``routes``
+  to be added.
+* ``Extension`` does the same but treats all collected objects as components and renders them.
+* ``Extendable`` wraps object with ``<name>Before``, ``<name>Replace`` and ``<name>After`` extensions, so we can add extra things
+  within main views.
+
+So ``navbar`` is one of ``Extendable`` wrappers that can be found within application and that's why we can add extra navbar item.
+
+Web plugins: how it works internally?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two requirements to be fulfilled by the plugin engine:
+
+* Plugin code needs to be loaded and executed along with the core application
+* Plugin must be allowed to reuse and extend the core application parts
+
+MWDB uses Rollup import aliases and Vite virtual modules to make a link between plugin code and the core.
+
+1. Vite build runtime looks for ``@mwdb-web/plugin-*`` packages that are installed in ``node_modules``
+
+   They can be regular packages or just links (see also `npm-link <https://docs.npmjs.com/cli/v9/commands/npm-link>`_)
+
+2. ``@mwdb-web/plugins`` virtual module defines dynamic imports that are further resolved by Vite to create separate bundles
+   for plugins that can be asynchronically loaded.
+
+   Virtual module code looks like below:
+
+   .. code-block:: javascript
+
+     export default {
+         "plugin-example": import("@mwdb-web/plugin-example"),
+         ...
+     }
+
+3. ``@mwdb-web/plugins`` package is then resolved at runtime by ``commons/plugins`` loader that resolves dynamic imports
+   and collects hook specification. Plugins are loaded before first render occurs.
+
+When plugin loader finishes its job, initial render kicks in and plugin is finally able to extend the application.
+MWDB uses Rollup capabilities to make plugins able to use components from web source root (``mwdb/web/src``) and expose them
+as ``@mwdb-web/*`` aliases:
+
+* ``@mwdb-web/commons`` contains core parts of application that are expected to be used by plugins as well. Main packages are:
+    - ``api`` module serving backend REST API bindings built on top of Axios
+    - ``auth`` module serving ``AuthContext`` with information about currently authenticated user
+    - ``config`` module with current server configuration and useful globals
+    - ``helpers`` with useful helper methods
+    - ``ui`` with UI components and utilities
+
+* ``@mwdb-web/components`` contains implementation of all application views and there is higher chance that something will
+  break across the versions if you use them directly.
+
+Building customized images
+--------------------------
+
+If you want to extend MWDB with new features using the plugin system, it's always useful to be able to build your own customized Docker images.
+
+There are two ways to do that:
+
+1. Simple way: clone https://github.com/CERT-Polska/mwdb-core repository. Then place your plugins
+   in ``docker/plugins`` and use Dockerfiles from ``deploy/docker`` to build everything from scratch.
+2. More extensible way: use ``certpl/mwdb`` and ``certpl/mwdb-web-source`` as base images and make your own Dockerfiles. This method
+   enables you to install additional dependencies and provide custom plugin-specific overrides.
+
+Building custom backend image is simple as in `Dockerfile` below:
+
+.. code-block:: Dockerfile
+
+    # It's recommended to pin to specific version
+    ARG MWDB_VERSION=v2.9.0
+    FROM certpl/mwdb:$MWDB_VERSION
+
+    # Install any Alpine dependencies you need
+    RUN apk add p7zip
+    # Install any Python dependencies you need (certpl/mwdb image uses venv internally)
+    RUN /app/venv/bin/pip install malduck
+
+    # Copy arbitrary backend plugins and mail templates
+    COPY mail_templates /app/mail_templates
+    COPY plugins /app/plugins
+
+Backend plugins are linked in runtime, so that part is pretty easy to extend. A bit more complicated thing is frontend part:
+
+.. code-block:: Dockerfile
+
+    ARG MWDB_VERSION=v2.9.0
+    FROM certpl/mwdb-web-source:$MWDB_VERSION AS build
+
+    # Copy web plugins code
+    COPY plugins /app/plugins
+
+    # Set workdir to /app, install plugins to ``/app/node_modules`` and rebuild everything
+    WORKDIR /app
+    RUN npm install --unsafe-perm $(find /app/plugins -name 'package.json' -exec dirname {} \; | sort -u) \
+        && CI=true npm run build
+
+    # Then next stage is copied from https://github.com/CERT-Polska/mwdb-core/blob/master/deploy/docker/Dockerfile-web
+    # You need to copy start-web.sh and ngnix.conf.template as well, or adapt them according to your needs
+    FROM nginx:stable
+
+    LABEL maintainer="[email protected]"
+
+    ENV PROXY_BACKEND_URL http://mwdb.:8080
+
+    COPY nginx.conf.template /etc/nginx/conf.d/default.conf.template
+    COPY start-web.sh /start-web.sh
+    COPY --from=build /app/dist /usr/share/nginx/html
+
+    # Give +r to everything in /usr/share/nginx/html and +x for directories
+    RUN chmod u=rX,go= -R /usr/share/nginx/html
+
+    # By default everything is owned by root - change owner to nginx
+    RUN chown nginx:nginx -R /usr/share/nginx/html
+
+    CMD ["/bin/sh", "/start-web.sh"]
+
+
+Room for improvement
+--------------------
+
+Plugin system was created mainly for mwdb.cert.pl, so not everything may fit your needs. Also things may break from time to time,
+but as we maintain our internal plugins ourselves, most important changes will be noted in changelog. You can also find broader explanation and
+migration recipes in :ref:`What's changed` chapter.
+
+So if you need another ``Extendable`` place within UI or yet another hook within backend: feel free to `create issue <https://github.com/CERT-Polska/mwdb-core/issues>`_ on
+our GitHub repository.