Merge branch 'main' into whatsapp-integration

.github/workflows/pytest.yml

@@ -93,7 +93,7 @@ jobs:
           psql -h localhost -U postgres geoname_testing -c "grant all privileges on schema public to $(whoami); grant all privileges on all tables in schema public to $(whoami); grant all privileges on all sequences in schema public to $(whoami);"
       - name: Test with pytest
         run: |
-          pytest -vv --showlocals --cov=funnel
+          pytest --gherkin-terminal-reporter -vv --showlocals --cov=funnel
       - name: Prepare coverage report
         run: |
           mkdir -p coverage

.pre-commit-config.yaml

@@ -152,6 +152,8 @@ repos:
         args: ['--remove']
       - id: forbid-new-submodules
       - id: mixed-line-ending
+      - id: name-tests-test
+        args: ['--pytest']
       - id: no-commit-to-branch
       - id: requirements-txt-fixer
         files: requirements/.*\.in

Makefile

@@ -57,6 +57,9 @@ deps-editable:
 deps-python: deps-editable
 	pip-compile-multi --backtracking --use-cache
 
+deps-python-noup:
+	pip-compile-multi --backtracking --use-cache --no-upgrade
+
 deps-python-rebuild: deps-editable
 	pip-compile-multi --backtracking
 

funnel/assets/sass/components/_markdown.scss

@@ -5,8 +5,6 @@
 .markdown {
   overflow-wrap: break-word;
   overflow: auto;
-  margin: 0 -$mui-grid-padding;
-  padding: 0 $mui-grid-padding;
 
   h1,
   h2,
@@ -17,13 +15,17 @@
     font-weight: 700;
     a {
       color: $mui-text-dark;
+      text-decoration: none;
+    }
+    a.header-anchor {
       display: none;
     }
-    &:hover a {
+    &:hover a.header-anchor {
       display: inline;
+      color: $mui-text-accent;
     }
     @media (any-pointer: coarse) {
-      a {
+      a.header-anchor {
         display: inline;
         color: $mui-text-accent;
       }

funnel/assets/sass/components/_tabs.scss

@@ -35,7 +35,7 @@
 
 .md-tablist-wrapper {
   display: flex;
-  margin: 0 -16px;
+  margin: 0 -$mui-grid-padding/2;
   align-items: center;
 
   > .mui-tabs__bar:not(.mui-tabs__bar--pills) {

funnel/models/site_membership.py

@@ -25,16 +25,22 @@ class SiteMembership(
     __allow_unmapped__ = True
 
     # List of is_role columns in this model
-    __data_columns__ = {'is_comment_moderator', 'is_user_moderator', 'is_site_editor'}
+    __data_columns__ = {
+        'is_comment_moderator',
+        'is_user_moderator',
+        'is_site_editor',
+        'is_sysadmin',
+    }
 
     __roles__ = {
-        'all': {
+        'subject': {
             'read': {
                 'urls',
                 'user',
                 'is_comment_moderator',
                 'is_user_moderator',
                 'is_site_editor',
+                'is_sysadmin',
             }
         }
     }
@@ -47,15 +53,21 @@ class SiteMembership(
     # Site admin roles (at least one must be True):
 
     #: Comment moderators can delete comments
-    is_comment_moderator: Mapped[bool] = sa.Column(
+    is_comment_moderator: Mapped[bool] = sa.orm.mapped_column(
         sa.Boolean, nullable=False, default=False
     )
     #: User moderators can suspend users
-    is_user_moderator: Mapped[bool] = sa.Column(
+    is_user_moderator: Mapped[bool] = sa.orm.mapped_column(
         sa.Boolean, nullable=False, default=False
     )
     #: Site editors can feature or reject projects
-    is_site_editor: Mapped[bool] = sa.Column(sa.Boolean, nullable=False, default=False)
+    is_site_editor: Mapped[bool] = sa.orm.mapped_column(
+        sa.Boolean, nullable=False, default=False
+    )
+    #: Sysadmins can manage technical settings
+    is_sysadmin: Mapped[bool] = sa.orm.mapped_column(
+        sa.Boolean, nullable=False, default=False
+    )
 
     @declared_attr.directive
     @classmethod
@@ -68,6 +80,7 @@ def __table_args__(cls) -> tuple:
                     cls.is_comment_moderator.is_(True),
                     cls.is_user_moderator.is_(True),
                     cls.is_site_editor.is_(True),
+                    cls.is_sysadmin.is_(True),
                 ),
                 name='site_membership_has_role',
             )
@@ -99,6 +112,8 @@ def offered_roles(self) -> Set[str]:
             roles.add('user_moderator')
         if self.is_site_editor:
             roles.add('site_editor')
+        if self.is_sysadmin:
+            roles.add('sysadmin')
         return roles
 
 
@@ -140,6 +155,14 @@ def is_site_editor(self) -> bool:
             and self.active_site_membership.is_site_editor
         )
 
+    @cached_property
+    def is_sysadmin(self) -> bool:
+        """Test if this user is a sysadmin."""
+        return (
+            self.active_site_membership is not None
+            and self.active_site_membership.is_sysadmin
+        )
+
     # site_admin means user has one or more of above roles
     @cached_property
     def is_site_admin(self) -> bool:

funnel/utils/mustache.py

@@ -14,7 +14,9 @@
 __all__ = ['mustache_html', 'mustache_md']
 
 
-def _render_with_escape(name: str, escapefunc: Callable[[str], str]) -> render:
+def _render_with_escape(
+    name: str, escapefunc: Callable[[str], str]
+) -> Callable[..., str]:
     """Make a copy of Chevron's render function with a replacement HTML escaper."""
     _globals = copy(render.__globals__)
     _globals['_html_escape'] = escapefunc

funnel/views/siteadmin.py

@@ -14,6 +14,11 @@
 
 from flask import abort, current_app, flash, render_template, request, url_for
 
+try:
+    import rq_dashboard
+except ModuleNotFoundError:
+    rq_dashboard = None
+
 from baseframe import _
 from baseframe.forms import Form
 from coaster.auth import current_auth
@@ -79,6 +84,30 @@ def wrapper(*args, **kwargs) -> Any:
     return cast(WrappedFunc, wrapper)
 
 
+def requires_site_editor(f: WrappedFunc) -> WrappedFunc:
+    """Decorate a view to require site editor privilege."""
+
+    @wraps(f)
+    def wrapper(*args, **kwargs) -> Any:
+        if not current_auth.user or not current_auth.user.is_site_editor:
+            abort(403)
+        return f(*args, **kwargs)
+
+    return cast(WrappedFunc, wrapper)
+
+
+def requires_user_moderator(f: WrappedFunc) -> WrappedFunc:
+    """Decorate a view to require user moderator privilege."""
+
+    @wraps(f)
+    def wrapper(*args, **kwargs) -> Any:
+        if not current_auth.user or not current_auth.user.is_user_moderator:
+            abort(403)
+        return f(*args, **kwargs)
+
+    return cast(WrappedFunc, wrapper)
+
+
 def requires_comment_moderator(f: WrappedFunc) -> WrappedFunc:
     """Decorate a view to require comment moderator privilege."""
 
@@ -91,6 +120,18 @@ def wrapper(*args, **kwargs) -> Any:
     return cast(WrappedFunc, wrapper)
 
 
+def requires_sysadmin(f: WrappedFunc) -> WrappedFunc:
+    """Decorate a view to require sysadmin privilege."""
+
+    @wraps(f)
+    def wrapper(*args, **kwargs) -> Any:
+        if not current_auth.user or not current_auth.user.is_sysadmin:
+            abort(403)
+        return f(*args, **kwargs)
+
+    return cast(WrappedFunc, wrapper)
+
+
 @route('/siteadmin')
 class SiteadminView(ClassView):
     """Site administrator views."""
@@ -400,3 +441,9 @@ def review_comment(self, report: str) -> ReturnRenderWith:
 
 
 SiteadminView.init_app(app)
+
+if rq_dashboard is not None:
+    rq_dashboard.blueprint.before_request(
+        lambda: None if current_auth and current_auth.user.is_sysadmin else abort(403)
+    )
+    app.register_blueprint(rq_dashboard.blueprint, url_prefix='/siteadmin/rq')

migrations/versions/7aa9eb80aab4_add_sysadmin_flag.py

@@ -0,0 +1,70 @@
+"""Add sysadmin flag.
+
+Revision ID: 7aa9eb80aab4
+Revises: 2151c9f8e955
+Create Date: 2023-03-08 22:10:27.937483
+
+"""
+
+from typing import Optional, Tuple, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = '7aa9eb80aab4'
+down_revision: str = '2151c9f8e955'
+branch_labels: Optional[Union[str, Tuple[str, ...]]] = None
+depends_on: Optional[Union[str, Tuple[str, ...]]] = None
+
+
+def upgrade(engine_name='') -> None:
+    """Upgrade all databases."""
+    # Do not modify. Edit `upgrade_` instead
+    globals().get(f'upgrade_{engine_name}', lambda: None)()
+
+
+def downgrade(engine_name='') -> None:
+    """Downgrade all databases."""
+    # Do not modify. Edit `downgrade_` instead
+    globals().get(f'downgrade_{engine_name}', lambda: None)()
+
+
+def upgrade_() -> None:
+    """Upgrade database bind ''."""
+    with op.batch_alter_table('site_membership', schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                'is_sysadmin',
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.sql.expression.false(),
+            )
+        )
+        batch_op.alter_column('is_sysadmin', server_default=None)
+        batch_op.drop_constraint('site_membership_has_role', type_='check')
+        batch_op.create_check_constraint(
+            'site_membership_has_role',
+            'is_comment_moderator IS true OR is_user_moderator IS true'
+            ' OR is_site_editor IS true OR is_sysadmin IS true',
+        )
+
+
+def downgrade_() -> None:
+    """Downgrade database bind ''."""
+    with op.batch_alter_table('site_membership', schema=None) as batch_op:
+        batch_op.drop_constraint('site_membership_has_role', type_='check')
+        batch_op.create_check_constraint(
+            'site_membership_has_role',
+            'is_comment_moderator IS true OR is_user_moderator IS true'
+            ' OR is_site_editor IS true',
+        )
+        batch_op.drop_column('is_sysadmin')
+
+
+def upgrade_geoname() -> None:
+    """Upgrade database bind 'geoname'."""
+
+
+def downgrade_geoname() -> None:
+    """Downgrade database bind 'geoname'."""

pyproject.toml

@@ -188,6 +188,7 @@ disable = [
   'duplicate-code',  # Too many false positives
   'fixme',  # Our workflow is to tag for future fixes
   'invalid-name',  # Flake8 covers our naming convention requirements
+  'line-too-long',  # Long lines are okay if Black doesn't wrap them
   'no-member',  # Pylint gets confused over how some members become part of an instance
   'too-few-public-methods',  # Data classes and validator classes have few methods
   'too-many-ancestors',  # Our models have a large number of mixin classes

requirements/base.in

@@ -56,6 +56,7 @@ qrcode
 requests
 rich
 rq
+git+https://github.com/jace/rq-dashboard#egg=rq-dashboard
 SQLAlchemy
 sqlalchemy-json
 SQLAlchemy-Utils

requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:87537613ba4f34ca5f6e28e02432eb68315d15ed
+# SHA1:3df4671c6231d4189c117aae5bf414a9948fc01c
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -27,6 +27,8 @@ argon2-cffi==21.3.0
     # via -r requirements/base.in
 argon2-cffi-bindings==21.2.0
     # via argon2-cffi
+arrow==1.2.3
+    # via rq-dashboard
 async-timeout==4.0.2
     # via
     #   aiohttp
@@ -124,6 +126,7 @@ flask==2.2.3
     #   flask-rq2
     #   flask-sqlalchemy
     #   flask-wtf
+    #   rq-dashboard
 flask-assets==2.0
     # via
     #   -r requirements/base.in
@@ -339,6 +342,7 @@ pypng==0.20220715.0
 python-dateutil==2.8.2
     # via
     #   -r requirements/base.in
+    #   arrow
     #   baseframe
     #   freezegun
     #   icalendar
@@ -370,7 +374,11 @@ redis==4.5.1
     #   baseframe
     #   flask-redis
     #   flask-rq2
+    #   redis-sentinel-url
     #   rq
+    #   rq-dashboard
+redis-sentinel-url==1.0.1
+    # via rq-dashboard
 regex==2022.10.31
     # via nltk
 requests==2.28.2
@@ -400,7 +408,10 @@ rq==1.13.0
     #   -r requirements/base.in
     #   baseframe
     #   flask-rq2
+    #   rq-dashboard
     #   rq-scheduler
+rq-dashboard @ git+https://github.com/jace/rq-dashboard
+    # via -r requirements/base.in
 rq-scheduler==0.13.0
     # via flask-rq2
 rsa==4.9

runfrontendtests.sh

@@ -5,11 +5,11 @@ export FLASK_ENV=testing
 # For macos: https://stackoverflow.com/a/52230415/78903
 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
-python -m tests.cypress.frontend_tests_initdb
+python -m tests.cypress.cypress_initdb_test
 flask run -p 3002 --no-reload --debugger 2>&1 1>/tmp/funnel-server.log & echo $! > /tmp/funnel-server.pid
 function killserver() {
     kill $(cat /tmp/funnel-server.pid)
-    python -m tests.cypress.frontend_tests_dropdb
+    python -m tests.cypress.cypress_dropdb_test
     rm /tmp/funnel-server.pid
 }
 trap killserver INT