Merge branch 'release/0.5.0'

.github/workflows/test.yml

@@ -15,9 +15,9 @@ on:
       - 'release/**'
 
 jobs:
-  build:
+  test-back:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     services:
       # Label used to access the service container
@@ -39,34 +39,25 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
 
-    strategy:
-      matrix:
-        python-version: ['3.10']
-        node-version: ['16.x', '18.x']
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
+        python-version: '3.9'
+        cache: 'pip'
+    - name: Use Node.js
+      uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.node-version }}
+        node-version: 18
+        cache: 'npm'
     - name: Install Python dependencies
       run: |
         cd backend
         python -m pip install --upgrade pip
         pip install virtualenv
         pip install -r requirements.txt
-    - name: Lint with flake8
-      run: |
-        pip install flake8
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Run all tests
+    - name: Bootstrap application
       env:
         # Provide PostgreSQL environment variables in order to default to TCP connection
         PGDATABASE: lettercraft
@@ -76,10 +67,23 @@ jobs:
         PGPASSWORD: lettercraft_pwd
       run: |
         cat bootstrap_ci.txt | python bootstrap.py
-        yarn
-        yarn django migrate
-        yarn static-p
-        find static
-        yarn start-back-p &
-        chromedriver --version
-        yarn test
+    - name: Run all tests
+      run: |
+        cd backend
+        pytest
+  test-front:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: 'npm'
+      - name: Install dependencies
+        run: |
+          yarn fyarn
+      - name: Run tests
+        run: |
+          yarn test-front

.nvmrc

@@ -1 +1 @@
-v14.21.2
+v18.19.1

CITATION.cff

@@ -29,5 +29,5 @@ authors:
     website: 'https://cdh.uu.nl/rsl/'
 repository-code: 'https://github.com/CentreForDigitalHumanities/lettercraft'
 license: BSD-3-Clause
-version: 0.4.0
-date-released: '2024-06-10'
+version: 0.5.0
+date-released: '2024-10-01'

CONTRIBUTING.md

@@ -13,161 +13,135 @@ You need to install the following software:
  - [Visual C++ for Python][1] (Windows only)
  - Node.js >= 16, <= 18
  - Yarn
- - [WebDriver][2] for at least one browser (only for functional testing)
 
 [1]: https://wiki.python.org/moin/WindowsCompilers
-[2]: https://pypi.org/project/selenium/#drivers
 
 
 ## How it works
 
-This project integrates three isolated subprojects, each inside its own subdirectory with its own code, package dependencies and tests:
+This project integrates two isolated subprojects, each inside its own subdirectory with its own code, package dependencies and tests:
 
- - **backend**: the server side web application based on [Django][3] and [DRF][4]
-
- - **frontend**: the client side web application based on [Angular](https://angular.io)
-
- - **functional-tests**: the functional test suite based on [Selenium][6] and [pytest][7]
-
-[3]: https://www.djangoproject.com
-[4]: https://www.django-rest-framework.org
-[6]: https://www.selenium.dev/documentation/webdriver/
-[7]: https://docs.pytest.org/en/latest/
+- **backend**: the server side web application based on [Django](https://www.djangoproject.com)
+- **frontend**: the client side web application based on [Angular](https://angular.dev)
 
 Each subproject is configurable from the outside. Integration is achieved using "magic configuration" which is contained inside the root directory together with this document. In this way, the subprojects can stay truly isolated from each other.
 
 If you are reading this document, you'll likely be working with the integrated project as a whole rather than with one of the subprojects in isolation. In this case, this document should be your primary source of information on how to develop or deploy the project. However, we recommend that you also read the "How it works" section in the README of each subproject.
 
-
 ## Development
 
 ### Quickstart
 
 First time after cloning this project:
 
-```console
-$ python bootstrap.py
+```sh
+python bootstrap.py
 ```
 
-Running the application in [development mode](#development-mode-vs-production-mode) (hit ctrl-C to stop):
+This will also activate a Python virtual environment. From within the environment, you can run the application in [development mode](#development-mode-vs-production-mode) (hit ctrl-C to stop):
 
-```console
-$ yarn start
+```sh
+yarn start
 ```
 
-This will run the backend and frontend applications, as well as their unittests, and watch all source files for changes. You can visit the frontend on http://localhost:8000/, the browsable backend API on http://localhost:8000/api/ and the backend admin on http://localhost:8000/admin/. On every change, unittests rerun, frontend code rebuilds and open browser tabs refresh automatically (livereload).
+This will run the backend and frontend applications, and watch all source files for changes. On every change, unittests rerun, frontend code rebuilds and open browser tabs refresh automatically (livereload).
 
+Some useful URLs:
 
-### Recommended order of development
+- https://localhost:8000/ to view the homepage for the frontend
+- https://localhost:8000/admin/ to view the backend admin
+- https://localhost:8000/api/graphql to explore the GraphQL API
 
-For each new feature, we suggested that you work through the steps listed below. This could be called a back-to-front or "bottom up" order. Of course, you may have reasons to choose otherwise. For example, if very precise specifications are provided, you could move step 8 to the front for a more test-driven approach.
+### Working with GraphQL
 
-Steps 1–5 also include updating the unittests. Only functions should be tested, especially critical and nontrivial ones.
+Lettercraft uses a GraphQL API for research data. (User authentication / management is not handled with GraphQL.)
 
- 1. Backend model changes including migrations.
- 2. Backend serializer changes and backend admin changes.
- 3. Backend API endpoint changes.
- 4. Frontend model changes.
- 5. Other frontend unit changes (templates, views, routers, FSMs).
- 6. Frontend integration (globals, event bindings).
- 7. Run functional tests, repair broken functionality and broken tests.
- 8. [Add functional tests](/functional-tests/README.md#writing-tests) for the new feature.
- 9. Update technical documentation.
+If you have worked on other applications from our lab, working with GraphQL may be new to you. See [Working with GraphQL](https://github.com/CentreForDigitalHumanities/dh-graphql/blob/main/Working%20with%20GraphQL.md) for a detailed guide on working with GraphQL in a Django/Angular project.
 
-For release branches, we suggest the following checklist.
+One notable change from the development workflow in other applications is code generation: frontend types are automatically generated based on the backend definitions.
 
- 1. Bump the version number in the `package.json` next to this document.
- 2. Run the functional tests in production mode, fix bugs if necessary.
- 3. Try using the application in production mode, look for problems that may have escaped the tests.
- 4. Add regression tests (unit or functional) that detect problems from step 3.
- 5. Work on the code until new regression tests from step 4 pass.
- 6. Optionally, repeat steps 2–5 with the application running in a real deployment setup (see [Deployment](#deployment)).
+While you have a backend development server running, run `yarn codegen` in a separate terminal. This process will watch the frontend and backend for changes to keep generated types up to date.
 
+(Alternatively, you can `yarn codegen-once`. This will start a backend server, run code generation once, and exit. Note that this only works if the backend isn't already running, and does not work on Windows.)
+
+### Recommended order of development
+
+For new features, it often makes sense to use a "bottom-up" approach, described below. Of course, you may have reasons to choose otherwise.
+
+1. Update backend models, including migrations.
+2. Adjust or add backend functions, including unit tests.
+3. Adjust backend queries and mutations.
+4. Integrate updated queries/mutations in the frontend.
+5. Add new frontend functionality, including unit tests.
+6. Update technical documentation.
 
 ### Commands for common tasks
 
-The `package.json` next to this document defines several shortcut commands to help streamline development. In total, there are over 30 commands. Most may be regarded as implementation details of other commands, although each command could be used directly. Below, we discuss the commands that are most likely to be useful to you. For full details, consult the `package.json`.
+The `package.json` next to this document defines several shortcut commands to help streamline development. Most may be regarded as implementation details of other commands, although each command could be used directly. Below, we discuss the commands that are most likely to be useful to you. For full details, consult the `package.json`.
+
+#### Installation
 
 Install the pinned versions of all package dependencies in all subprojects:
 
-```console
-$ yarn
+```sh
+yarn
 ```
 
-Run backend and frontend in [production mode](#development-mode-vs-production-mode):
+Alternatively, you can use separate commands for the frontend and backend:
 
-```console
-$ yarn start-p
+```sh
+yarn install-back
+yarn install-front
 ```
 
-Run the functional test suite:
-
-```console
-$ yarn test-func [FUNCTIONAL TEST OPTIONS]
-```
+#### Running a development server
 
-The functional test suite by default assumes that you have the application running locally in production mode (i.e., on port `4200`). See [Configuring the browsers](/functional-tests/README.md#configuring-the-browsers) and [Configuring the base address](/functional-tests/README.md#configuring-the-base-address) in `functional-tests/README` for options.
+As described above, `yarn start` will start a development server. While it can be useful to start the whole application with a single command, most developers prefer to run each service in a separate terminal while working on the code. This makes the terminal output easier to read.
 
-Run *all* tests (mostly useful for continuous integration):
+You can do this with the following commands (running each in a separate terminal):
 
-```console
-$ yarn test [FUNCTIONAL TEST OPTIONS]
+```sh
+yarn start-back # start the backend
+yarn start-front # start the frontend
+yarn watch-back # send livereload requests when the backend is updated (optional)
+yarn codegen # run frontend code generation when the backend is updated
 ```
 
-Run an arbitrary command from within the root of a subproject:
+Note: it's not necessary to run `yarn codegen` if you won't be making changes that affect the GraphQL API. (For instance, if you're only working on the frontend.)
 
-```console
-$ yarn back  [ARBITRARY BACKEND COMMAND HERE]
-$ yarn front [ARBITRARY FRONTEND COMMAND HERE]
-$ yarn func  [ARBITRARY FUNCTIONAL TESTS COMMAND HERE]
-```
+#### Running unit tests
 
-For example,
+You can run all unit tests with:
 
-```console
-$ yarn back less README.md
+```sh
+yarn test
 ```
 
-is equivalent to
+Backend tests are run with [pytest](https://docs.pytest.org), frontend tests with [jasmine](https://jasmine.github.io/). For more detailed control over testing, like running a single test, consult the documentation of those libraries.
 
-```console
-$ cd backend
-$ less README.md
-$ cd ..
-```
+#### Running Django commands
 
 Run `python manage.py` within the `backend` directory:
 
-```console
-$ yarn django [SUBCOMMAND] [OPTIONS]
+```sh
+yarn django [SUBCOMMAND] [OPTIONS]
 ```
 
 `yarn django` is a shorthand for `yarn back python manage.py`. This command is useful for managing database migrations, among other things.
 
-Manage the frontend package dependencies:
-
-```console
-$ yarn fyarn (add|remove|upgrade|...) (PACKAGE ...) [OPTIONS]
-```
-
-
+### Managing dependencies
 
-### Notes on Python package dependencies
+#### Python dependencies
 
-Both the backend and the functional test suite are Python-based and package versions are pinned using [pip-tools][13] in both subprojects. For ease of development, you most likely want to use the same virtualenv for both and this is also what the `bootstrap.py` assumes.
+Python dependencies are managed using pip and recorded in [requirements.in](/backend/requirements.in) / [requirements.txt](/backend/requirements.txt). To update dependencies:
 
-[13]: https://pypi.org/project/pip-tools/
+- Edit `requirements.in` to describe the new dependencies.
+- Run `pip-compile` to update the full manifest in `requirements.txt`.
+- Run `pip install -r requirements.txt` to update your Python environment.
 
-This comes with a small catch: the subprojects each have their own separate `requirements.txt`. If you run `pip-sync` in one subproject, the dependencies of the other will be uninstalled. In order to avoid this, you run `pip install -r requirements.txt` instead. The `yarn` command does this correctly by default.
-
-Another thing to be aware of, is that `pip-compile` takes the old contents of your `requirements.txt` into account when building the new version based on your `requirements.in`. You can use the following trick to keep the requirements in both projects aligned so the versions of common packages don't conflict:
-
-```console
-$ yarn back pip-compile
-# append contents of backend/requirements.txt to functional-tests/requirements.txt
-$ yarn func pip-compile
-```
+#### Frontend dependencies
 
+Frontend dependencies are managed using [yarn](https://classic.yarnpkg.com/en/). See the [usage guide](https://classic.yarnpkg.com/en/docs/usage) for details.
 
 ### Development mode vs production mode
 

backend/case_study/admin.py

@@ -5,5 +5,5 @@
 @admin.register(models.CaseStudy)
 class CaseStudyAdmin(admin.ModelAdmin):
     list_display = ["name", "description"]
-    fields = ["name", "description", "episodes", "key_persons", "key_sites"]
-    filter_horizontal = ["episodes", "key_persons", "key_sites"]
+    fields = ["name", "description", "series", "key_persons", "key_sites"]
+    filter_horizontal = ["series", "key_persons", "key_sites"]

backend/case_study/migrations/0004_alter_casestudy_episodes.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.7 on 2024-06-27 07:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("event", "0017_remove_eventdescriptionagent_source_terminology_and_more"),
+        ("case_study", "0003_casestudy_key_persons_casestudy_key_sites"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="casestudy",
+            name="episodes",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Series involved in this case study",
+                to="event.episode",
+            ),
+        ),
+    ]

backend/case_study/migrations/0005_rename_episodes_casestudy_series.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-06-27 07:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("case_study", "0004_alter_casestudy_episodes"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="casestudy",
+            old_name="episodes",
+            new_name="series",
+        ),
+    ]