Merge branch 'main' into issue-1145-consitent-docker-usernames

.codespell/ignore_lines.txt

@@ -8,7 +8,7 @@
 ;; frontend/i18n/data/en.json5
 ;; Prettier insists we escape a single quote rather than the double quotes and codespell
 ;; does not understand the escaped `\'t` as "couldn't". It instead just sees "couldn".
-    heading: 'We couldn\'t find anything for "{query}".',
+  "noResults.heading": 'We couldn\'t find anything for "{query}".',
 
 ;; catalog/tests/dags/providers/provider_api_scripts/test_wikimedia_commons.py
 ;; "Titel" matches "title", but the phrase is in Dutch, not English, so "titel"

.github/renovate.json5

@@ -66,5 +66,10 @@
       groupName: "@testing-library",
       matchPackageNames: ["@testing-library{/,}**"],
     },
+    {
+      matchPackageNames: ["/elasticsearch/", "/postgres/", "/redis/"],
+      matchManagers: ["docker-compose", "dockerfile"],
+      enabled: false,
+    },
   ],
 }

.vale/styles/Openverse/TermCasing.yml

@@ -12,12 +12,13 @@ swap:
   # [^/\.] prevents matching things that look like URLs, file paths, or GitHub team mentions
   # For example: @WordPress/openverse-maintainers
   '[^/\.]openverse[^/\.]': Openverse
-  # OpenVerse should never be used, except as an example of something that is always wrong,
-  # in which case we'll tell Vale to ignore that line.
-  "OpenVerse": Openverse
   '[^/\.]wordpress[^/\.]': WordPress
-  # Wordpress is the same as OpenVerse
-  "Wordpress": WordPress
   '[^/\.]github[^/\.]': GitHub
-  # Github is the same as Wordpress and OpenVerse
+  '[^/\.]vue[^/\.]': Vue
+  # OpenVerse, Wordpress, Github and Glotpress should never be used, except as an example of
+  # something that is always wrong, in which case we'll tell Vale to ignore that line.
+  "OpenVerse": Openverse
+  "Wordpress": WordPress
   "Github": GitHub
+  "Glotpress": GlotPress
+  "vue": Vue

api/compose.yml

@@ -19,10 +19,13 @@ services:
         - PDM_INSTALL_ARGS=--dev
     image: openverse-api:${API_PDM_HASH:-latest}
     pull_policy: never
+    # During debugging, use the following command to override.
+    # command: python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 run.py
     volumes:
       - .:/api:z
       - ../packages/python:/packages/python:z
     ports:
+      # - "50256:5678" # Uncomment this to map port for debugging.
       - "50280:50280"
     depends_on:
       - db

api/pyproject.toml

@@ -53,6 +53,7 @@ dev = [
     "pgcli >=4.1, <5",
     "remote-pdb >=2.1.0, <3",
     "setuptools >=75.2,<76",
+    "debugpy >= 1.8.9,<2",
     # FIXME: Should be removed when Python build issues are resolved,
     # FIXME: since we don't explicitly use this library.
     # FIXME: https://github.com/WordPress/openverse/issues/5146

catalog/dags/common/cloudwatch.py

@@ -70,5 +70,7 @@ def enable_or_disable_alarms(enable):
         resource_type="cloudwatch",
     )
     cw_wrapper = CloudWatchWrapper(cloudwatch.get_conn())
-    cw_wrapper.enable_alarm_actions("ES Production CPU utilization above 50%", enable)
-    cw_wrapper.enable_alarm_actions("ES Prod CPU utilization above 90%", enable)
+    for percentage in [50, 90]:
+        cw_wrapper.enable_alarm_actions(
+            f"ES Production CPU utilization above {percentage}%", enable
+        )

catalog/requirements-dev.txt

@@ -5,10 +5,10 @@
 flaky==3.8.1
 freezegun==1.5.1
 ipython
-pook==2.0.0
+pook==2.1.2
 pytest-env
 pytest-mock
 pytest-raises==0.11
 pytest-socket==0.7.0
-pytest-sugar==0.9.7
+pytest-sugar==1.0.0
 pytest-xdist

documentation/api/guides/debugging.md

@@ -0,0 +1,71 @@
+# API debugging guidelines
+
+```{note}
+This is an opinionated guide that only applies to VS Code users. If you use a
+different editor or IDE, these instructions will not apply to you.
+```
+
+This is the guide to debugging the API using VS Code. This uses Microsoft's
+`debugpy` package.
+
+## Prerequisites
+
+1. Install the
+   [Python Debugger extension](https://marketplace.visualstudio.com/items?itemName=ms-python.debugpy).
+1. [Set up a VS Code workspace](/general/workspace.md) for the Openverse
+   monorepo.
+
+## Steps
+
+1. Add a launch configuration to the Openverse workspace configuration. This
+   configuration does the following things.
+
+   - Specifies that the debugger used should be `debugpy`.
+   - Configures the debugger to "attach" to a running process instead of
+     launching a new one.
+   - Specifies the port on which the `debugpy` server is running so that VS Code
+     can connect to it.
+   - Maps source code in the local repo clone to paths inside the Docker
+     container so you can set breakpoints in the editor.
+
+   ```json
+   {
+     // existing configuration
+     "launch": {
+       "version": "0.2.0",
+       "configurations": [
+         {
+           "name": "API",
+           "type": "debugpy",
+           "request": "attach",
+           "connect": {
+             "host": "localhost",
+             "port": 50256
+           },
+           "pathMappings": [
+             {
+               "localRoot": "${workspaceFolder:api}",
+               "remoteRoot": "/api"
+             }
+           ],
+           "justMyCode": true
+         }
+       ]
+     }
+   }
+   ```
+
+1. Edit the `compose.yml` file inside the `api/` directory to uncomment the
+   `command` field and port mapping for port 5678.
+
+1. Run the API server inside Docker using the instructions in the
+   [quickstart](/api/guides/quickstart.md) guide.
+
+1. Connect the debugger to the running instance from the Run and Debug panel.
+
+   ![VS Code Run and Debug panel](/_static/vs_code_debug_panel.png)
+
+1. Read the
+   [Visual Code debugger's official instructions](https://code.visualstudio.com/docs/editor/debugging)
+   to better understand how to use the debugger interface to accomplish tasks
+   like setting breakpoints, watching variables etc.

documentation/api/guides/index.md

@@ -6,4 +6,5 @@
 quickstart
 test
 documentation
+debugging
 ```

documentation/frontend/reference/i18n.md

@@ -9,7 +9,7 @@ WordPress uses GlotPress for managing translations, which is built on top of the
 Nuxt (and most JS-based i18n libraries) use
 [JSON](https://kazupon.github.io/vue-i18n/guide/formatting.html) for managing
 translations. This disconnect means that Openverse translations must convert
-from JSON to POT and back again. Hence there is quite a bit of scaffolding
+from JSON to POT and back again. Hence, there is quite a bit of scaffolding
 involved.
 
 ## Upload pipeline
@@ -25,7 +25,7 @@ frontend application and provided to GlotPress for translation.
 
 - Upload this
   [POT file](https://github.com/WordPress/openverse/blob/translations/openverse.pot)
-  to a fixed URL. Currently the file is hosted in the `translations` branch of
+  to a fixed URL. Currently, the file is hosted in the `translations` branch of
   the [WordPress/openverse](https://github.com/WordPress/openverse) repo.
 
 - GlotPress presents the strings, fuzzy translations and other helpful context
@@ -34,42 +34,39 @@ frontend application and provided to GlotPress for translation.
 ## Download pipeline
 
 This pipeline deals with how translations are retrieved from GlotPress,
-processed and loaded into Nuxt via the Nuxt i18n module.
+processed and loaded into Nuxt via the Nuxt i18n module. The main entry point,
+`i18n/scripts/setup.mjs`, orchestrates the entire process based on the command
+line arguments. There are three main modes of operation:
 
-### Steps
-
-- Parse and extract the list of all locales from GlotPress's PHP source code.
-  Then narrow down the list to locales available in the WP GlotPress instance
-  and populate their coverage percentage from the
-  [GlotPress stats](https://translate.wordpress.org/projects/meta/openverse/).
-
-  The output is written to `wp-locales.json`.
+- for production, it can download translations from GlotPress, parse the
+  locales, and update the Vue i18n plugin configuration.
+- for local development, it can convert `en.json5` to JSON format used by the
+  Nuxt app, and save the empty `valid-locales.json` file to be used by the Nuxt
+  app.
+- for testing, it can copy the test locale metadata, and test translations to
+  the main `i18n/locales` folder.
 
-  **Script:** `i18n:create-locales-list`
+### Production Steps
 
-- Download all translations from GlotPress as JED 1.x JSON files. The flattened
-  JED 1.x (derived from the flattened POT files) files are converted back into
-  the nested JSON as expected by Nuxt i18n.
+- Download all translations from GlotPress as NGX JSON files - flat json files.
 
   This script downloads all available translations in bulk as a ZIP file and
   then extracts JSON files from the ZIP file. This prevents excessive calls to
   GlotPress, which can be throttled and cause some locales to be missed.
 
-  **Script:** `i18n:get-translations`
+  **Script:** `i18n/scripts/translations.mjs`
 
-- Separate the locales into three groups based on the JSON files emitted by
-  `i18n:get-translations`.
+- Parse and extract the list of all locales from GlotPress's PHP source code.
+  Then narrow down the list to locales available in the WP GlotPress instance
+  and calculate their coverage percentage from the number of keys in the
+  translation and the number of keys in the main `en.json5` file. After that,
+  separate the locale metadata into two groups based on the JSON files emitted
+  by the previous step.
 
   - **translated:** JSON file is present with mappings, written to
     `valid-locales.json`.
-  - **untranslated:** JSON file is present but empty, written to both
-    `valid-locales.json` and `untranslated-locales.json`.
-  - **invalid:** JSON file is not present, written to `invalid-locales.json`.
-
-  **Script:** `i18n:update-locales`
-
-- Pass the list of valid locales (along with extra fields) into the Nuxt i18n
-  module. This is configured in the Nuxt configuration file, `nuxt.config.ts`.
+  - (only if --verbose flag is on) **untranslated:** JSON file is present but
+    empty, written to `untranslated-locales.json`.
 
 - Pass the fallback locale mappings to the underlying Vue i18n plugin. This is
   configured in the plugin configuration file,
@@ -78,7 +75,7 @@ processed and loaded into Nuxt via the Nuxt i18n module.
 ## Test locales
 
 Three locales are maintained in the code base (as opposed to downloaded from
-Glotpress) for use in testing, alongside the default English locale. These
+GlotPress) for use in testing, alongside the default English locale. These
 locales are **not** meant to be representative of actual final translations.
 They are merely used to approximate length and language features that help
 identify potential layout issues in the application.

documentation/general/index.md

@@ -10,5 +10,5 @@ test
 zero_downtime_database_management
 logging
 stack
-
+workspace
 ```

documentation/general/workspace.md

@@ -0,0 +1,67 @@
+# Using workspaces
+
+```{note}
+This is an opinionated guide that only applies to VS Code users. If you use a
+different editor or IDE, these instructions will not apply to you.
+```
+
+This is a guide to uses VS Code's multi-root workspace feature to work on
+multiple sub-stacks of Openverse.
+
+## Steps
+
+1. Clone the GitHub repository.
+
+   ```shell
+   git clone --filter=blob:none https://github.com/WordPress/openverse.git # or your fork
+   ```
+
+1. In the same directory as the repository, create a workspace file
+   `openverse.code-workspace` with the following configuration.
+
+   ```json
+   {
+     "folders": [
+       {
+         "name": "monorepo",
+         "path": "openverse"
+       },
+       {
+         "name": "catalog",
+         "path": "openverse/catalog"
+       },
+       {
+         "name": "indexer_worker",
+         "path": "openverse/indexer_worker"
+       },
+       {
+         "name": "ingestion_server",
+         "path": "openverse/ingestion_server"
+       },
+       {
+         "name": "api",
+         "path": "openverse/api"
+       },
+       {
+         "name": "frontend",
+         "path": "openverse/frontend"
+       },
+       {
+         "name": "documentation",
+         "path": "openverse/documentation"
+       },
+       {
+         "name": "python/openverse-attribution",
+         "path": "openverse/packages/python/openverse-attribution"
+       }
+     ],
+     "settings": {
+       "terminal.integrated.cwd": "${workspaceFolder:monorepo}"
+     }
+   }
+   ```
+
+1. From VS Code, open this workspace file. You should see the entire repository
+   open in VS Code with all sub-stacks depicted as top-level folders.
+
+   ![VS Code workspace](/_static/vs_code_workspace.png)

frontend/i18n/README.md

@@ -0,0 +1,10 @@
+# Locales
+
+The primary internationalisation file is [`data/en.json5`](../data/en.json5).
+All `.json` files present in the [`locales`](./locales) directory are
+re-generated when updating translations, so they should not be modified.
+
+# Locale scripts
+
+Locale scripts should be run in the root of the repository using their
+respective pnpm commands, in the `i18n` namespace.