[Locked Figure Labels] Util function to generate spoken math + use it…

… within Locked Point aria labels (#1839)

## Summary:
When auto-generating the aria labels for locked figures, we want it to use
words as if they were spoken rather than math expressions that might be
read incorrectly by the screen reader.

- Create a utility function using the MathJax speech engine that converts
  math details (TeX denoted by the `$...$` blocks) to spoken words.
- Use this utility within LockedPointSettings. (Async + useEffect because
  the speech engine is async. I had to use mocks in the tests for this.)

If this approach is given the okay, I'll go ahead and apply it to the
other locked figures as well in a following PR.

Issue: https://khanacademy.atlassian.net/browse/LEMS-2548

## Test plan:
`yarn jest packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts`
`yarn jest packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx`

Storybook
- `http://localhost:6006/iframe.html?args=&id=perseuseditor-widgets-interactive-graph--mafs-with-locked-figure-labels-all-flags&viewMode=story`
- Open the locked point settings
- Change the visible label to have a mix of TeX (with `$...$`) and non-TeX
- Press the "Auto-generate" button
- Confirm that the input changes to include spoken math words for the Tex
- Also try this with no labels and with multiple labels

<img width="387" alt="image" src="https://github.com/user-attachments/assets/f5bd3d47-9e93-47ee-8af9-ec217a6cbbae">

### Loading state demo

https://github.com/user-attachments/assets/1ee80dee-3b10-46ab-b3f5-d296612b7a07

Author: nishasy

Reviewers: benchristel, nishasy, anakaren-rojas, catandthemachines

Required Reviewers:

Approved By: anakaren-rojas, benchristel

Checks: ✅ Publish npm snapshot (ubuntu-latest, 20.x), ✅ Lint, Typecheck, Format, and Test (ubuntu-latest, 20.x), ✅ Cypress (ubuntu-latest, 20.x), ✅ Check builds for changes in size (ubuntu-latest, 20.x), ✅ Check for .changeset entries for all changed files (ubuntu-latest, 20.x), ✅ Publish Storybook to Chromatic (ubuntu-latest, 20.x), ✅ gerald

Pull Request URL: #1839

.changeset/rare-lamps-cough.md

@@ -0,0 +1,6 @@
+---
+"@khanacademy/perseus": patch
+"@khanacademy/perseus-editor": patch
+---
+
+[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels

packages/perseus-editor/package.json

@@ -40,6 +40,7 @@
         "@khanacademy/math-input": "^21.1.4",
         "@khanacademy/perseus": "^41.4.0",
         "@khanacademy/perseus-core": "1.5.3",
+        "@khanacademy/pure-markdown": "^0.3.11",
         "mafs": "^0.19.0"
     },
     "devDependencies": {

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-figure-aria.tsx

@@ -13,15 +13,23 @@ const {InfoTip} = components;
 
 type Props = {
     ariaLabel: string | undefined;
-    prePopulatedAriaLabel: string;
+    prePopulatedAriaLabel?: string;
+    getPrepopulatedAriaLabel?: () => Promise<string>;
     onChangeProps: (props: {ariaLabel?: string | undefined}) => void;
 };
 
 function LockedFigureAria(props: Props) {
-    const {ariaLabel, prePopulatedAriaLabel, onChangeProps} = props;
+    const {
+        ariaLabel,
+        prePopulatedAriaLabel,
+        getPrepopulatedAriaLabel,
+        onChangeProps,
+    } = props;
     const id = React.useId();
     const ariaLabelId = `aria-label-${id}`;
 
+    const [loading, setLoading] = React.useState(false);
+
     return (
         <View>
             <Strut size={spacing.xSmall_8} />
@@ -52,7 +60,7 @@ function LockedFigureAria(props: Props) {
             <Strut size={spacing.xxSmall_6} />
             <TextArea
                 id={ariaLabelId}
-                value={ariaLabel ?? ""}
+                value={loading ? "Loading..." : ariaLabel ?? ""}
                 onChange={(newValue) => {
                     onChangeProps({
                         // Save as undefined if the field is empty.
@@ -69,9 +77,18 @@ function LockedFigureAria(props: Props) {
                 startIcon={pencilCircle}
                 style={styles.button}
                 onClick={() => {
-                    onChangeProps({
-                        ariaLabel: prePopulatedAriaLabel,
-                    });
+                    // TODO(LEMS-2548): remove the prePopulatedAriaLabel prop
+                    // after all the locked figures are updated to use
+                    // getPrepopulatedAriaLabel.
+                    if (prePopulatedAriaLabel) {
+                        onChangeProps({ariaLabel: prePopulatedAriaLabel});
+                    } else if (getPrepopulatedAriaLabel) {
+                        setLoading(true);
+                        getPrepopulatedAriaLabel().then((ariaLabel) => {
+                            setLoading(false);
+                            onChangeProps({ariaLabel});
+                        });
+                    }
                 }}
             >
                 Auto-generate

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx

@@ -26,6 +26,14 @@ const defaultProps = {
 
 const defaultLabel = getDefaultFigureForType("label");
 
+// Mock the async function generateSpokenMathDetails
+jest.mock("./util", () => ({
+    ...jest.requireActual("./util"),
+    generateSpokenMathDetails: (input) => {
+        return Promise.resolve(`Spoken math details for ${input}`);
+    },
+}));
+
 describe("LockedPointSettings", () => {
     let userEvent: UserEvent;
     beforeEach(() => {
@@ -409,8 +417,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -439,8 +450,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -473,8 +487,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A, B at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A, B at (0, 0). Appearance solid gray.",
         });
     });
 });

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx

@@ -24,6 +24,7 @@ import LockedFigureSettingsActions from "./locked-figure-settings-actions";
 import LockedLabelSettings from "./locked-label-settings";
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -105,20 +106,24 @@ const LockedPointSettings = (props: Props) => {
     const isDefiningPoint = !onMove && !onRemove;
 
     /**
-     * Get a prepopulated aria label for the point.
+     * Get a prepopulated aria label for the point, with the math
+     * details converted into spoken words.
      *
      * If the point has no labels, the aria label will just be
      * "Point at (x, y)".
      *
      * If the point has labels, the aria label will be
-     * "Point at (x, y) with label1, label2, label3".
+     * "Point label1, label2, label3 at (x, y)".
      */
-    function getPrepopulatedAriaLabel() {
+    async function getPrepopulatedAriaLabel() {
         let visiblelabel = "";
         if (labels && labels.length > 0) {
             visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`;
         }
-        let str = `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`;
+
+        let str = await generateSpokenMathDetails(
+            `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`,
+        );
 
         const pointAppearance =
             generateLockedFigureAppearanceDescription(pointColor);
@@ -247,7 +252,7 @@ const LockedPointSettings = (props: Props) => {
 
                     <LockedFigureAria
                         ariaLabel={ariaLabel}
-                        prePopulatedAriaLabel={getPrepopulatedAriaLabel()}
+                        getPrepopulatedAriaLabel={getPrepopulatedAriaLabel}
                         onChangeProps={(newProps) => {
                             onChangeProps(newProps);
                         }}

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts

@@ -1,5 +1,6 @@
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -258,3 +259,84 @@ describe("generateLockedFigureAppearanceDescription", () => {
         },
     );
 });
+
+// TODO(LEMS-2616): Update these tests to mock SpeechRuleEngine.setup()
+// so that the tests don't have to make HTTP requests.
+describe("generateMathDetails", () => {
+    test("should convert TeX to spoken language (root, fraction)", async () => {
+        const mathString = "$\\sqrt{\\frac{1}{2}}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("StartRoot one half EndRoot");
+    });
+
+    test("should convert TeX to spoken language (exponent)", async () => {
+        const mathString = "$x^{2}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("x Superscript 2");
+    });
+
+    test("should convert TeX to spoken language (negative)", async () => {
+        const mathString = "$-2$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("negative 2");
+    });
+
+    test("should converte TeX to spoken language (subtraction)", async () => {
+        const mathString = "$2-1$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("2 minus 1");
+    });
+
+    test("should convert TeX to spoken language (normal words)", async () => {
+        const mathString = "$\\text{square b}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("square b");
+    });
+
+    test("should convert TeX to spoken language (random letters)", async () => {
+        const mathString = "$cat$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("c a t");
+    });
+
+    test("should keep non-math text as is", async () => {
+        const mathString = "Circle with radius $\\frac{1}{2}$ units";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Circle with radius one half units");
+    });
+
+    test("should read dollar signs as dollars inside tex", async () => {
+        const mathString = "This sandwich costs ${$}12.34$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs dollar sign 12.34");
+    });
+
+    test("should read dollar signs as dollars outside tex", async () => {
+        const mathString = "This sandwich costs \\$12.34";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs $12.34");
+    });
+
+    test("should read curly braces", async () => {
+        const mathString = "Hello}{";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Hello}{");
+    });
+
+    test("should read backslashes", async () => {
+        const mathString = "\\";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("\\");
+    });
+});

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts

@@ -1,3 +1,5 @@
+import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer";
+import * as SimpleMarkdown from "@khanacademy/pure-markdown";
 import {UnreachableCaseError} from "@khanacademy/wonder-stuff-core";
 
 import type {
@@ -124,3 +126,29 @@ export function generateLockedFigureAppearanceDescription(
             throw new UnreachableCaseError(fill);
     }
 }
+
+export async function generateSpokenMathDetails(mathString: string) {
+    const engine = await SpeechRuleEngine.setup("en");
+    let convertedSpeech = "";
+
+    // All the information we need is in the first section,
+    // whether it's typed as "blockmath" or "paragraph"
+    const firstSection = SimpleMarkdown.parse(mathString)[0];
+
+    // If it's blockMath, the outer level has the full math content.
+    if (firstSection.type === "blockMath") {
+        convertedSpeech += engine.texToSpeech(firstSection.content);
+    }
+
+    // If it's a paragraph, we need to iterate through the sections
+    // to look for individual math blocks.
+    if (firstSection.type === "paragraph") {
+        for (const piece of firstSection.content) {
+            piece.type === "math"
+                ? (convertedSpeech += engine.texToSpeech(piece.content))
+                : (convertedSpeech += piece.content);
+        }
+    }
+
+    return convertedSpeech;
+}