[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels

.changeset/rare-lamps-cough.md

@@ -0,0 +1,6 @@
+---
+"@khanacademy/perseus": patch
+"@khanacademy/perseus-editor": patch
+---
+
+[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels

packages/perseus-editor/package.json

@@ -40,6 +40,7 @@
         "@khanacademy/math-input": "^21.1.4",
         "@khanacademy/perseus": "^41.4.0",
         "@khanacademy/perseus-core": "1.5.3",
+        "@khanacademy/pure-markdown": "^0.3.11",
         "mafs": "^0.19.0"
     },
     "devDependencies": {

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-figure-aria.tsx

@@ -13,15 +13,23 @@ const {InfoTip} = components;
 
 type Props = {
     ariaLabel: string | undefined;
-    prePopulatedAriaLabel: string;
+    prePopulatedAriaLabel?: string;
+    getPrepopulatedAriaLabel?: () => Promise<string>;
     onChangeProps: (props: {ariaLabel?: string | undefined}) => void;
 };
 
 function LockedFigureAria(props: Props) {
-    const {ariaLabel, prePopulatedAriaLabel, onChangeProps} = props;
+    const {
+        ariaLabel,
+        prePopulatedAriaLabel,
+        getPrepopulatedAriaLabel,
+        onChangeProps,
+    } = props;
     const id = React.useId();
     const ariaLabelId = `aria-label-${id}`;
 
+    const [loading, setLoading] = React.useState(false);
+
     return (
         <View>
             <Strut size={spacing.xSmall_8} />
@@ -52,7 +60,7 @@ function LockedFigureAria(props: Props) {
             <Strut size={spacing.xxSmall_6} />
             <TextArea
                 id={ariaLabelId}
-                value={ariaLabel ?? ""}
+                value={loading ? "Loading..." : ariaLabel ?? ""}
                 onChange={(newValue) => {
                     onChangeProps({
                         // Save as undefined if the field is empty.
@@ -69,9 +77,18 @@ function LockedFigureAria(props: Props) {
                 startIcon={pencilCircle}
                 style={styles.button}
                 onClick={() => {
-                    onChangeProps({
-                        ariaLabel: prePopulatedAriaLabel,
-                    });
+                    // TODO(LEMS-2548): remove the prePopulatedAriaLabel prop
+                    // after all the locked figures are updated to use
+                    // getPrepopulatedAriaLabel.
+                    if (prePopulatedAriaLabel) {
+                        onChangeProps({ariaLabel: prePopulatedAriaLabel});
+                    } else if (getPrepopulatedAriaLabel) {
+                        setLoading(true);
+                        getPrepopulatedAriaLabel().then((ariaLabel) => {
+                            setLoading(false);
+                            onChangeProps({ariaLabel});
+                        });
+                    }
                 }}
             >
                 Auto-generate

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx

@@ -26,6 +26,14 @@ const defaultProps = {
 
 const defaultLabel = getDefaultFigureForType("label");
 
+// Mock the async function generateSpokenMathDetails
+jest.mock("./util", () => ({
+    ...jest.requireActual("./util"),
+    generateSpokenMathDetails: (input) => {
+        return Promise.resolve(`Spoken math details for ${input}`);
+    },
+}));
+
 describe("LockedPointSettings", () => {
     let userEvent: UserEvent;
     beforeEach(() => {
@@ -409,8 +417,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -439,8 +450,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -473,8 +487,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A, B at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A, B at (0, 0). Appearance solid gray.",
         });
     });
 });

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx

@@ -24,6 +24,7 @@ import LockedFigureSettingsActions from "./locked-figure-settings-actions";
 import LockedLabelSettings from "./locked-label-settings";
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -105,20 +106,24 @@ const LockedPointSettings = (props: Props) => {
     const isDefiningPoint = !onMove && !onRemove;
 
     /**
-     * Get a prepopulated aria label for the point.
+     * Get a prepopulated aria label for the point, with the math
+     * details converted into spoken words.
      *
      * If the point has no labels, the aria label will just be
      * "Point at (x, y)".
      *
      * If the point has labels, the aria label will be
-     * "Point at (x, y) with label1, label2, label3".
+     * "Point label1, label2, label3 at (x, y)".
      */
-    function getPrepopulatedAriaLabel() {
+    async function getPrepopulatedAriaLabel() {
         let visiblelabel = "";
         if (labels && labels.length > 0) {
             visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`;
         }
-        let str = `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`;
+
+        let str = await generateSpokenMathDetails(
+            `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`,
+        );
 
         const pointAppearance =
             generateLockedFigureAppearanceDescription(pointColor);
@@ -247,7 +252,7 @@ const LockedPointSettings = (props: Props) => {
 
                     <LockedFigureAria
                         ariaLabel={ariaLabel}
-                        prePopulatedAriaLabel={getPrepopulatedAriaLabel()}
+                        getPrepopulatedAriaLabel={getPrepopulatedAriaLabel}
                         onChangeProps={(newProps) => {
                             onChangeProps(newProps);
                         }}

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts

@@ -1,5 +1,6 @@
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -258,3 +259,84 @@ describe("generateLockedFigureAppearanceDescription", () => {
         },
     );
 });
+
+// TODO(LEMS-2616): Update these tests to mock SpeechRuleEngine.setup()
+// so that the tests don't have to make HTTP requests.
+describe("generateMathDetails", () => {
+    test("should convert TeX to spoken language (root, fraction)", async () => {
+        const mathString = "$\\sqrt{\\frac{1}{2}}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("StartRoot one half EndRoot");
+    });
+
+    test("should convert TeX to spoken language (exponent)", async () => {
+        const mathString = "$x^{2}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("x Superscript 2");
+    });
+
+    test("should convert TeX to spoken language (negative)", async () => {
+        const mathString = "$-2$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("negative 2");
+    });
+
+    test("should converte TeX to spoken language (subtraction)", async () => {
+        const mathString = "$2-1$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("2 minus 1");
+    });
+
+    test("should convert TeX to spoken language (normal words)", async () => {
+        const mathString = "$\\text{square b}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("square b");
+    });
+
+    test("should convert TeX to spoken language (random letters)", async () => {
+        const mathString = "$cat$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("c a t");
+    });
+
+    test("should keep non-math text as is", async () => {
+        const mathString = "Circle with radius $\\frac{1}{2}$ units";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Circle with radius one half units");
+    });
+
+    test("should read dollar signs as dollars inside tex", async () => {
+        const mathString = "This sandwich costs ${$}12.34$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs dollar sign 12.34");
+    });
+
+    test("should read dollar signs as dollars outside tex", async () => {
+        const mathString = "This sandwich costs \\$12.34";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs $12.34");
+    });
+
+    test("should read curly braces", async () => {
+        const mathString = "Hello}{";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Hello}{");
+    });
+
+    test("should read backslashes", async () => {
+        const mathString = "\\";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("\\");
+    });
+});

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts

@@ -1,3 +1,5 @@
+import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer";
+import * as SimpleMarkdown from "@khanacademy/pure-markdown";
 import {UnreachableCaseError} from "@khanacademy/wonder-stuff-core";
 
 import type {
@@ -124,3 +126,29 @@ export function generateLockedFigureAppearanceDescription(
             throw new UnreachableCaseError(fill);
     }
 }
+
+export async function generateSpokenMathDetails(mathString: string) {
+    const engine = await SpeechRuleEngine.setup("en");
+    let convertedSpeech = "";
+
+    // All the information we need is in the first section,
+    // whether it's typed as "blockmath" or "paragraph"
+    const firstSection = SimpleMarkdown.parse(mathString)[0];
+
+    // If it's blockMath, the outer level has the full math content.
+    if (firstSection.type === "blockMath") {
+        convertedSpeech += engine.texToSpeech(firstSection.content);
+    }
+
+    // If it's a paragraph, we need to iterate through the sections
+    // to look for individual math blocks.
+    if (firstSection.type === "paragraph") {
+        for (const piece of firstSection.content) {
+            piece.type === "math"
+                ? (convertedSpeech += engine.texToSpeech(piece.content))
+                : (convertedSpeech += piece.content);
+        }
+    }
+
+    return convertedSpeech;
+}