Add OpenAI TTS model as Auto option

components/Chat/SpeechButton.tsx

@@ -45,20 +45,52 @@ export const SpeechButton: React.FC<Props> = ({ inputText }) => {
     return currentSpeechId === componentSpeechId;
   }, [componentSpeechId, currentSpeechId]);
 
-  const removeEmojis = (text: string): string => {
+  const removeEmojisOnly = (text: string): string => {
+    // Remove emojis and other unicode symbols, but keep basic punctuation
     return text.replace(
-      /(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])/g,
+      /(?:[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff]|[\u0023-\u0039]\ufe0f?\u20e3|\u3299|\u3297|\u303d|\u3030|\u24c2|\ud83c[\udd70-\udd71]|\ud83c[\udd7e-\udd7f]|\ud83c\udd8e|\ud83c[\udd91-\udd9a]|\ud83c[\udde6-\uddff]|\ud83c[\ude01-\ude02]|\ud83c\ude1a|\ud83c\ude2f|\ud83c[\ude32-\ude3a]|\ud83c[\ude50-\ude51]|\u203c|\u2049|[\u25aa-\u25ab]|\u25b6|\u25c0|[\u25fb-\u25fe]|\u00a9|\u00ae|\u2122|\u2139|\ud83c\udc04|[\u2600-\u26FF]|\u2b05|\u2b06|\u2b07|\u2b1b|\u2b1c|\u2b50|\u2b55|\u231a|\u231b|\u2328|\u23cf|[\u23e9-\u23f3]|[\u23f8-\u23fa]|\ud83c\udccf|\u2934|\u2935|[\u2190-\u21ff])/g,
       '',
     );
   };
 
+  const normalizePunctuations = (text: string): string => {
+    return text.replace(/\p{P}/gu, (match) => {
+      // Full stop (period) equivalents
+      if (/[\u3002\uFF0E\uFF61]/u.test(match)) {
+        return '. ';
+      }
+      // Comma equivalents
+      if (/[\u060C\u3001\uFF0C\u2E41\u2E34\u2E32]/u.test(match)) {
+        return ', ';
+      }
+      // Question mark equivalents
+      if (/[\uFF1F\u037E\u061F]/u.test(match)) {
+        return '? ';
+      }
+      // Exclamation mark equivalents
+      if (/[\uFF01\u01C3\u2D51\u05C6\u061B]/u.test(match)) {
+        return '! ';
+      }
+      // For other punctuation, replace with a space
+      return ' ';
+    });
+  };
+
   const playStopOnClick = async () => {
     if (loadingTts) return;
     if (playingSpeech && isComponentCurrentlyBeingPlayed) {
       closePlayer();
       closeSpeechSynthesizer();
     } else {
-      const sanitizedInputText = removeEmojis(inputText);
+      let sanitizedInputText;
+      try {
+        sanitizedInputText = normalizePunctuations(
+          removeEmojisOnly(inputText),
+        ).trim();
+      } catch (error) {
+        sanitizedInputText = inputText;
+      }
+
       await playMessage(sanitizedInputText, componentSpeechId);
       logGeneralEvent('speech');
       trackEvent('AI speech play button clicked');

components/EnhancedMenu/ChangeOutputLanguageButton.tsx

@@ -32,7 +32,7 @@ function ChangeOutputLanguageButton() {
       <div className="w-fit rounded-lg border border-neutral-200 bg-transparent pr-1 text-neutral-900 focus:outline-none dark:border-neutral-600 dark:text-white">
         <select
           className="bg-transparent p-2 focus:outline-none"
-          placeholder={t('Select a lang') || ''}
+          placeholder={t('Select a lang') || 'auto'}
           value={outputLanguage}
           onChange={(e) => {
             homeDispatch({ field: 'outputLanguage', value: e.target.value });

components/Hooks/useConversation.ts

@@ -14,9 +14,12 @@ export const useConversation = () => {
   const segmenter = useRef<Intl.Segmenter>();
 
   useEffect(() => {
-    segmenter.current = new Intl.Segmenter(outputLanguage || 'en', {
-      granularity: 'sentence',
-    });
+    segmenter.current = new Intl.Segmenter(
+      outputLanguage === 'auto' ? 'en' : outputLanguage,
+      {
+        granularity: 'sentence',
+      },
+    );
   }, [outputLanguage]);
 
   useEffect(() => {

components/User/Settings/Settings_App.tsx

@@ -51,28 +51,35 @@ export default function Settings_App() {
               {t('Speech configuration')}
             </span>
           </div>
-          <div className="mt-0 flex items-center bg-transparent md:space-x-4">
-            <div className="w-1/2">
-              <label
-                htmlFor="speechSpeed"
-                className="block text-sm font-medium"
-              >
-                {t('Speech Speed')}
-              </label>
-            </div>
-            <div className="w-1/2">
-              <select
-                id="speechSpeed"
-                name="speechSpeed"
-                value={speechSpeedConfig}
-                onChange={(e) => speechSpeedOnChange(e.target.value)}
-                className="mt-1 block w-full rounded-md bg-[#171717] py-2 pl-3 pr-10 text-end text-sm focus:outline-none"
-              >
-                <option value={'slow'}>{t('Slow')}</option>
-                <option value={'normal'}>{t('Normal')}</option>
-                <option value={'fast'}>{t('Fast')}</option>
-              </select>
+          <div className="mt-0 flex flex-col items-center bg-transparent md:space-x-4">
+            <div className="flex w-full flex-row">
+              <div className="w-1/2">
+                <label
+                  htmlFor="speechSpeed"
+                  className="block text-sm font-medium"
+                >
+                  {t('Speech Speed')}
+                </label>
+              </div>
+              <div className="w-1/2">
+                <select
+                  id="speechSpeed"
+                  name="speechSpeed"
+                  value={speechSpeedConfig}
+                  onChange={(e) => speechSpeedOnChange(e.target.value)}
+                  className="mt-1 block w-full rounded-md bg-[#171717] py-2 pl-3 pr-10 text-end text-sm focus:outline-none"
+                >
+                  <option value={'slow'}>{t('Slow')}</option>
+                  <option value={'normal'}>{t('Normal')}</option>
+                  <option value={'fast'}>{t('Fast')}</option>
+                </select>
+              </div>
             </div>
+            <p className="!ml-0 mt-2 w-full text-left text-xs text-gray-500">
+              {t(
+                "The 'Auto' option in Speech language will not be affected by the Speech speed setting.",
+              )}
+            </p>
           </div>
         </>
       )}

components/home/home.state.tsx

@@ -100,7 +100,7 @@ export const initialState: HomeInitialState = {
   messageError: false,
   searchTerm: '',
   defaultModelId: undefined,
-  outputLanguage: '',
+  outputLanguage: 'auto',
   currentDrag: undefined,
 
   // Supabase / Cloud Sync

public/locales/zh-Hans/model.json

@@ -138,6 +138,7 @@
   "You can send message to the topic specified (with retained flag on), Chat Everywhere will be able to access the latest message.": "設定MQTT裝置來發送訊息到你指定的Topic (請確認訊息中有retain的標籤)。",
   "Speech configuration": "AI朗读设置",
   "Speech Speed": "语速",
+  "The 'Auto' option in Speech language will not be affected by the Speech speed setting.": "语速设置只适用于非'Auto'的语音语言",
   "Slow": "慢",
   "Normal": "正常",
   "Fast": "快",

public/locales/zh-Hant/model.json

@@ -136,6 +136,7 @@
   "You can send message to the topic specified (with retained flag on), Chat Everywhere will be able to access the latest message.": "設定MQTT裝置來發送訊息到你指定的Topic (請確認訊息中有retain的標籤)。",
   "Speech configuration": "AI朗讀設定",
   "Speech Speed": "語速",
+  "The 'Auto' option in Speech language will not be affected by the Speech speed setting.": "語速設定只適用於非'Auto'的語音語言",
   "Slow": "慢",
   "Normal": "正常",
   "Fast": "快",

utils/app/eventTracking.ts

@@ -162,7 +162,6 @@ export type PayloadType = {
   mjQueueCleanupJobOneWeekAgo?: string;
   mjQueueCleanupJobFiveMinutesAgo?: string;
   mjQueueJobDetail?: MjJob;
-
 };
 
 export interface UserPostHogProfile {

utils/app/i18n.ts

@@ -27,9 +27,9 @@ function getAvailableLocales() {
 // https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/language-support?tabs=stt#supported-languages
 function getAvailableSpeechSourceLanguages() {
   return [
+    { name: 'Auto', value: 'auto' },
     { name: 'English', value: 'en-US' },
-    { name: '國語(簡体)', value: 'zh-CN' },
-    { name: '國語(繁體)', value: 'zh-TW' },
+    { name: '國語', value: 'zh-TW' },
     { name: '廣東話', value: 'zh-HK' },
     { name: 'বাংলা', value: 'bn-IN' },
     { name: 'Deutsch', value: 'de-DE' },
@@ -53,8 +53,8 @@ function getAvailableSpeechSourceLanguages() {
 }
 
 const voiceMap: { [language: string]: string } = {
+  auto: 'en-US-NovaMultilingualNeural',
   'en-US': 'en-US-JennyNeural',
-  'zh-CN': 'zh-CN-XiaoxiaoNeural',
   'zh-TW': 'zh-TW-HsiaoChenNeural',
   'zh-HK': 'zh-HK-HiuMaanNeural',
   'bn-IN': 'bn-IN-TanishaaNeural',

utils/server/mjServiceServerHelper.ts

@@ -35,7 +35,7 @@ export const trackFailedEvent = (jobInfo: MjJob, errorMessage: string) => {
       mjImageGenTotalProcessingTimeInSeconds: totalProcessingTimeInSeconds,
       mjImageGenErrorMessage: errorMessage,
       usedOnDemandCredit: jobInfo.usedOnDemandCredit || false,
-      lastUsedKey: jobInfo.lastUsedKey
+      lastUsedKey: jobInfo.lastUsedKey,
     },
   );
   return trackEventPromise;
@@ -67,7 +67,7 @@ export const trackSuccessEvent = (jobInfo: MjJob) => {
         totalWaitingInQueueTimeInSeconds,
       mjImageGenTotalProcessingTimeInSeconds: totalProcessingTimeInSeconds,
       usedOnDemandCredit: jobInfo.usedOnDemandCredit || false,
-      lastUsedKey: jobInfo.lastUsedKey
+      lastUsedKey: jobInfo.lastUsedKey,
     },
   );
 
@@ -82,8 +82,8 @@ export const trackCleanupJobEvent = ({
   fiveMinutesAgo,
 }: {
   event:
-  | 'MJ Queue Cleanup Completed / Failed Job'
-  | 'MJ Queue Cleanup Processing Job';
+    | 'MJ Queue Cleanup Completed / Failed Job'
+    | 'MJ Queue Cleanup Processing Job';
   executedAt: string;
   enqueuedAt: string;
   oneWeekAgo?: string;
@@ -120,8 +120,9 @@ export const OriginalMjLogEvent = async ({
   if (errorMessage) {
     payloadToLog.imageGenerationFailed = 'true';
     payloadToLog.imageGenerationErrorMessage = errorMessage;
-    payloadToLog.imageGenerationPrompt = `${promptBeforeProcessing || 'N/A'
-      } -> ${generationPrompt || 'N/A'}`;
+    payloadToLog.imageGenerationPrompt = `${
+      promptBeforeProcessing || 'N/A'
+    } -> ${generationPrompt || 'N/A'}`;
   }
 
   await serverSideTrackEvent(userId, 'AI image generation', payloadToLog);