From e5a421a4e37b54f6876c94f98b2ac45c56e70925 Mon Sep 17 00:00:00 2001 From: erion Date: Sun, 5 Feb 2023 12:15:55 +0100 Subject: [PATCH] * Swap out TDSR speech server with libspeak. * Add a SpeechRate parameter in ModConfig. --- stardew-access/ModConfig.cs | 1 + .../ScreenReader/ScreenReaderMac.cs | 188 ++++++++++++++---- 2 files changed, 153 insertions(+), 36 deletions(-) diff --git a/stardew-access/ModConfig.cs b/stardew-access/ModConfig.cs index bce66e6..b35437d 100644 --- a/stardew-access/ModConfig.cs +++ b/stardew-access/ModConfig.cs @@ -80,6 +80,7 @@ namespace stardew_access public Boolean Warning { get; set; } = true; // Toggles the warnings feature public Boolean TTS { get; set; } = true; // Toggles the screen reader/tts. public Boolean TrackDroppedItems {get; set;} = true; // Toggles detecting the dropped items. + public Single SpeechRate {get; set;} = 220; // Sets speech rate for the Mac TTS. #endregion // TODO Add the exclusion and focus list too diff --git a/stardew-access/ScreenReader/ScreenReaderMac.cs b/stardew-access/ScreenReader/ScreenReaderMac.cs index 5b20745..2393736 100644 --- a/stardew-access/ScreenReader/ScreenReaderMac.cs +++ b/stardew-access/ScreenReader/ScreenReaderMac.cs @@ -1,12 +1,120 @@ using System; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using static System.Net.Mime.MediaTypeNames; namespace stardew_access.ScreenReader { public class ScreenReaderMac : IScreenReader { - private Process? _speakProcess; + // The speaker instance + private static IntPtr speaker; + //Stuff for the runloop thread + private CancellationTokenSource cts = new CancellationTokenSource(); + private Thread rt; + //Speech queue for interrupt + private static Queue speechQueue = new Queue(); + // DidFinishSpeaking callback for interrupt + dfs_callback fscb = new dfs_callback(DoneSpeaking); + + // Dylib imports + /////////// + // Speaker + // + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void init_speaker(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void speak(string text); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_voice(Int32 index); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern UInt32 available_voices_count(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_language(Int32 index); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern UInt32 available_languages_count(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void get_voice_name(UInt32 idx, String pszOut); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_volume(Single volume); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Single get_volume(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_rate(Single rate); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Single get_rate(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void stop(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void cleanup_speaker(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void mainloop_speaker(IntPtr speaker); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Boolean is_speaking(IntPtr speaker); + + /////////////// + // Speaker OO + // + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr make_speaker(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void speak_with(IntPtr speaker, String text); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_voice_with(IntPtr speaker, Int32 index); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_volume_with(IntPtr speaker, Single volume); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Single get_volume_with(IntPtr speaker); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void set_rate_with(IntPtr speaker, Single rate); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Single get_rate_with(IntPtr speaker); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void stop_with(IntPtr speaker); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void cleanup_with(IntPtr speaker); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate void wsw_callback(String p1); + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate void wsp_callback(Int16 p1); + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate void dfs_callback(); + + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void register_will_speak_word_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]wsw_callback cb); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void register_will_speak_phoneme_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]wsp_callback cb); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void register_did_finish_speaking_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]dfs_callback cb); + + ///////////////// + // Recognizer OO + // + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr make_listener(); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void start_listening(IntPtr listener); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void stop_listening(IntPtr listener); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void add_command(IntPtr listener, String command); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void cleanup_listener(IntPtr listener); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void mainloop_listener(IntPtr listener); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern Boolean is_listening(IntPtr listener); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate void drc_callback(String p1); + [DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)] + private static extern void register_did_recognize_command_callback(IntPtr listener, drc_callback cb); + + + public string PrevTextTile @@ -17,39 +125,55 @@ namespace stardew_access.ScreenReader public string prevText = "", prevTextTile = " ", prevChatText = "", prevMenuText = ""; - public void InitializeScreenReader() + private static void SpeakLoop(object obj) + { + CancellationToken ct = (CancellationToken)obj; + + while (!ct.IsCancellationRequested) + { + mainloop_speaker(speaker); + Thread.Sleep(20); + } + } + + public void InitializeScreenReader() { MainClass.InfoLog("Screen reader initialized"); - _speakProcess = new Process(); - _speakProcess.StartInfo.FileName = "mac"; - _speakProcess.StartInfo.Verb = "mac"; - _speakProcess.StartInfo.RedirectStandardInput = true; - _speakProcess.Start(); - // set rate (probably should be done through a config parameter) -_speakProcess.StandardInput.WriteLine("r600"); - Speak("Mac screen reader ready", true); - } + speaker = make_speaker(); + rt = new Thread(new ParameterizedThreadStart(SpeakLoop)); + rt.Start(cts.Token); + register_did_finish_speaking_callback(speaker, fscb); + set_rate_with(speaker, MainClass.Config.SpeechRate); + Say("Mac screen reader ready", true); + } public void CloseScreenReader() { - MainClass.InfoLog("Screen reader closed"); - _speakProcess.Kill(); + cts.Cancel(); + rt.Join(); + cts.Dispose(); + cleanup_with(speaker); } public void Say(string text, bool interrupt) { if (text == null) return; - MainClass.InfoLog($"{text}"); - Speak(text, interrupt); + if (interrupt) + { + speechQueue.Clear(); + speak_with(speaker,text); + } else { + speechQueue.Enqueue(text); + } } public void SayWithChecker(string text, bool interrupt) { if (text == null) return; - MainClass.InfoLog($"{text}"); if (text != prevText) { - Speak(text, interrupt); + MainClass.InfoLog($"{text}"); + Say(text, interrupt); prevText = text; } } @@ -57,10 +181,10 @@ _speakProcess.StandardInput.WriteLine("r600"); public void SayWithMenuChecker(string text, bool interrupt) { if (text == null) return; - MainClass.InfoLog($"{text}"); if (text != prevMenuText) { - Speak(text, interrupt); + MainClass.InfoLog($"{text}"); + Say(text, interrupt); prevMenuText = text; } } @@ -68,10 +192,10 @@ _speakProcess.StandardInput.WriteLine("r600"); public void SayWithChatChecker(string text, bool interrupt) { if (text == null) return; - MainClass.InfoLog($"{text}"); if (text != prevChatText) { - Speak(text, interrupt); + MainClass.InfoLog($"{text}"); + Say(text, interrupt); prevChatText = text; } } @@ -79,29 +203,21 @@ _speakProcess.StandardInput.WriteLine("r600"); public void SayWithTileQuery(string text, int x, int y, bool interrupt) { if (text == null) return; - MainClass.InfoLog($"{text}"); if (text != prevTextTile) { - Speak(text, interrupt); + MainClass.InfoLog($"{text}"); + Say(text, interrupt); prevTextTile = text; +} } - } - - private void Speak(string text, bool interupt) + private static void DoneSpeaking() { - if (_speakProcess.HasExited) { - InitializeScreenReader(); - } - if (interupt) + if (speechQueue.Count != 0) { - _speakProcess.StandardInput.WriteLine("x"); - } - var lines = text.Split("\n").ToArray(); - foreach (var line in lines) { - _speakProcess.StandardInput.WriteLine($"s{line}"); + speak_with(speaker, speechQueue.Dequeue()); } + } - } } } \ No newline at end of file