* Swap out TDSR speech server with libspeak.

* Add a SpeechRate parameter in ModConfig.
master
erion 2023-02-05 12:15:55 +01:00
parent 44f31419b0
commit e5a421a4e3
2 changed files with 153 additions and 36 deletions

View File

@ -80,6 +80,7 @@ namespace stardew_access
public Boolean Warning { get; set; } = true; // Toggles the warnings feature public Boolean Warning { get; set; } = true; // Toggles the warnings feature
public Boolean TTS { get; set; } = true; // Toggles the screen reader/tts. public Boolean TTS { get; set; } = true; // Toggles the screen reader/tts.
public Boolean TrackDroppedItems {get; set;} = true; // Toggles detecting the dropped items. public Boolean TrackDroppedItems {get; set;} = true; // Toggles detecting the dropped items.
public Single SpeechRate {get; set;} = 220; // Sets speech rate for the Mac TTS.
#endregion #endregion
// TODO Add the exclusion and focus list too // TODO Add the exclusion and focus list too

View File

@ -1,12 +1,120 @@
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using static System.Net.Mime.MediaTypeNames;
namespace stardew_access.ScreenReader namespace stardew_access.ScreenReader
{ {
public class ScreenReaderMac : IScreenReader public class ScreenReaderMac : IScreenReader
{ {
private Process? _speakProcess; // The speaker instance
private static IntPtr speaker;
//Stuff for the runloop thread
private CancellationTokenSource cts = new CancellationTokenSource();
private Thread rt;
//Speech queue for interrupt
private static Queue<string> speechQueue = new Queue<string>();
// DidFinishSpeaking callback for interrupt
dfs_callback fscb = new dfs_callback(DoneSpeaking);
// Dylib imports
///////////
// Speaker
//
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void init_speaker();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void speak(string text);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_voice(Int32 index);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern UInt32 available_voices_count();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_language(Int32 index);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern UInt32 available_languages_count();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void get_voice_name(UInt32 idx, String pszOut);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_volume(Single volume);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Single get_volume();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_rate(Single rate);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Single get_rate();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void stop();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void cleanup_speaker();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void mainloop_speaker(IntPtr speaker);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Boolean is_speaking(IntPtr speaker);
///////////////
// Speaker OO
//
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr make_speaker();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void speak_with(IntPtr speaker, String text);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_voice_with(IntPtr speaker, Int32 index);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_volume_with(IntPtr speaker, Single volume);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Single get_volume_with(IntPtr speaker);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void set_rate_with(IntPtr speaker, Single rate);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Single get_rate_with(IntPtr speaker);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void stop_with(IntPtr speaker);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void cleanup_with(IntPtr speaker);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public delegate void wsw_callback(String p1);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public delegate void wsp_callback(Int16 p1);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public delegate void dfs_callback();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void register_will_speak_word_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]wsw_callback cb);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void register_will_speak_phoneme_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]wsp_callback cb);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void register_did_finish_speaking_callback(IntPtr speaker, [MarshalAs(UnmanagedType.FunctionPtr)]dfs_callback cb);
/////////////////
// Recognizer OO
//
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr make_listener();
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void start_listening(IntPtr listener);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void stop_listening(IntPtr listener);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void add_command(IntPtr listener, String command);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void cleanup_listener(IntPtr listener);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void mainloop_listener(IntPtr listener);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern Boolean is_listening(IntPtr listener);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public delegate void drc_callback(String p1);
[DllImport("libspeak", CallingConvention = CallingConvention.Cdecl)]
private static extern void register_did_recognize_command_callback(IntPtr listener, drc_callback cb);
public string PrevTextTile public string PrevTextTile
@ -17,39 +125,55 @@ namespace stardew_access.ScreenReader
public string prevText = "", prevTextTile = " ", prevChatText = "", prevMenuText = ""; public string prevText = "", prevTextTile = " ", prevChatText = "", prevMenuText = "";
private static void SpeakLoop(object obj)
{
CancellationToken ct = (CancellationToken)obj;
while (!ct.IsCancellationRequested)
{
mainloop_speaker(speaker);
Thread.Sleep(20);
}
}
public void InitializeScreenReader() public void InitializeScreenReader()
{ {
MainClass.InfoLog("Screen reader initialized"); MainClass.InfoLog("Screen reader initialized");
_speakProcess = new Process(); speaker = make_speaker();
_speakProcess.StartInfo.FileName = "mac"; rt = new Thread(new ParameterizedThreadStart(SpeakLoop));
_speakProcess.StartInfo.Verb = "mac"; rt.Start(cts.Token);
_speakProcess.StartInfo.RedirectStandardInput = true; register_did_finish_speaking_callback(speaker, fscb);
_speakProcess.Start(); set_rate_with(speaker, MainClass.Config.SpeechRate);
// set rate (probably should be done through a config parameter) Say("Mac screen reader ready", true);
_speakProcess.StandardInput.WriteLine("r600");
Speak("Mac screen reader ready", true);
} }
public void CloseScreenReader() public void CloseScreenReader()
{ {
MainClass.InfoLog("Screen reader closed"); cts.Cancel();
_speakProcess.Kill(); rt.Join();
cts.Dispose();
cleanup_with(speaker);
} }
public void Say(string text, bool interrupt) public void Say(string text, bool interrupt)
{ {
if (text == null) return; if (text == null) return;
MainClass.InfoLog($"{text}"); if (interrupt)
Speak(text, interrupt); {
speechQueue.Clear();
speak_with(speaker,text);
} else {
speechQueue.Enqueue(text);
}
} }
public void SayWithChecker(string text, bool interrupt) public void SayWithChecker(string text, bool interrupt)
{ {
if (text == null) return; if (text == null) return;
MainClass.InfoLog($"{text}");
if (text != prevText) if (text != prevText)
{ {
Speak(text, interrupt); MainClass.InfoLog($"{text}");
Say(text, interrupt);
prevText = text; prevText = text;
} }
} }
@ -57,10 +181,10 @@ _speakProcess.StandardInput.WriteLine("r600");
public void SayWithMenuChecker(string text, bool interrupt) public void SayWithMenuChecker(string text, bool interrupt)
{ {
if (text == null) return; if (text == null) return;
MainClass.InfoLog($"{text}");
if (text != prevMenuText) if (text != prevMenuText)
{ {
Speak(text, interrupt); MainClass.InfoLog($"{text}");
Say(text, interrupt);
prevMenuText = text; prevMenuText = text;
} }
} }
@ -68,10 +192,10 @@ _speakProcess.StandardInput.WriteLine("r600");
public void SayWithChatChecker(string text, bool interrupt) public void SayWithChatChecker(string text, bool interrupt)
{ {
if (text == null) return; if (text == null) return;
MainClass.InfoLog($"{text}");
if (text != prevChatText) if (text != prevChatText)
{ {
Speak(text, interrupt); MainClass.InfoLog($"{text}");
Say(text, interrupt);
prevChatText = text; prevChatText = text;
} }
} }
@ -79,29 +203,21 @@ _speakProcess.StandardInput.WriteLine("r600");
public void SayWithTileQuery(string text, int x, int y, bool interrupt) public void SayWithTileQuery(string text, int x, int y, bool interrupt)
{ {
if (text == null) return; if (text == null) return;
MainClass.InfoLog($"{text}");
if (text != prevTextTile) if (text != prevTextTile)
{ {
Speak(text, interrupt); MainClass.InfoLog($"{text}");
Say(text, interrupt);
prevTextTile = text; prevTextTile = text;
} }
} }
private void Speak(string text, bool interupt) private static void DoneSpeaking()
{ {
if (_speakProcess.HasExited) { if (speechQueue.Count != 0)
InitializeScreenReader();
}
if (interupt)
{ {
_speakProcess.StandardInput.WriteLine("x"); speak_with(speaker, speechQueue.Dequeue());
} }
var lines = text.Split("\n").ToArray();
foreach (var line in lines) {
_speakProcess.StandardInput.WriteLine($"s{line}");
} }
} }
} }
}