tom/hermes-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Using IHostingService for TTS listening and playing in background.

Browse Source
This commit is contained in:
Tom 2024-08-12 16:42:53 +00:00
parent 2643feeca7
commit f0071cae81
10 changed files with 208 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Chat/Commands/SkipCommand.cs
View File

@ -1,5 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Hermes.Socket;
using TwitchChatTTS.Twitch.Socket.Messages;
using static TwitchChatTTS.Chat.Commands.TTSCommands;

2
Chat/Messaging/ChatMessageReader.cs
View File

@ -7,7 +7,7 @@ using TwitchChatTTS.Chat.Commands;
using TwitchChatTTS.Chat.Emotes;
using TwitchChatTTS.Chat.Groups;
using TwitchChatTTS.Chat.Groups.Permissions;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Hermes.Socket;
using TwitchChatTTS.OBS.Socket;
using TwitchChatTTS.Twitch.Socket;

2
Chat/Speech/TTSPlayer.cs
View File

@ -1,6 +1,6 @@
using NAudio.Wave;
namespace TwitchChatTTS.Chat.Soeech
namespace TwitchChatTTS.Chat.Speech
{
public class TTSPlayer
{

4
Startup.cs
View File

@ -29,7 +29,7 @@ using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Twitch.Socket.Messages;
using TwitchChatTTS.Twitch.Socket.Handlers;
using CommonSocketLibrary.Backoff;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Chat.Messaging;
// dotnet publish -r linux-x64 -p:PublishSingleFile=true --self-contained true
@ -156,5 +156,7 @@ s.AddKeyedSingleton<MessageTypeManager<IWebSocketHandler>, HermesMessageTypeMana
s.AddKeyedSingleton<SocketClient<WebSocketMessage>, HermesSocketClient>("hermes");
s.AddHostedService<TTS>();
s.AddHostedService<TTSListening>();
s.AddHostedService<TTSEngine>();
using IHost host = builder.Build();
await host.RunAsync();

122
TTS.cs
View File

@ -15,7 +15,7 @@ using TwitchChatTTS.Twitch.Socket.Messages;
using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Chat.Commands;
using System.Text;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using NAudio.Wave;
namespace TwitchChatTTS
@ -127,6 +127,14 @@ namespace TwitchChatTTS
return;
}
_playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) =>
{
if (_player.Playing?.Audio == e.SampleProvider)
{
_player.Playing = null;
}
});
try
{
await _twitch.Connect();
@ -147,123 +155,15 @@ namespace TwitchChatTTS
await InitializeEmotes(_sevenApiClient, emoteSet);
await InitializeSevenTv();
await InitializeObs();
_playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) =>
{
if (_player.Playing?.Audio == e.SampleProvider)
{
_player.Playing = null;
}
});
Task.Run(async () =>
{
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.Warning("TTS Buffer - Cancellation requested.");
return;
}
var group = _player.ReceiveBuffer();
if (group == null)
{
await Task.Delay(200);
continue;
}
Task.Run(() =>
{
var list = new List<ISampleProvider>();
foreach (var message in group.Messages)
{
if (string.IsNullOrEmpty(message.Message))
{
using (var reader2 = new AudioFileReader(message.File))
{
list.Add(_playback.ConvertSound(reader2.ToWaveProvider()));
}
continue;
}
try
{
string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}";
var nws = new NetworkWavSound(url);
var provider = new CachedWavProvider(nws);
var data = _playback.ConvertSound(provider);
var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate);
list.Add(resampled);
}
catch (Exception e)
{
_logger.Error(e, "Failed to fetch TTS message for ");
}
}
var merged = new ConcatenatingSampleProvider(list);
group.Audio = merged;
_player.Ready(group);
});
}
catch (COMException e)
{
_logger.Error(e, "Failed to send request for TTS [HResult: " + e.HResult + "]");
}
catch (Exception e)
{
_logger.Error(e, "Failed to send request for TTS.");
}
}
});
Task.Run(async () =>
{
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.Warning("TTS Queue - Cancellation requested.");
return;
}
while (_player.IsEmpty() || _player.Playing != null)
{
await Task.Delay(200);
continue;
}
var g = _player.ReceiveReady();
if (g == null)
{
continue;
}
var audio = g.Audio;
if (audio != null)
{
_player.Playing = g;
_playback.AddMixerInput(audio);
}
}
catch (Exception e)
{
_logger.Error(e, "Failed to play a TTS audio message");
}
}
});
}
public async Task StopAsync(CancellationToken cancellationToken)
public Task StopAsync(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
_logger.Warning("Application has stopped due to cancellation token.");
else
_logger.Warning("Application has stopped.");
return Task.CompletedTask;
}
private async Task InitializeHermesWebsocket()

71
TTSEngine.cs Normal file
View File

@ -0,0 +1,71 @@
using System.Runtime.InteropServices;
using System.Web;
using Microsoft.Extensions.Hosting;
using NAudio.Wave;
using NAudio.Wave.SampleProviders;
using Serilog;
using TwitchChatTTS.Chat.Speech;
namespace TwitchChatTTS
{
public class TTSEngine : IHostedService
{
private readonly AudioPlaybackEngine _playback;
private readonly TTSPlayer _player;
private readonly ILogger _logger;
public TTSEngine(AudioPlaybackEngine playback, TTSPlayer player, ILogger logger)
{
_playback = playback;
_player = player;
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
Task.Run(async () =>
{
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.Warning("TTS Engine - Cancellation requested.");
return;
}
while (_player.IsEmpty() || _player.Playing != null)
{
await Task.Delay(200, cancellationToken);
continue;
}
var messageData = _player.ReceiveReady();
if (messageData == null)
continue;
if (messageData.Audio != null)
{
_player.Playing = messageData;
_playback.AddMixerInput(messageData.Audio);
string message = string.Join(" ", messageData.Messages.Select(m => m.File == null ? m.Message : '(' + m.File + ')'));
_logger.Debug($"Playing TTS message [message: {message}][chatter id: {messageData.ChatterId}][priority: {messageData.Priority}][message id: {messageData.MessageId}][broadcaster id: {messageData.RoomId}]");
}
}
catch (Exception e)
{
_logger.Error(e, "Failed to play a TTS audio message");
}
}
});
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}

117
TTSListening.cs Normal file
View File

@ -0,0 +1,117 @@
using System.Runtime.InteropServices;
using System.Web;
using Microsoft.Extensions.Hosting;
using NAudio.Wave;
using NAudio.Wave.SampleProviders;
using Serilog;
using TwitchChatTTS.Chat.Speech;
namespace TwitchChatTTS
{
public class TTSListening : IHostedService
{
private readonly AudioPlaybackEngine _playback;
private readonly TTSPlayer _player;
private readonly ILogger _logger;
public TTSListening(AudioPlaybackEngine playback, TTSPlayer player, ILogger logger)
{
_playback = playback;
_player = player;
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
Task.Run(async () =>
{
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.Warning("TTS Listening - Cancellation requested.");
return;
}
var group = _player.ReceiveBuffer();
if (group == null)
{
await Task.Delay(200, cancellationToken);
continue;
}
if (cancellationToken.IsCancellationRequested)
{
_logger.Warning("TTS Listening - Cancellation requested.");
return;
}
FetchMasterAudio(group);
}
catch (COMException e)
{
_logger.Error(e, "Failed to send request for TTS [HResult: " + e.HResult + "]");
}
catch (Exception e)
{
_logger.Error(e, "Failed to send request for TTS.");
}
}
});
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
private Task FetchMasterAudio(TTSGroupedMessage group)
{
return Task.Run(() =>
{
var list = new List<ISampleProvider>();
foreach (var message in group.Messages)
{
if (string.IsNullOrEmpty(message.Message))
{
using (var reader = new AudioFileReader(message.File))
{
var data = _playback.ConvertSound(reader.ToWaveProvider());
var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate);
list.Add(resampled);
}
continue;
}
if (string.IsNullOrEmpty(message.Voice))
{
_logger.Error($"No voice has been selected for this message [message: {message.Message}]");
continue;
}
try
{
string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}";
var nws = new NetworkWavSound(url);
var provider = new CachedWavProvider(nws);
var data = _playback.ConvertSound(provider);
var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate);
list.Add(resampled);
}
catch (Exception e)
{
_logger.Error(e, "Failed to fetch TTS message for ");
}
}
var merged = new ConcatenatingSampleProvider(list);
group.Audio = merged;
_player.Ready(group);
});
}
}
}

2
Twitch/Socket/Handlers/ChannelChatClearHandler.cs
View File

@ -1,5 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers

2
Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs
View File

@ -1,5 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers

2
Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs
View File

@ -1,5 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Speech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers