From 2056b2cd482d9a1829b0760ff80bbd5a0d6773ef Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 12 Aug 2024 05:44:31 +0000 Subject: [PATCH] TTS messages are now merged. --- Chat/Commands/SkipCommand.cs | 1 + Chat/Messaging/ChatMessageReader.cs | 345 ++++++++++++++++++ Chat/Speech/TTSPlayer.cs | 336 +++++++++-------- Hermes/Socket/HermesSocketClient.cs | 67 ++-- Startup.cs | 4 + TTS.cs | 68 ++-- .../Handlers/ChannelChatClearHandler.cs | 1 + .../Handlers/ChannelChatClearUserHandler.cs | 1 + .../ChannelChatDeleteMessageHandler.cs | 3 +- .../Handlers/ChannelChatMessageHandler.cs | 304 +-------------- .../Handlers/ChannelResubscriptionHandler.cs | 5 +- User.cs | 2 +- 12 files changed, 614 insertions(+), 523 deletions(-) create mode 100644 Chat/Messaging/ChatMessageReader.cs diff --git a/Chat/Commands/SkipCommand.cs b/Chat/Commands/SkipCommand.cs index 6b03be1..92e46c6 100644 --- a/Chat/Commands/SkipCommand.cs +++ b/Chat/Commands/SkipCommand.cs @@ -1,4 +1,5 @@ using Serilog; +using TwitchChatTTS.Chat.Soeech; using TwitchChatTTS.Hermes.Socket; using TwitchChatTTS.Twitch.Socket; using TwitchChatTTS.Twitch.Socket.Messages; diff --git a/Chat/Messaging/ChatMessageReader.cs b/Chat/Messaging/ChatMessageReader.cs new file mode 100644 index 0000000..97c91f0 --- /dev/null +++ b/Chat/Messaging/ChatMessageReader.cs @@ -0,0 +1,345 @@ +using System.Text.RegularExpressions; +using CommonSocketLibrary.Abstract; +using CommonSocketLibrary.Common; +using Microsoft.Extensions.DependencyInjection; +using Serilog; +using TwitchChatTTS.Chat.Commands; +using TwitchChatTTS.Chat.Emotes; +using TwitchChatTTS.Chat.Groups; +using TwitchChatTTS.Chat.Groups.Permissions; +using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Hermes.Socket; +using TwitchChatTTS.OBS.Socket; +using TwitchChatTTS.Twitch.Socket; +using TwitchChatTTS.Twitch.Socket.Messages; + +namespace TwitchChatTTS.Chat.Messaging +{ + public class ChatMessageReader + { + public string Name => "channel.chat.message"; + + private readonly User _user; + private readonly TTSPlayer _player; + private readonly ICommandManager _commands; + private readonly IGroupPermissionManager _permissionManager; + private readonly IChatterGroupManager _chatterGroupManager; + private readonly IEmoteDatabase _emotes; + private readonly OBSSocketClient _obs; + private readonly HermesSocketClient _hermes; + private readonly Configuration _configuration; + private readonly ILogger _logger; + + private readonly Regex _sfxRegex; + + + public ChatMessageReader( + User user, + TTSPlayer player, + ICommandManager commands, + IGroupPermissionManager permissionManager, + IChatterGroupManager chatterGroupManager, + IEmoteDatabase emotes, + [FromKeyedServices("hermes")] SocketClient hermes, + [FromKeyedServices("obs")] SocketClient obs, + Configuration configuration, + ILogger logger + ) + { + _user = user; + _player = player; + _commands = commands; + _permissionManager = permissionManager; + _chatterGroupManager = chatterGroupManager; + _emotes = emotes; + _obs = (obs as OBSSocketClient)!; + _hermes = (hermes as HermesSocketClient)!; + _configuration = configuration; + _logger = logger; + + _sfxRegex = new Regex(@"\(([A-Za-z0-9_-]+)\)", RegexOptions.Compiled); + _logger = logger; + } + + public async Task Execute(TwitchWebsocketClient sender, ChannelChatMessage message) + { + if (_hermes.Connected && !_hermes.Ready) + { + _logger.Debug($"TTS is not yet ready. Ignoring chat messages [message id: {message.MessageId}]"); + return; // new MessageResult(MessageStatus.NotReady, -1, -1); + } + if (_configuration.Twitch?.TtsWhenOffline != true && !_obs.Streaming) + { + _logger.Debug($"OBS is not streaming. Ignoring chat messages [message id: {message.MessageId}]"); + return; // new MessageResult(MessageStatus.NotReady, -1, -1); + } + + var chatterId = long.Parse(message.ChatterUserId); + var broadcasterId = long.Parse(message.BroadcasterUserId); + var messageId = message.MessageId; + var groups = GetGroups(message, chatterId); + var commandResult = await CheckForChatCommand(message.Message.Text, message, groups); + if (commandResult != ChatCommandResult.Unknown) + return; + + var bits = GetTotalBits(message); + if (!HasPermission(message, chatterId, groups, bits)) + { + _logger.Debug($"Blocked message by {message.ChatterUserLogin}: {message.Message.Text}"); + return; + } + + var emoteUsage = GetEmoteUsage(message); + var tasks = new List(); + if (_obs.Streaming) + { + if (emoteUsage.NewEmotes.Any()) + tasks.Add(_hermes.SendEmoteDetails(emoteUsage.NewEmotes)); + if (emoteUsage.EmotesUsed.Any()) + tasks.Add(_hermes.SendEmoteUsage(message.MessageId, chatterId, emoteUsage.EmotesUsed)); + if (!_user.Chatters.Contains(chatterId)) + { + tasks.Add(_hermes.SendChatterDetails(chatterId, message.ChatterUserLogin)); + _user.Chatters.Add(chatterId); + } + } + + if (_user.Raids.TryGetValue(message.BroadcasterUserId, out var raid) && !raid.Chatters.Contains(chatterId)) + { + _logger.Information($"Potential chat message from raider ignored due to potential raid message spam [chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}]"); + return; + } + + var msg = FilterMessage(message); + int priority = _chatterGroupManager.GetPriorityFor(groups); + string voiceSelected = GetSelectedVoiceFor(chatterId); + var messages = GetPartialTTSMessages(msg, voiceSelected).ToList(); + var groupedMessage = new TTSGroupedMessage(broadcasterId, chatterId, messageId, messages, DateTime.UtcNow, priority); + _player.Add(groupedMessage, groupedMessage.Priority); + + if (tasks.Any()) + await Task.WhenAll(tasks); + } + + private IEnumerable HandlePartialMessage(string voice, string message) + { + var parts = _sfxRegex.Split(message); + + if (parts.Length == 1) + { + return [new TTSMessage() + { + Voice = voice, + Message = message, + }]; + } + + var list = new List(); + var sfxMatches = _sfxRegex.Matches(message); + for (var i = 0; i < sfxMatches.Count; i++) + { + var sfxMatch = sfxMatches[i]; + var sfxName = sfxMatch.Groups[1]?.ToString()?.ToLower(); + + if (!File.Exists("sfx/" + sfxName + ".mp3")) + { + parts[i * 2 + 2] = parts[i * 2] + " (" + parts[i * 2 + 1] + ")" + parts[i * 2 + 2]; + continue; + } + + if (!string.IsNullOrWhiteSpace(parts[i * 2])) + { + list.Add(new TTSMessage() + { + Voice = voice, + Message = parts[i * 2] + }); + } + + list.Add(new TTSMessage() + { + Voice = voice, + File = $"sfx/{sfxName}.mp3" + }); + } + + var lastContent = parts.Last(); + if (!string.IsNullOrWhiteSpace(lastContent)) + { + list.Add(new TTSMessage() + { + Voice = voice, + Message = lastContent + }); + } + return list; + } + + private async Task CheckForChatCommand(string arguments, ChannelChatMessage message, IEnumerable groups) + { + try + { + var commandResult = await _commands.Execute(arguments, message, groups); + return commandResult; + } + catch (Exception ex) + { + _logger.Error(ex, $"Failed executing a chat command [message: {arguments}][chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}][message id: {message.MessageId}]"); + } + return ChatCommandResult.Fail; + } + + private string FilterMessage(ChannelChatMessage message) + { + var msg = string.Join(string.Empty, message.Message.Fragments.Where(f => f.Type != "cheermote").Select(f => f.Text)).Trim(); + if (message.Reply != null) + msg = msg.Substring(message.Reply.ParentUserLogin.Length + 2); + + // Replace filtered words. + if (_user.RegexFilters != null) + { + foreach (var wf in _user.RegexFilters) + { + if (wf.Search == null || wf.Replace == null) + continue; + + if (wf.Regex != null) + { + try + { + msg = wf.Regex.Replace(msg, wf.Replace); + continue; + } + catch (Exception) + { + wf.Regex = null; + } + } + + msg = msg.Replace(wf.Search, wf.Replace); + } + } + return msg; + } + + private ChatMessageEmoteUsage GetEmoteUsage(ChannelChatMessage message) + { + var emotesUsed = new HashSet(); + var newEmotes = new Dictionary(); + foreach (var fragment in message.Message.Fragments) + { + if (fragment.Emote != null) + { + if (_emotes.Get(fragment.Text) == null) + { + newEmotes.Add(fragment.Text, fragment.Emote.Id); + _emotes.Add(fragment.Text, fragment.Emote.Id); + } + emotesUsed.Add(fragment.Emote.Id); + continue; + } + + if (fragment.Mention != null) + continue; + + var text = fragment.Text.Trim(); + var textFragments = text.Split(' '); + foreach (var f in textFragments) + { + var emoteId = _emotes.Get(f); + if (emoteId != null) + { + emotesUsed.Add(emoteId); + } + } + } + return new ChatMessageEmoteUsage(emotesUsed, newEmotes); + } + + private int GetTotalBits(ChannelChatMessage message) + { + return message.Message.Fragments.Where(f => f.Type == "cheermote" && f.Cheermote != null) + .Select(f => f.Cheermote!.Bits) + .Sum(); + } + + private string GetGroupNameByBadgeName(string badgeName) + { + if (badgeName == "subscriber") + return "subscribers"; + if (badgeName == "moderator") + return "moderators"; + return badgeName.ToLower(); + } + + private IEnumerable GetGroups(ChannelChatMessage message, long chatterId) + { + var defaultGroups = new string[] { "everyone" }; + var badgesGroups = message.Badges.Select(b => b.SetId).Select(GetGroupNameByBadgeName); + var customGroups = _chatterGroupManager.GetGroupNamesFor(chatterId); + return defaultGroups.Union(badgesGroups).Union(customGroups); + } + + private IEnumerable GetPartialTTSMessages(string message, string defaultVoice) + { + var matches = _user.VoiceNameRegex?.Matches(message).ToArray(); + if (matches == null || matches.FirstOrDefault() == null || matches.First().Index < 0) + { + return [new TTSMessage() + { + Voice = defaultVoice, + Message = message + }]; + } + + return matches.Cast().SelectMany(match => + { + var m = match.Groups["message"].Value; + if (string.IsNullOrWhiteSpace(m)) + return []; + + var voiceSelected = match.Groups["voice"].Value; + voiceSelected = voiceSelected[0].ToString().ToUpper() + voiceSelected.Substring(1).ToLower(); + return HandlePartialMessage(voiceSelected, m); + }); + } + + private string GetSelectedVoiceFor(long chatterId) + { + string? voiceSelected = _user.DefaultTTSVoice; + if (_user.VoicesSelected?.ContainsKey(chatterId) == true) + { + var voiceId = _user.VoicesSelected[chatterId]; + if (_user.VoicesAvailable.TryGetValue(voiceId, out string? voiceName) && voiceName != null) + { + if (_user.VoicesEnabled.Contains(voiceName) || chatterId == _user.OwnerId) + voiceSelected = voiceName; + } + } + return voiceSelected ?? "Brian"; + } + + private bool HasPermission(ChannelChatMessage message, long chatterId, IEnumerable groups, int bits) + { + var permissionPath = "tts.chat.messages.read"; + if (!string.IsNullOrWhiteSpace(message.ChannelPointsCustomRewardId)) + permissionPath = "tts.chat.redemptions.read"; + else if (bits > 0) + permissionPath = "tts.chat.bits.read"; + + return chatterId == _user.OwnerId ? true : _permissionManager.CheckIfAllowed(groups, permissionPath) == true; + } + + private class ChatMessageEmoteUsage + { + public readonly HashSet EmotesUsed = new HashSet(); + public readonly IDictionary NewEmotes = new Dictionary(); + + public ChatMessageEmoteUsage(HashSet emotesUsed, IDictionary newEmotes) + { + EmotesUsed = emotesUsed; + NewEmotes = newEmotes; + } + } + } +} \ No newline at end of file diff --git a/Chat/Speech/TTSPlayer.cs b/Chat/Speech/TTSPlayer.cs index 89ccd87..748dc33 100644 --- a/Chat/Speech/TTSPlayer.cs +++ b/Chat/Speech/TTSPlayer.cs @@ -1,198 +1,218 @@ using NAudio.Wave; -using TwitchChatTTS.Twitch.Socket.Messages; -public class TTSPlayer +namespace TwitchChatTTS.Chat.Soeech { - private readonly PriorityQueue _messages; // ready to play - private readonly PriorityQueue _buffer; - private readonly Mutex _mutex; - private readonly Mutex _mutex2; - - public TTSMessage? Playing { get; set; } - - public TTSPlayer() + public class TTSPlayer { - _messages = new PriorityQueue(new DescendingOrder()); - _buffer = new PriorityQueue(new DescendingOrder()); - _mutex = new Mutex(); - _mutex2 = new Mutex(); - } + private readonly PriorityQueue _messages; // ready to play + private readonly PriorityQueue _buffer; + private readonly Mutex _mutex; + private readonly Mutex _mutex2; - public void Add(TTSMessage message) - { - try + //public TTSGroupedMessage? PlayingGroup { get; set; } + public TTSGroupedMessage? Playing { get; set; } + + public TTSPlayer() { - _mutex2.WaitOne(); - _buffer.Enqueue(message, message.Priority); + _messages = new PriorityQueue(new DescendingOrder()); + _buffer = new PriorityQueue(new DescendingOrder()); + _mutex = new Mutex(); + _mutex2 = new Mutex(); } - finally - { - _mutex2.ReleaseMutex(); - } - } - public TTSMessage? ReceiveReady() - { - try + public void Add(TTSGroupedMessage message, int priority) { - _mutex.WaitOne(); - if (_messages.TryDequeue(out TTSMessage? message, out int _)) + try { - return message; + _mutex2.WaitOne(); + _buffer.Enqueue(message, priority); } - return null; - } - finally - { - _mutex.ReleaseMutex(); - } - } - - public TTSMessage? ReceiveBuffer() - { - try - { - _mutex2.WaitOne(); - if (_buffer.TryDequeue(out TTSMessage? message, out int _)) + finally { - return message; + _mutex2.ReleaseMutex(); } - return null; - } - finally - { - _mutex2.ReleaseMutex(); - } - } - - public void Ready(TTSMessage message) - { - try - { - _mutex.WaitOne(); - _messages.Enqueue(message, message.Priority); - } - finally - { - _mutex.ReleaseMutex(); - } - } - - public void RemoveAll() - { - try - { - _mutex2.WaitOne(); - _buffer.Clear(); - } - finally - { - _mutex2.ReleaseMutex(); } - try + public TTSGroupedMessage? ReceiveReady() { - _mutex.WaitOne(); - _messages.Clear(); - } - finally - { - _mutex.ReleaseMutex(); - } - } - - public void RemoveAll(long broadcasterId, long chatterId) - { - try - { - _mutex2.WaitOne(); - if (_buffer.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId)) + try { - var list = _buffer.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray(); + _mutex.WaitOne(); + if (_messages.TryDequeue(out TTSGroupedMessage? message, out int _)) + { + return message; + } + return null; + } + finally + { + _mutex.ReleaseMutex(); + } + } + + public TTSGroupedMessage? ReceiveBuffer() + { + try + { + _mutex2.WaitOne(); + if (_buffer.TryDequeue(out TTSGroupedMessage? messages, out int _)) + { + return messages; + } + return null; + } + finally + { + _mutex2.ReleaseMutex(); + } + } + + public void Ready(TTSGroupedMessage message) + { + try + { + _mutex.WaitOne(); + _messages.Enqueue(message, message.Priority); + } + finally + { + _mutex.ReleaseMutex(); + } + } + + public void RemoveAll() + { + try + { + _mutex2.WaitOne(); _buffer.Clear(); - foreach (var item in list) - _buffer.Enqueue(item.Element, item.Element.Priority); } - } - finally - { - _mutex2.ReleaseMutex(); - } - - try - { - _mutex.WaitOne(); - if (_messages.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId)) + finally { - var list = _messages.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray(); + _mutex2.ReleaseMutex(); + } + + try + { + _mutex.WaitOne(); _messages.Clear(); - foreach (var item in list) - _messages.Enqueue(item.Element, item.Element.Priority); } - } - finally - { - _mutex.ReleaseMutex(); - } - } - - public void RemoveMessage(string messageId) - { - try - { - _mutex2.WaitOne(); - if (_buffer.UnorderedItems.Any(i => i.Element.MessageId == messageId)) + finally { - var list = _buffer.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray(); - _buffer.Clear(); - foreach (var item in list) - _buffer.Enqueue(item.Element, item.Element.Priority); - return; + _mutex.ReleaseMutex(); } } - finally - { - _mutex2.ReleaseMutex(); - } - try + public void RemoveAll(long broadcasterId, long chatterId) { - _mutex.WaitOne(); - if (_messages.UnorderedItems.Any(i => i.Element.MessageId == messageId)) + try { - var list = _messages.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray(); - _messages.Clear(); - foreach (var item in list) - _messages.Enqueue(item.Element, item.Element.Priority); + _mutex2.WaitOne(); + if (_buffer.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId)) + { + var list = _buffer.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray(); + _buffer.Clear(); + foreach (var item in list) + _buffer.Enqueue(item.Element, item.Element.Priority); + } + } + finally + { + _mutex2.ReleaseMutex(); + } + + try + { + _mutex.WaitOne(); + if (_messages.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId)) + { + var list = _messages.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray(); + _messages.Clear(); + foreach (var item in list) + _messages.Enqueue(item.Element, item.Element.Priority); + } + } + finally + { + _mutex.ReleaseMutex(); } } - finally + + public void RemoveMessage(string messageId) { - _mutex.ReleaseMutex(); + try + { + _mutex2.WaitOne(); + if (_buffer.UnorderedItems.Any(i => i.Element.MessageId == messageId)) + { + var list = _buffer.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray(); + _buffer.Clear(); + foreach (var item in list) + _buffer.Enqueue(item.Element, item.Element.Priority); + return; + } + } + finally + { + _mutex2.ReleaseMutex(); + } + + try + { + _mutex.WaitOne(); + if (_messages.UnorderedItems.Any(i => i.Element.MessageId == messageId)) + { + var list = _messages.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray(); + _messages.Clear(); + foreach (var item in list) + _messages.Enqueue(item.Element, item.Element.Priority); + } + } + finally + { + _mutex.ReleaseMutex(); + } + } + + public bool IsEmpty() + { + return _messages.Count == 0; + } + + private class DescendingOrder : IComparer + { + public int Compare(int x, int y) => y.CompareTo(x); } } - public bool IsEmpty() + public class TTSMessage { - return _messages.Count == 0; + public string? Voice { get; set; } + public string? Message { get; set; } + public string? File { get; set; } } - private class DescendingOrder : IComparer + public class TTSGroupedMessage { - public int Compare(int x, int y) => y.CompareTo(x); - } -} + public long RoomId { get; set; } + public long ChatterId { get; set; } + public string MessageId { get; set; } + public DateTime Timestamp { get; set; } + public int Priority { get; set; } + public IList Messages { get; set; } + //public IList Audios { get; set; } + public ISampleProvider? Audio { get; set; } -public class TTSMessage -{ - public string? Voice { get; set; } - public long RoomId { get; set; } - public long ChatterId { get; set; } - public string MessageId { get; set; } - public string? Message { get; set; } - public string? File { get; set; } - public DateTime Timestamp { get; set; } - public IEnumerable Badges { get; set; } - public int Priority { get; set; } - public ISampleProvider? Audio { get; set; } + + public TTSGroupedMessage(long broadcasterId, long chatterId, string messageId, IList messages, DateTime timestamp, int priority) + { + RoomId = broadcasterId; + ChatterId = chatterId; + MessageId = messageId; + Messages = messages; + Timestamp = timestamp; + Priority = priority; + //Audios = new List(); + } + } } \ No newline at end of file diff --git a/Hermes/Socket/HermesSocketClient.cs b/Hermes/Socket/HermesSocketClient.cs index 4a0c1c3..bec3018 100644 --- a/Hermes/Socket/HermesSocketClient.cs +++ b/Hermes/Socket/HermesSocketClient.cs @@ -27,6 +27,7 @@ namespace TwitchChatTTS.Hermes.Socket public string? UserId { get; set; } private readonly System.Timers.Timer _heartbeatTimer; private readonly IBackoff _backoff; + private readonly object _lock; public bool Connected { get; set; } public bool LoggedIn { get; set; } @@ -56,13 +57,18 @@ namespace TwitchChatTTS.Hermes.Socket LastHeartbeatReceived = LastHeartbeatSent = DateTime.UtcNow; URL = $"wss://{BASE_URL}"; + + _lock = new object(); } public async Task Connect() { - if (Connected) - return; + lock (_lock) + { + if (Connected) + return; + } _logger.Debug($"Attempting to connect to {URL}"); await ConnectAsync(URL); @@ -70,8 +76,11 @@ namespace TwitchChatTTS.Hermes.Socket private async Task Disconnect() { - if (!Connected) - return; + lock (_lock) + { + if (!Connected) + return; + } await DisconnectAsync(new SocketDisconnectionEventArgs("Normal disconnection", "Disconnection was executed")); } @@ -212,7 +221,12 @@ namespace TwitchChatTTS.Hermes.Socket OnConnected += async (sender, e) => { - Connected = true; + lock (_lock) + { + if (Connected) + return; + Connected = true; + } _logger.Information("Hermes websocket client connected."); _heartbeatTimer.Enabled = true; @@ -228,7 +242,13 @@ namespace TwitchChatTTS.Hermes.Socket OnDisconnected += async (sender, e) => { - Connected = false; + lock (_lock) + { + if (!Connected) + return; + Connected = false; + } + LoggedIn = false; Ready = false; _logger.Warning("Hermes websocket client disconnected."); @@ -334,7 +354,7 @@ namespace TwitchChatTTS.Hermes.Socket { var signalTime = e.SignalTime.ToUniversalTime(); - if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(30)) + if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(60)) { if (LastHeartbeatReceived > LastHeartbeatSent) { @@ -349,7 +369,7 @@ namespace TwitchChatTTS.Hermes.Socket _logger.Error(ex, "Failed to send a heartbeat back to the Hermes websocket server."); } } - else if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(60)) + else if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(120)) { try { @@ -369,37 +389,6 @@ namespace TwitchChatTTS.Hermes.Socket } } - private async Task Reconnect(ElapsedEventArgs e) - { - if (Connected) - { - try - { - await Disconnect(); - } - catch (Exception ex) - { - _logger.Error(ex, "Failed to disconnect from Hermes websocket server."); - Ready = false; - LoggedIn = false; - Connected = false; - } - } - - try - { - await Connect(); - } - catch (WebSocketException wse) when (wse.Message.Contains("502")) - { - _logger.Error($"Hermes websocket server cannot be found [code: {wse.ErrorCode}]"); - } - catch (Exception ex) - { - _logger.Error(ex, "Failed to reconnect to Hermes websocket server."); - } - } - public new async Task Send(int opcode, T message) { if (!Connected) diff --git a/Startup.cs b/Startup.cs index b077187..03f16e3 100644 --- a/Startup.cs +++ b/Startup.cs @@ -29,6 +29,8 @@ using TwitchChatTTS.Twitch.Socket; using TwitchChatTTS.Twitch.Socket.Messages; using TwitchChatTTS.Twitch.Socket.Handlers; using CommonSocketLibrary.Backoff; +using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Messaging; // dotnet publish -r linux-x64 -p:PublishSingleFile=true --self-contained true // dotnet publish -r win-x64 -p:PublishSingleFile=true --self-contained true @@ -112,6 +114,8 @@ s.AddKeyedSingleton, SevenSocketClient>("7tv"); // Nightbot s.AddSingleton(); +s.AddSingleton(); + // twitch websocket s.AddKeyedSingleton("twitch", new ExponentialBackoff(1000, 120 * 1000)); s.AddSingleton(); diff --git a/TTS.cs b/TTS.cs index a1eaa8d..9a52de0 100644 --- a/TTS.cs +++ b/TTS.cs @@ -15,6 +15,8 @@ using TwitchChatTTS.Twitch.Socket.Messages; using TwitchChatTTS.Twitch.Socket; using TwitchChatTTS.Chat.Commands; using System.Text; +using TwitchChatTTS.Chat.Soeech; +using NAudio.Wave; namespace TwitchChatTTS { @@ -26,7 +28,6 @@ namespace TwitchChatTTS private readonly User _user; private readonly HermesApiClient _hermesApiClient; private readonly SevenApiClient _sevenApiClient; - private readonly TwitchApiClient _twitchApiClient; private readonly HermesSocketClient _hermes; private readonly OBSSocketClient _obs; private readonly SevenSocketClient _seven; @@ -149,7 +150,7 @@ namespace TwitchChatTTS _playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) => { - if (e.SampleProvider == _player.Playing?.Audio) + if (_player.Playing?.Audio == e.SampleProvider) { _player.Playing = null; } @@ -167,22 +168,46 @@ namespace TwitchChatTTS return; } - var m = _player.ReceiveBuffer(); - if (m == null) + var group = _player.ReceiveBuffer(); + if (group == null) { await Task.Delay(200); continue; } - string url = $"https://api.streamelements.com/kappa/v2/speech?voice={m.Voice}&text={HttpUtility.UrlEncode(m.Message)}"; - var sound = new NetworkWavSound(url); - var provider = new CachedWavProvider(sound); - var data = _playback.ConvertSound(provider); - var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate); - _logger.Verbose("Fetched TTS audio data."); + Task.Run(() => + { + var list = new List(); + foreach (var message in group.Messages) + { + if (string.IsNullOrEmpty(message.Message)) + { + using (var reader2 = new AudioFileReader(message.File)) + { + list.Add(_playback.ConvertSound(reader2.ToWaveProvider())); + } + continue; + } - m.Audio = resampled; - _player.Ready(m); + try + { + string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}"; + var nws = new NetworkWavSound(url); + var provider = new CachedWavProvider(nws); + var data = _playback.ConvertSound(provider); + var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate); + list.Add(resampled); + } + catch (Exception e) + { + _logger.Error(e, "Failed to fetch TTS message for "); + } + } + + var merged = new ConcatenatingSampleProvider(list); + group.Audio = merged; + _player.Ready(group); + }); } catch (COMException e) { @@ -211,25 +236,18 @@ namespace TwitchChatTTS await Task.Delay(200); continue; } - var m = _player.ReceiveReady(); - if (m == null) + var g = _player.ReceiveReady(); + if (g == null) { continue; } - if (!string.IsNullOrWhiteSpace(m.File) && File.Exists(m.File)) - { - _logger.Debug("Playing audio file via TTS: " + m.File); - _playback.PlaySound(m.File); - continue; - } + var audio = g.Audio; - _logger.Debug("Playing message via TTS: " + m.Message); - - if (m.Audio != null) + if (audio != null) { - _player.Playing = m; - _playback.AddMixerInput(m.Audio); + _player.Playing = g; + _playback.AddMixerInput(audio); } } catch (Exception e) diff --git a/Twitch/Socket/Handlers/ChannelChatClearHandler.cs b/Twitch/Socket/Handlers/ChannelChatClearHandler.cs index 2c0dffa..241d108 100644 --- a/Twitch/Socket/Handlers/ChannelChatClearHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatClearHandler.cs @@ -1,4 +1,5 @@ using Serilog; +using TwitchChatTTS.Chat.Soeech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers diff --git a/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs b/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs index 8bb0af4..05b3dcf 100644 --- a/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs @@ -1,4 +1,5 @@ using Serilog; +using TwitchChatTTS.Chat.Soeech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers diff --git a/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs b/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs index 2d455fa..691445b 100644 --- a/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs @@ -1,4 +1,5 @@ using Serilog; +using TwitchChatTTS.Chat.Soeech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers @@ -25,7 +26,7 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers if (_player.Playing?.MessageId == message.MessageId) { - _playback.RemoveMixerInput(_player.Playing.Audio!); + _playback.RemoveMixerInput(_player.Playing!.Audio!); _player.Playing = null; } else diff --git a/Twitch/Socket/Handlers/ChannelChatMessageHandler.cs b/Twitch/Socket/Handlers/ChannelChatMessageHandler.cs index 295c24c..b8c62e3 100644 --- a/Twitch/Socket/Handlers/ChannelChatMessageHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatMessageHandler.cs @@ -1,14 +1,5 @@ -using System.Text.RegularExpressions; -using CommonSocketLibrary.Abstract; -using CommonSocketLibrary.Common; -using Microsoft.Extensions.DependencyInjection; using Serilog; -using TwitchChatTTS.Chat.Commands; -using TwitchChatTTS.Chat.Emotes; -using TwitchChatTTS.Chat.Groups; -using TwitchChatTTS.Chat.Groups.Permissions; -using TwitchChatTTS.Hermes.Socket; -using TwitchChatTTS.OBS.Socket; +using TwitchChatTTS.Chat.Messaging; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers @@ -17,48 +8,23 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers { public string Name => "channel.chat.message"; - private readonly User _user; - private readonly TTSPlayer _player; - private readonly ICommandManager _commands; - private readonly IGroupPermissionManager _permissionManager; - private readonly IChatterGroupManager _chatterGroupManager; - private readonly IEmoteDatabase _emotes; - private readonly OBSSocketClient _obs; - private readonly HermesSocketClient _hermes; + private readonly ChatMessageReader _reader; private readonly Configuration _configuration; private readonly ILogger _logger; - private readonly Regex _sfxRegex; - public ChannelChatMessageHandler( - User user, - TTSPlayer player, - ICommandManager commands, - IGroupPermissionManager permissionManager, - IChatterGroupManager chatterGroupManager, - IEmoteDatabase emotes, - [FromKeyedServices("hermes")] SocketClient hermes, - [FromKeyedServices("obs")] SocketClient obs, + ChatMessageReader reader, Configuration configuration, ILogger logger ) { - _user = user; - _player = player; - _commands = commands; - _permissionManager = permissionManager; - _chatterGroupManager = chatterGroupManager; - _emotes = emotes; - _obs = (obs as OBSSocketClient)!; - _hermes = (hermes as HermesSocketClient)!; + _reader = reader; _configuration = configuration; _logger = logger; - - _sfxRegex = new Regex(@"\(([A-Za-z0-9_-]+)\)", RegexOptions.Compiled); - _logger = logger; } + public async Task Execute(TwitchWebsocketClient sender, object data) { if (sender == null) @@ -66,265 +32,7 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers if (data is not ChannelChatMessage message) return; - if (_hermes.Connected && !_hermes.Ready) - { - _logger.Debug($"TTS is not yet ready. Ignoring chat messages [message id: {message.MessageId}]"); - return; // new MessageResult(MessageStatus.NotReady, -1, -1); - } - if (_configuration.Twitch?.TtsWhenOffline != true && !_obs.Streaming) - { - _logger.Debug($"OBS is not streaming. Ignoring chat messages [message id: {message.MessageId}]"); - return; // new MessageResult(MessageStatus.NotReady, -1, -1); - } - - var msg = string.Join(string.Empty, message.Message.Fragments.Where(f => f.Type != "cheermote").Select(f => f.Text)).Trim(); - var chatterId = long.Parse(message.ChatterUserId); - var tasks = new List(); - - var defaultGroups = new string[] { "everyone" }; - var badgesGroups = message.Badges.Select(b => b.SetId).Select(GetGroupNameByBadgeName); - var customGroups = _chatterGroupManager.GetGroupNamesFor(chatterId); - var groups = defaultGroups.Union(badgesGroups).Union(customGroups); - - try - { - var commandResult = await _commands.Execute(msg, message, groups); - if (commandResult != ChatCommandResult.Unknown) - return; // new MessageResult(MessageStatus.Command, -1, -1); - } - catch (Exception ex) - { - _logger.Error(ex, $"Failed executing a chat command [message: {msg}][chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}][message id: {message.MessageId}]"); - return; - } - - if (message.Reply != null) - msg = msg.Substring(message.Reply.ParentUserLogin.Length + 2); - - var bits = message.Message.Fragments.Where(f => f.Type == "cheermote" && f.Cheermote != null) - .Select(f => f.Cheermote!.Bits) - .Sum(); - var permissionPath = "tts.chat.messages.read"; - if (!string.IsNullOrWhiteSpace(message.ChannelPointsCustomRewardId)) - permissionPath = "tts.chat.redemptions.read"; - else if (bits > 0) - permissionPath = "tts.chat.bits.read"; - - var permission = chatterId == _user.OwnerId ? true : _permissionManager.CheckIfAllowed(groups, permissionPath); - if (permission != true) - { - _logger.Debug($"Blocked message by {message.ChatterUserLogin}: {msg}"); - return; // new MessageResult(MessageStatus.Blocked, -1, -1); - } - - // Keep track of emotes usage - var emotesUsed = new HashSet(); - var newEmotes = new Dictionary(); - foreach (var fragment in message.Message.Fragments) - { - if (fragment.Emote != null) - { - if (_emotes.Get(fragment.Text) == null) - { - newEmotes.Add(fragment.Text, fragment.Emote.Id); - _emotes.Add(fragment.Text, fragment.Emote.Id); - } - emotesUsed.Add(fragment.Emote.Id); - continue; - } - - if (fragment.Mention != null) - continue; - - var text = fragment.Text.Trim(); - var textFragments = text.Split(' '); - foreach (var f in textFragments) - { - var emoteId = _emotes.Get(f); - if (emoteId != null) - { - emotesUsed.Add(emoteId); - } - } - } - if (_obs.Streaming) - { - if (newEmotes.Any()) - tasks.Add(_hermes.SendEmoteDetails(newEmotes)); - if (emotesUsed.Any()) - tasks.Add(_hermes.SendEmoteUsage(message.MessageId, chatterId, emotesUsed)); - if (!_user.Chatters.Contains(chatterId)) - { - tasks.Add(_hermes.SendChatterDetails(chatterId, message.ChatterUserLogin)); - _user.Chatters.Add(chatterId); - } - } - - if (_user.Raids.TryGetValue(message.BroadcasterUserId, out var raid) && !raid.Chatters.Contains(chatterId)) - { - _logger.Information($"Potential chat message from raider ignored due to potential raid message spam [chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}]"); - return; - } - - // Replace filtered words. - if (_user.RegexFilters != null) - { - foreach (var wf in _user.RegexFilters) - { - if (wf.Search == null || wf.Replace == null) - continue; - - if (wf.Regex != null) - { - try - { - msg = wf.Regex.Replace(msg, wf.Replace); - continue; - } - catch (Exception) - { - wf.Regex = null; - } - } - - msg = msg.Replace(wf.Search, wf.Replace); - } - } - - // Determine the priority of this message - int priority = _chatterGroupManager.GetPriorityFor(groups); // + m.SubscribedMonthCount * (m.IsSubscriber ? 10 : 5); - - // Determine voice selected. - string voiceSelected = _user.DefaultTTSVoice; - if (_user.VoicesSelected?.ContainsKey(chatterId) == true) - { - var voiceId = _user.VoicesSelected[chatterId]; - if (_user.VoicesAvailable.TryGetValue(voiceId, out string? voiceName) && voiceName != null) - { - if (_user.VoicesEnabled.Contains(voiceName) || chatterId == _user.OwnerId) - voiceSelected = voiceName; - } - } - - // Determine additional voices used - var matches = _user.VoiceNameRegex?.Matches(msg).ToArray(); - if (matches == null || matches.FirstOrDefault() == null || matches.First().Index < 0) - { - HandlePartialMessage(priority, voiceSelected, msg.Trim(), message); - return; // new MessageResult(MessageStatus.None, _user.TwitchUserId, chatterId, emotesUsed); - } - - HandlePartialMessage(priority, voiceSelected, msg.Substring(0, matches.First().Index).Trim(), message); - foreach (Match match in matches) - { - var m = match.Groups[2].ToString(); - if (string.IsNullOrWhiteSpace(m)) - continue; - - var voice = match.Groups[1].ToString(); - voice = voice[0].ToString().ToUpper() + voice.Substring(1).ToLower(); - HandlePartialMessage(priority, voice, m.Trim(), message); - } - - if (tasks.Any()) - await Task.WhenAll(tasks); - } - - private void HandlePartialMessage(int priority, string voice, string message, ChannelChatMessage e) - { - if (string.IsNullOrWhiteSpace(message)) - return; - - var parts = _sfxRegex.Split(message); - var chatterId = long.Parse(e.ChatterUserId); - var broadcasterId = long.Parse(e.BroadcasterUserId); - var badgesString = string.Join(", ", e.Badges.Select(b => b.SetId + '|' + b.Id + '=' + b.Info)); - - if (parts.Length == 1) - { - _logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {message}; Reward Id: {e.ChannelPointsCustomRewardId}; {badgesString}"); - _player.Add(new TTSMessage() - { - Voice = voice, - Message = message, - Timestamp = DateTime.UtcNow, - RoomId = broadcasterId, - ChatterId = chatterId, - MessageId = e.MessageId, - Badges = e.Badges, - Priority = priority - }); - return; - } - - var sfxMatches = _sfxRegex.Matches(message); - var sfxStart = sfxMatches.FirstOrDefault()?.Index ?? message.Length; - - for (var i = 0; i < sfxMatches.Count; i++) - { - var sfxMatch = sfxMatches[i]; - var sfxName = sfxMatch.Groups[1]?.ToString()?.ToLower(); - - if (!File.Exists("sfx/" + sfxName + ".mp3")) - { - parts[i * 2 + 2] = parts[i * 2] + " (" + parts[i * 2 + 1] + ")" + parts[i * 2 + 2]; - continue; - } - - if (!string.IsNullOrWhiteSpace(parts[i * 2])) - { - _logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {parts[i * 2]}; {badgesString}"); - _player.Add(new TTSMessage() - { - Voice = voice, - Message = parts[i * 2], - Timestamp = DateTime.UtcNow, - RoomId = broadcasterId, - ChatterId = chatterId, - MessageId = e.MessageId, - Badges = e.Badges, - Priority = priority - }); - } - - _logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; SFX: {sfxName}; {badgesString}"); - _player.Add(new TTSMessage() - { - Voice = voice, - File = $"sfx/{sfxName}.mp3", - Timestamp = DateTime.UtcNow, - RoomId = broadcasterId, - ChatterId = chatterId, - MessageId = e.MessageId, - Badges = e.Badges, - Priority = priority - }); - } - - if (!string.IsNullOrWhiteSpace(parts.Last())) - { - _logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {parts.Last()}; {badgesString}"); - _player.Add(new TTSMessage() - { - Voice = voice, - Message = parts.Last(), - Timestamp = DateTime.UtcNow, - RoomId = broadcasterId, - ChatterId = chatterId, - MessageId = e.MessageId, - Badges = e.Badges, - Priority = priority - }); - } - } - - private string GetGroupNameByBadgeName(string badgeName) - { - if (badgeName == "subscriber") - return "subscribers"; - if (badgeName == "moderator") - return "moderators"; - return badgeName.ToLower(); + await _reader.Execute(sender, message); } } } \ No newline at end of file diff --git a/Twitch/Socket/Handlers/ChannelResubscriptionHandler.cs b/Twitch/Socket/Handlers/ChannelResubscriptionHandler.cs index 4fb96e9..42ecd26 100644 --- a/Twitch/Socket/Handlers/ChannelResubscriptionHandler.cs +++ b/Twitch/Socket/Handlers/ChannelResubscriptionHandler.cs @@ -1,4 +1,5 @@ using Serilog; +using TwitchChatTTS.Chat.Messaging; using TwitchChatTTS.Twitch.Redemptions; using TwitchChatTTS.Twitch.Socket.Messages; @@ -8,11 +9,13 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers { public string Name => "channel.subscription.message"; + private readonly ChatMessageReader _reader; private readonly IRedemptionManager _redemptionManager; private readonly ILogger _logger; - public ChannelResubscriptionHandler(IRedemptionManager redemptionManager, ILogger logger) + public ChannelResubscriptionHandler(ChatMessageReader reader, IRedemptionManager redemptionManager, ILogger logger) { + _reader = reader; _redemptionManager = redemptionManager; _logger = logger; } diff --git a/User.cs b/User.cs index 21f55ba..2d4fe87 100644 --- a/User.cs +++ b/User.cs @@ -43,7 +43,7 @@ namespace TwitchChatTTS return null; var enabledVoicesString = string.Join("|", VoicesAvailable.Where(v => VoicesEnabled == null || !VoicesEnabled.Any() || VoicesEnabled.Contains(v.Value)).Select(v => v.Value)); - return new Regex($@"\b({enabledVoicesString})\:(.*?)(?=\Z|\b(?:{enabledVoicesString})\:)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + return new Regex($@"(?:\A|\s)(?{enabledVoicesString}):(?.*?)(?=\Z|\s(?:{enabledVoicesString})\:)", RegexOptions.IgnoreCase | RegexOptions.Compiled); } } } \ No newline at end of file