tom/hermes-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TTS messages are now merged.

Browse Source
This commit is contained in:
Tom 2024-08-12 05:44:31 +00:00
parent 9f61295d2d
commit 2056b2cd48
12 changed files with 614 additions and 523 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Chat/Commands/SkipCommand.cs
View File

@ -1,4 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Hermes.Socket;
using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Twitch.Socket.Messages;

345
Chat/Messaging/ChatMessageReader.cs Normal file
View File

@ -0,0 +1,345 @@
using System.Text.RegularExpressions;
using CommonSocketLibrary.Abstract;
using CommonSocketLibrary.Common;
using Microsoft.Extensions.DependencyInjection;
using Serilog;
using TwitchChatTTS.Chat.Commands;
using TwitchChatTTS.Chat.Emotes;
using TwitchChatTTS.Chat.Groups;
using TwitchChatTTS.Chat.Groups.Permissions;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Hermes.Socket;
using TwitchChatTTS.OBS.Socket;
using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Chat.Messaging
{
public class ChatMessageReader
{
public string Name => "channel.chat.message";
private readonly User _user;
private readonly TTSPlayer _player;
private readonly ICommandManager _commands;
private readonly IGroupPermissionManager _permissionManager;
private readonly IChatterGroupManager _chatterGroupManager;
private readonly IEmoteDatabase _emotes;
private readonly OBSSocketClient _obs;
private readonly HermesSocketClient _hermes;
private readonly Configuration _configuration;
private readonly ILogger _logger;
private readonly Regex _sfxRegex;
public ChatMessageReader(
User user,
TTSPlayer player,
ICommandManager commands,
IGroupPermissionManager permissionManager,
IChatterGroupManager chatterGroupManager,
IEmoteDatabase emotes,
[FromKeyedServices("hermes")] SocketClient<WebSocketMessage> hermes,
[FromKeyedServices("obs")] SocketClient<WebSocketMessage> obs,
Configuration configuration,
ILogger logger
)
{
_user = user;
_player = player;
_commands = commands;
_permissionManager = permissionManager;
_chatterGroupManager = chatterGroupManager;
_emotes = emotes;
_obs = (obs as OBSSocketClient)!;
_hermes = (hermes as HermesSocketClient)!;
_configuration = configuration;
_logger = logger;
_sfxRegex = new Regex(@"\(([A-Za-z0-9_-]+)\)", RegexOptions.Compiled);
_logger = logger;
}
public async Task Execute(TwitchWebsocketClient sender, ChannelChatMessage message)
{
if (_hermes.Connected && !_hermes.Ready)
{
_logger.Debug($"TTS is not yet ready. Ignoring chat messages [message id: {message.MessageId}]");
return; // new MessageResult(MessageStatus.NotReady, -1, -1);
}
if (_configuration.Twitch?.TtsWhenOffline != true && !_obs.Streaming)
{
_logger.Debug($"OBS is not streaming. Ignoring chat messages [message id: {message.MessageId}]");
return; // new MessageResult(MessageStatus.NotReady, -1, -1);
}
var chatterId = long.Parse(message.ChatterUserId);
var broadcasterId = long.Parse(message.BroadcasterUserId);
var messageId = message.MessageId;
var groups = GetGroups(message, chatterId);
var commandResult = await CheckForChatCommand(message.Message.Text, message, groups);
if (commandResult != ChatCommandResult.Unknown)
return;
var bits = GetTotalBits(message);
if (!HasPermission(message, chatterId, groups, bits))
{
_logger.Debug($"Blocked message by {message.ChatterUserLogin}: {message.Message.Text}");
return;
}
var emoteUsage = GetEmoteUsage(message);
var tasks = new List<Task>();
if (_obs.Streaming)
{
if (emoteUsage.NewEmotes.Any())
tasks.Add(_hermes.SendEmoteDetails(emoteUsage.NewEmotes));
if (emoteUsage.EmotesUsed.Any())
tasks.Add(_hermes.SendEmoteUsage(message.MessageId, chatterId, emoteUsage.EmotesUsed));
if (!_user.Chatters.Contains(chatterId))
{
tasks.Add(_hermes.SendChatterDetails(chatterId, message.ChatterUserLogin));
_user.Chatters.Add(chatterId);
}
}
if (_user.Raids.TryGetValue(message.BroadcasterUserId, out var raid) && !raid.Chatters.Contains(chatterId))
{
_logger.Information($"Potential chat message from raider ignored due to potential raid message spam [chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}]");
return;
}
var msg = FilterMessage(message);
int priority = _chatterGroupManager.GetPriorityFor(groups);
string voiceSelected = GetSelectedVoiceFor(chatterId);
var messages = GetPartialTTSMessages(msg, voiceSelected).ToList();
var groupedMessage = new TTSGroupedMessage(broadcasterId, chatterId, messageId, messages, DateTime.UtcNow, priority);
_player.Add(groupedMessage, groupedMessage.Priority);
if (tasks.Any())
await Task.WhenAll(tasks);
}
private IEnumerable<TTSMessage> HandlePartialMessage(string voice, string message)
{
var parts = _sfxRegex.Split(message);
if (parts.Length == 1)
{
return [new TTSMessage()
{
Voice = voice,
Message = message,
}];
}
var list = new List<TTSMessage>();
var sfxMatches = _sfxRegex.Matches(message);
for (var i = 0; i < sfxMatches.Count; i++)
{
var sfxMatch = sfxMatches[i];
var sfxName = sfxMatch.Groups[1]?.ToString()?.ToLower();
if (!File.Exists("sfx/" + sfxName + ".mp3"))
{
parts[i * 2 + 2] = parts[i * 2] + " (" + parts[i * 2 + 1] + ")" + parts[i * 2 + 2];
continue;
}
if (!string.IsNullOrWhiteSpace(parts[i * 2]))
{
list.Add(new TTSMessage()
{
Voice = voice,
Message = parts[i * 2]
});
}
list.Add(new TTSMessage()
{
Voice = voice,
File = $"sfx/{sfxName}.mp3"
});
}
var lastContent = parts.Last();
if (!string.IsNullOrWhiteSpace(lastContent))
{
list.Add(new TTSMessage()
{
Voice = voice,
Message = lastContent
});
}
return list;
}
private async Task<ChatCommandResult> CheckForChatCommand(string arguments, ChannelChatMessage message, IEnumerable<string> groups)
{
try
{
var commandResult = await _commands.Execute(arguments, message, groups);
return commandResult;
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed executing a chat command [message: {arguments}][chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}][message id: {message.MessageId}]");
}
return ChatCommandResult.Fail;
}
private string FilterMessage(ChannelChatMessage message)
{
var msg = string.Join(string.Empty, message.Message.Fragments.Where(f => f.Type != "cheermote").Select(f => f.Text)).Trim();
if (message.Reply != null)
msg = msg.Substring(message.Reply.ParentUserLogin.Length + 2);
// Replace filtered words.
if (_user.RegexFilters != null)
{
foreach (var wf in _user.RegexFilters)
{
if (wf.Search == null || wf.Replace == null)
continue;
if (wf.Regex != null)
{
try
{
msg = wf.Regex.Replace(msg, wf.Replace);
continue;
}
catch (Exception)
{
wf.Regex = null;
}
}
msg = msg.Replace(wf.Search, wf.Replace);
}
}
return msg;
}
private ChatMessageEmoteUsage GetEmoteUsage(ChannelChatMessage message)
{
var emotesUsed = new HashSet<string>();
var newEmotes = new Dictionary<string, string>();
foreach (var fragment in message.Message.Fragments)
{
if (fragment.Emote != null)
{
if (_emotes.Get(fragment.Text) == null)
{
newEmotes.Add(fragment.Text, fragment.Emote.Id);
_emotes.Add(fragment.Text, fragment.Emote.Id);
}
emotesUsed.Add(fragment.Emote.Id);
continue;
}
if (fragment.Mention != null)
continue;
var text = fragment.Text.Trim();
var textFragments = text.Split(' ');
foreach (var f in textFragments)
{
var emoteId = _emotes.Get(f);
if (emoteId != null)
{
emotesUsed.Add(emoteId);
}
}
}
return new ChatMessageEmoteUsage(emotesUsed, newEmotes);
}
private int GetTotalBits(ChannelChatMessage message)
{
return message.Message.Fragments.Where(f => f.Type == "cheermote" && f.Cheermote != null)
.Select(f => f.Cheermote!.Bits)
.Sum();
}
private string GetGroupNameByBadgeName(string badgeName)
{
if (badgeName == "subscriber")
return "subscribers";
if (badgeName == "moderator")
return "moderators";
return badgeName.ToLower();
}
private IEnumerable<string> GetGroups(ChannelChatMessage message, long chatterId)
{
var defaultGroups = new string[] { "everyone" };
var badgesGroups = message.Badges.Select(b => b.SetId).Select(GetGroupNameByBadgeName);
var customGroups = _chatterGroupManager.GetGroupNamesFor(chatterId);
return defaultGroups.Union(badgesGroups).Union(customGroups);
}
private IEnumerable<TTSMessage> GetPartialTTSMessages(string message, string defaultVoice)
{
var matches = _user.VoiceNameRegex?.Matches(message).ToArray();
if (matches == null || matches.FirstOrDefault() == null || matches.First().Index < 0)
{
return [new TTSMessage()
{
Voice = defaultVoice,
Message = message
}];
}
return matches.Cast<Match>().SelectMany(match =>
{
var m = match.Groups["message"].Value;
if (string.IsNullOrWhiteSpace(m))
return [];
var voiceSelected = match.Groups["voice"].Value;
voiceSelected = voiceSelected[0].ToString().ToUpper() + voiceSelected.Substring(1).ToLower();
return HandlePartialMessage(voiceSelected, m);
});
}
private string GetSelectedVoiceFor(long chatterId)
{
string? voiceSelected = _user.DefaultTTSVoice;
if (_user.VoicesSelected?.ContainsKey(chatterId) == true)
{
var voiceId = _user.VoicesSelected[chatterId];
if (_user.VoicesAvailable.TryGetValue(voiceId, out string? voiceName) && voiceName != null)
{
if (_user.VoicesEnabled.Contains(voiceName) || chatterId == _user.OwnerId)
voiceSelected = voiceName;
}
}
return voiceSelected ?? "Brian";
}
private bool HasPermission(ChannelChatMessage message, long chatterId, IEnumerable<string> groups, int bits)
{
var permissionPath = "tts.chat.messages.read";
if (!string.IsNullOrWhiteSpace(message.ChannelPointsCustomRewardId))
permissionPath = "tts.chat.redemptions.read";
else if (bits > 0)
permissionPath = "tts.chat.bits.read";
return chatterId == _user.OwnerId ? true : _permissionManager.CheckIfAllowed(groups, permissionPath) == true;
}
private class ChatMessageEmoteUsage
{
public readonly HashSet<string> EmotesUsed = new HashSet<string>();
public readonly IDictionary<string, string> NewEmotes = new Dictionary<string, string>();
public ChatMessageEmoteUsage(HashSet<string> emotesUsed, IDictionary<string, string> newEmotes)
{
EmotesUsed = emotesUsed;
NewEmotes = newEmotes;
}
}
}
}

336
Chat/Speech/TTSPlayer.cs
View File

@ -1,198 +1,218 @@
using NAudio.Wave;
using TwitchChatTTS.Twitch.Socket.Messages;
public class TTSPlayer
namespace TwitchChatTTS.Chat.Soeech
{
private readonly PriorityQueue<TTSMessage, int> _messages; // ready to play
private readonly PriorityQueue<TTSMessage, int> _buffer;
private readonly Mutex _mutex;
private readonly Mutex _mutex2;
public TTSMessage? Playing { get; set; }
public TTSPlayer()
public class TTSPlayer
{
_messages = new PriorityQueue<TTSMessage, int>(new DescendingOrder());
_buffer = new PriorityQueue<TTSMessage, int>(new DescendingOrder());
_mutex = new Mutex();
_mutex2 = new Mutex();
}
private readonly PriorityQueue<TTSGroupedMessage, int> _messages; // ready to play
private readonly PriorityQueue<TTSGroupedMessage, int> _buffer;
private readonly Mutex _mutex;
private readonly Mutex _mutex2;
public void Add(TTSMessage message)
{
try
//public TTSGroupedMessage? PlayingGroup { get; set; }
public TTSGroupedMessage? Playing { get; set; }
public TTSPlayer()
{
_mutex2.WaitOne();
_buffer.Enqueue(message, message.Priority);
_messages = new PriorityQueue<TTSGroupedMessage, int>(new DescendingOrder());
_buffer = new PriorityQueue<TTSGroupedMessage, int>(new DescendingOrder());
_mutex = new Mutex();
_mutex2 = new Mutex();
}
finally
{
_mutex2.ReleaseMutex();
}
}
public TTSMessage? ReceiveReady()
{
try
public void Add(TTSGroupedMessage message, int priority)
{
_mutex.WaitOne();
if (_messages.TryDequeue(out TTSMessage? message, out int _))
try
{
return message;
_mutex2.WaitOne();
_buffer.Enqueue(message, priority);
}
return null;
}
finally
{
_mutex.ReleaseMutex();
}
}
public TTSMessage? ReceiveBuffer()
{
try
{
_mutex2.WaitOne();
if (_buffer.TryDequeue(out TTSMessage? message, out int _))
finally
{
return message;
_mutex2.ReleaseMutex();
}
return null;
}
finally
{
_mutex2.ReleaseMutex();
}
}
public void Ready(TTSMessage message)
{
try
{
_mutex.WaitOne();
_messages.Enqueue(message, message.Priority);
}
finally
{
_mutex.ReleaseMutex();
}
}
public void RemoveAll()
{
try
{
_mutex2.WaitOne();
_buffer.Clear();
}
finally
{
_mutex2.ReleaseMutex();
}
try
public TTSGroupedMessage? ReceiveReady()
{
_mutex.WaitOne();
_messages.Clear();
}
finally
{
_mutex.ReleaseMutex();
}
}
public void RemoveAll(long broadcasterId, long chatterId)
{
try
{
_mutex2.WaitOne();
if (_buffer.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId))
try
{
var list = _buffer.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray();
_mutex.WaitOne();
if (_messages.TryDequeue(out TTSGroupedMessage? message, out int _))
{
return message;
}
return null;
}
finally
{
_mutex.ReleaseMutex();
}
}
public TTSGroupedMessage? ReceiveBuffer()
{
try
{
_mutex2.WaitOne();
if (_buffer.TryDequeue(out TTSGroupedMessage? messages, out int _))
{
return messages;
}
return null;
}
finally
{
_mutex2.ReleaseMutex();
}
}
public void Ready(TTSGroupedMessage message)
{
try
{
_mutex.WaitOne();
_messages.Enqueue(message, message.Priority);
}
finally
{
_mutex.ReleaseMutex();
}
}
public void RemoveAll()
{
try
{
_mutex2.WaitOne();
_buffer.Clear();
foreach (var item in list)
_buffer.Enqueue(item.Element, item.Element.Priority);
}
}
finally
{
_mutex2.ReleaseMutex();
}
try
{
_mutex.WaitOne();
if (_messages.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId))
finally
{
var list = _messages.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray();
_mutex2.ReleaseMutex();
}
try
{
_mutex.WaitOne();
_messages.Clear();
foreach (var item in list)
_messages.Enqueue(item.Element, item.Element.Priority);
}
}
finally
{
_mutex.ReleaseMutex();
}
}
public void RemoveMessage(string messageId)
{
try
{
_mutex2.WaitOne();
if (_buffer.UnorderedItems.Any(i => i.Element.MessageId == messageId))
finally
{
var list = _buffer.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray();
_buffer.Clear();
foreach (var item in list)
_buffer.Enqueue(item.Element, item.Element.Priority);
return;
_mutex.ReleaseMutex();
}
}
finally
{
_mutex2.ReleaseMutex();
}
try
public void RemoveAll(long broadcasterId, long chatterId)
{
_mutex.WaitOne();
if (_messages.UnorderedItems.Any(i => i.Element.MessageId == messageId))
try
{
var list = _messages.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray();
_messages.Clear();
foreach (var item in list)
_messages.Enqueue(item.Element, item.Element.Priority);
_mutex2.WaitOne();
if (_buffer.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId))
{
var list = _buffer.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray();
_buffer.Clear();
foreach (var item in list)
_buffer.Enqueue(item.Element, item.Element.Priority);
}
}
finally
{
_mutex2.ReleaseMutex();
}
try
{
_mutex.WaitOne();
if (_messages.UnorderedItems.Any(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId == chatterId))
{
var list = _messages.UnorderedItems.Where(i => i.Element.RoomId == broadcasterId && i.Element.ChatterId != chatterId).ToArray();
_messages.Clear();
foreach (var item in list)
_messages.Enqueue(item.Element, item.Element.Priority);
}
}
finally
{
_mutex.ReleaseMutex();
}
}
finally
public void RemoveMessage(string messageId)
{
_mutex.ReleaseMutex();
try
{
_mutex2.WaitOne();
if (_buffer.UnorderedItems.Any(i => i.Element.MessageId == messageId))
{
var list = _buffer.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray();
_buffer.Clear();
foreach (var item in list)
_buffer.Enqueue(item.Element, item.Element.Priority);
return;
}
}
finally
{
_mutex2.ReleaseMutex();
}
try
{
_mutex.WaitOne();
if (_messages.UnorderedItems.Any(i => i.Element.MessageId == messageId))
{
var list = _messages.UnorderedItems.Where(i => i.Element.MessageId != messageId).ToArray();
_messages.Clear();
foreach (var item in list)
_messages.Enqueue(item.Element, item.Element.Priority);
}
}
finally
{
_mutex.ReleaseMutex();
}
}
public bool IsEmpty()
{
return _messages.Count == 0;
}
private class DescendingOrder : IComparer<int>
{
public int Compare(int x, int y) => y.CompareTo(x);
}
}
public bool IsEmpty()
public class TTSMessage
{
return _messages.Count == 0;
public string? Voice { get; set; }
public string? Message { get; set; }
public string? File { get; set; }
}
private class DescendingOrder : IComparer<int>
public class TTSGroupedMessage
{
public int Compare(int x, int y) => y.CompareTo(x);
}
}
public long RoomId { get; set; }
public long ChatterId { get; set; }
public string MessageId { get; set; }
public DateTime Timestamp { get; set; }
public int Priority { get; set; }
public IList<TTSMessage> Messages { get; set; }
//public IList<ISampleProvider> Audios { get; set; }
public ISampleProvider? Audio { get; set; }
public class TTSMessage
{
public string? Voice { get; set; }
public long RoomId { get; set; }
public long ChatterId { get; set; }
public string MessageId { get; set; }
public string? Message { get; set; }
public string? File { get; set; }
public DateTime Timestamp { get; set; }
public IEnumerable<TwitchBadge> Badges { get; set; }
public int Priority { get; set; }
public ISampleProvider? Audio { get; set; }
public TTSGroupedMessage(long broadcasterId, long chatterId, string messageId, IList<TTSMessage> messages, DateTime timestamp, int priority)
{
RoomId = broadcasterId;
ChatterId = chatterId;
MessageId = messageId;
Messages = messages;
Timestamp = timestamp;
Priority = priority;
//Audios = new List<ISampleProvider>();
}
}
}

67
Hermes/Socket/HermesSocketClient.cs
View File

@ -27,6 +27,7 @@ namespace TwitchChatTTS.Hermes.Socket
public string? UserId { get; set; }
private readonly System.Timers.Timer _heartbeatTimer;
private readonly IBackoff _backoff;
private readonly object _lock;
public bool Connected { get; set; }
public bool LoggedIn { get; set; }
@ -56,13 +57,18 @@ namespace TwitchChatTTS.Hermes.Socket
LastHeartbeatReceived = LastHeartbeatSent = DateTime.UtcNow;
URL = $"wss://{BASE_URL}";
_lock = new object();
}
public async Task Connect()
{
if (Connected)
return;
lock (_lock)
{
if (Connected)
return;
}
_logger.Debug($"Attempting to connect to {URL}");
await ConnectAsync(URL);
@ -70,8 +76,11 @@ namespace TwitchChatTTS.Hermes.Socket
private async Task Disconnect()
{
if (!Connected)
return;
lock (_lock)
{
if (!Connected)
return;
}
await DisconnectAsync(new SocketDisconnectionEventArgs("Normal disconnection", "Disconnection was executed"));
}
@ -212,7 +221,12 @@ namespace TwitchChatTTS.Hermes.Socket
OnConnected += async (sender, e) =>
{
Connected = true;
lock (_lock)
{
if (Connected)
return;
Connected = true;
}
_logger.Information("Hermes websocket client connected.");
_heartbeatTimer.Enabled = true;
@ -228,7 +242,13 @@ namespace TwitchChatTTS.Hermes.Socket
OnDisconnected += async (sender, e) =>
{
Connected = false;
lock (_lock)
{
if (!Connected)
return;
Connected = false;
}
LoggedIn = false;
Ready = false;
_logger.Warning("Hermes websocket client disconnected.");
@ -334,7 +354,7 @@ namespace TwitchChatTTS.Hermes.Socket
{
var signalTime = e.SignalTime.ToUniversalTime();
if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(30))
if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(60))
{
if (LastHeartbeatReceived > LastHeartbeatSent)
{
@ -349,7 +369,7 @@ namespace TwitchChatTTS.Hermes.Socket
_logger.Error(ex, "Failed to send a heartbeat back to the Hermes websocket server.");
}
}
else if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(60))
else if (signalTime - LastHeartbeatReceived > TimeSpan.FromSeconds(120))
{
try
{
@ -369,37 +389,6 @@ namespace TwitchChatTTS.Hermes.Socket
}
}
private async Task Reconnect(ElapsedEventArgs e)
{
if (Connected)
{
try
{
await Disconnect();
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to disconnect from Hermes websocket server.");
Ready = false;
LoggedIn = false;
Connected = false;
}
}
try
{
await Connect();
}
catch (WebSocketException wse) when (wse.Message.Contains("502"))
{
_logger.Error($"Hermes websocket server cannot be found [code: {wse.ErrorCode}]");
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to reconnect to Hermes websocket server.");
}
}
public new async Task Send<T>(int opcode, T message)
{
if (!Connected)

4
Startup.cs
View File

@ -29,6 +29,8 @@ using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Twitch.Socket.Messages;
using TwitchChatTTS.Twitch.Socket.Handlers;
using CommonSocketLibrary.Backoff;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Chat.Messaging;
// dotnet publish -r linux-x64 -p:PublishSingleFile=true --self-contained true
// dotnet publish -r win-x64 -p:PublishSingleFile=true --self-contained true
@ -112,6 +114,8 @@ s.AddKeyedSingleton<SocketClient<WebSocketMessage>, SevenSocketClient>("7tv");
// Nightbot
s.AddSingleton<NightbotApiClient>();
s.AddSingleton<ChatMessageReader>();
// twitch websocket
s.AddKeyedSingleton<IBackoff>("twitch", new ExponentialBackoff(1000, 120 * 1000));
s.AddSingleton<ITwitchConnectionManager, TwitchConnectionManager>();

68
TTS.cs
View File

@ -15,6 +15,8 @@ using TwitchChatTTS.Twitch.Socket.Messages;
using TwitchChatTTS.Twitch.Socket;
using TwitchChatTTS.Chat.Commands;
using System.Text;
using TwitchChatTTS.Chat.Soeech;
using NAudio.Wave;
namespace TwitchChatTTS
{
@ -26,7 +28,6 @@ namespace TwitchChatTTS
private readonly User _user;
private readonly HermesApiClient _hermesApiClient;
private readonly SevenApiClient _sevenApiClient;
private readonly TwitchApiClient _twitchApiClient;
private readonly HermesSocketClient _hermes;
private readonly OBSSocketClient _obs;
private readonly SevenSocketClient _seven;
@ -149,7 +150,7 @@ namespace TwitchChatTTS
_playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) =>
{
if (e.SampleProvider == _player.Playing?.Audio)
if (_player.Playing?.Audio == e.SampleProvider)
{
_player.Playing = null;
}
@ -167,22 +168,46 @@ namespace TwitchChatTTS
return;
}
var m = _player.ReceiveBuffer();
if (m == null)
var group = _player.ReceiveBuffer();
if (group == null)
{
await Task.Delay(200);
continue;
}
string url = $"https://api.streamelements.com/kappa/v2/speech?voice={m.Voice}&text={HttpUtility.UrlEncode(m.Message)}";
var sound = new NetworkWavSound(url);
var provider = new CachedWavProvider(sound);
var data = _playback.ConvertSound(provider);
var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate);
_logger.Verbose("Fetched TTS audio data.");
Task.Run(() =>
{
var list = new List<ISampleProvider>();
foreach (var message in group.Messages)
{
if (string.IsNullOrEmpty(message.Message))
{
using (var reader2 = new AudioFileReader(message.File))
{
list.Add(_playback.ConvertSound(reader2.ToWaveProvider()));
}
continue;
}
m.Audio = resampled;
_player.Ready(m);
try
{
string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}";
var nws = new NetworkWavSound(url);
var provider = new CachedWavProvider(nws);
var data = _playback.ConvertSound(provider);
var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate);
list.Add(resampled);
}
catch (Exception e)
{
_logger.Error(e, "Failed to fetch TTS message for ");
}
}
var merged = new ConcatenatingSampleProvider(list);
group.Audio = merged;
_player.Ready(group);
});
}
catch (COMException e)
{
@ -211,25 +236,18 @@ namespace TwitchChatTTS
await Task.Delay(200);
continue;
}
var m = _player.ReceiveReady();
if (m == null)
var g = _player.ReceiveReady();
if (g == null)
{
continue;
}
if (!string.IsNullOrWhiteSpace(m.File) && File.Exists(m.File))
{
_logger.Debug("Playing audio file via TTS: " + m.File);
_playback.PlaySound(m.File);
continue;
}
var audio = g.Audio;
_logger.Debug("Playing message via TTS: " + m.Message);
if (m.Audio != null)
if (audio != null)
{
_player.Playing = m;
_playback.AddMixerInput(m.Audio);
_player.Playing = g;
_playback.AddMixerInput(audio);
}
}
catch (Exception e)

1
Twitch/Socket/Handlers/ChannelChatClearHandler.cs
View File

@ -1,4 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers

1
Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs
View File

@ -1,4 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers

3
Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs
View File

@ -1,4 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Soeech;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers
@ -25,7 +26,7 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers
if (_player.Playing?.MessageId == message.MessageId)
{
_playback.RemoveMixerInput(_player.Playing.Audio!);
_playback.RemoveMixerInput(_player.Playing!.Audio!);
_player.Playing = null;
}
else

304
Twitch/Socket/Handlers/ChannelChatMessageHandler.cs
View File

@ -1,14 +1,5 @@
using System.Text.RegularExpressions;
using CommonSocketLibrary.Abstract;
using CommonSocketLibrary.Common;
using Microsoft.Extensions.DependencyInjection;
using Serilog;
using TwitchChatTTS.Chat.Commands;
using TwitchChatTTS.Chat.Emotes;
using TwitchChatTTS.Chat.Groups;
using TwitchChatTTS.Chat.Groups.Permissions;
using TwitchChatTTS.Hermes.Socket;
using TwitchChatTTS.OBS.Socket;
using TwitchChatTTS.Chat.Messaging;
using TwitchChatTTS.Twitch.Socket.Messages;
namespace TwitchChatTTS.Twitch.Socket.Handlers
@ -17,48 +8,23 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers
{
public string Name => "channel.chat.message";
private readonly User _user;
private readonly TTSPlayer _player;
private readonly ICommandManager _commands;
private readonly IGroupPermissionManager _permissionManager;
private readonly IChatterGroupManager _chatterGroupManager;
private readonly IEmoteDatabase _emotes;
private readonly OBSSocketClient _obs;
private readonly HermesSocketClient _hermes;
private readonly ChatMessageReader _reader;
private readonly Configuration _configuration;
private readonly ILogger _logger;
private readonly Regex _sfxRegex;
public ChannelChatMessageHandler(
User user,
TTSPlayer player,
ICommandManager commands,
IGroupPermissionManager permissionManager,
IChatterGroupManager chatterGroupManager,
IEmoteDatabase emotes,
[FromKeyedServices("hermes")] SocketClient<WebSocketMessage> hermes,
[FromKeyedServices("obs")] SocketClient<WebSocketMessage> obs,
ChatMessageReader reader,
Configuration configuration,
ILogger logger
)
{
_user = user;
_player = player;
_commands = commands;
_permissionManager = permissionManager;
_chatterGroupManager = chatterGroupManager;
_emotes = emotes;
_obs = (obs as OBSSocketClient)!;
_hermes = (hermes as HermesSocketClient)!;
_reader = reader;
_configuration = configuration;
_logger = logger;
_sfxRegex = new Regex(@"\(([A-Za-z0-9_-]+)\)", RegexOptions.Compiled);
_logger = logger;
}
public async Task Execute(TwitchWebsocketClient sender, object data)
{
if (sender == null)
@ -66,265 +32,7 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers
if (data is not ChannelChatMessage message)
return;
if (_hermes.Connected && !_hermes.Ready)
{
_logger.Debug($"TTS is not yet ready. Ignoring chat messages [message id: {message.MessageId}]");
return; // new MessageResult(MessageStatus.NotReady, -1, -1);
}
if (_configuration.Twitch?.TtsWhenOffline != true && !_obs.Streaming)
{
_logger.Debug($"OBS is not streaming. Ignoring chat messages [message id: {message.MessageId}]");
return; // new MessageResult(MessageStatus.NotReady, -1, -1);
}
var msg = string.Join(string.Empty, message.Message.Fragments.Where(f => f.Type != "cheermote").Select(f => f.Text)).Trim();
var chatterId = long.Parse(message.ChatterUserId);
var tasks = new List<Task>();
var defaultGroups = new string[] { "everyone" };
var badgesGroups = message.Badges.Select(b => b.SetId).Select(GetGroupNameByBadgeName);
var customGroups = _chatterGroupManager.GetGroupNamesFor(chatterId);
var groups = defaultGroups.Union(badgesGroups).Union(customGroups);
try
{
var commandResult = await _commands.Execute(msg, message, groups);
if (commandResult != ChatCommandResult.Unknown)
return; // new MessageResult(MessageStatus.Command, -1, -1);
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed executing a chat command [message: {msg}][chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}][message id: {message.MessageId}]");
return;
}
if (message.Reply != null)
msg = msg.Substring(message.Reply.ParentUserLogin.Length + 2);
var bits = message.Message.Fragments.Where(f => f.Type == "cheermote" && f.Cheermote != null)
.Select(f => f.Cheermote!.Bits)
.Sum();
var permissionPath = "tts.chat.messages.read";
if (!string.IsNullOrWhiteSpace(message.ChannelPointsCustomRewardId))
permissionPath = "tts.chat.redemptions.read";
else if (bits > 0)
permissionPath = "tts.chat.bits.read";
var permission = chatterId == _user.OwnerId ? true : _permissionManager.CheckIfAllowed(groups, permissionPath);
if (permission != true)
{
_logger.Debug($"Blocked message by {message.ChatterUserLogin}: {msg}");
return; // new MessageResult(MessageStatus.Blocked, -1, -1);
}
// Keep track of emotes usage
var emotesUsed = new HashSet<string>();
var newEmotes = new Dictionary<string, string>();
foreach (var fragment in message.Message.Fragments)
{
if (fragment.Emote != null)
{
if (_emotes.Get(fragment.Text) == null)
{
newEmotes.Add(fragment.Text, fragment.Emote.Id);
_emotes.Add(fragment.Text, fragment.Emote.Id);
}
emotesUsed.Add(fragment.Emote.Id);
continue;
}
if (fragment.Mention != null)
continue;
var text = fragment.Text.Trim();
var textFragments = text.Split(' ');
foreach (var f in textFragments)
{
var emoteId = _emotes.Get(f);
if (emoteId != null)
{
emotesUsed.Add(emoteId);
}
}
}
if (_obs.Streaming)
{
if (newEmotes.Any())
tasks.Add(_hermes.SendEmoteDetails(newEmotes));
if (emotesUsed.Any())
tasks.Add(_hermes.SendEmoteUsage(message.MessageId, chatterId, emotesUsed));
if (!_user.Chatters.Contains(chatterId))
{
tasks.Add(_hermes.SendChatterDetails(chatterId, message.ChatterUserLogin));
_user.Chatters.Add(chatterId);
}
}
if (_user.Raids.TryGetValue(message.BroadcasterUserId, out var raid) && !raid.Chatters.Contains(chatterId))
{
_logger.Information($"Potential chat message from raider ignored due to potential raid message spam [chatter: {message.ChatterUserLogin}][chatter id: {message.ChatterUserId}]");
return;
}
// Replace filtered words.
if (_user.RegexFilters != null)
{
foreach (var wf in _user.RegexFilters)
{
if (wf.Search == null || wf.Replace == null)
continue;
if (wf.Regex != null)
{
try
{
msg = wf.Regex.Replace(msg, wf.Replace);
continue;
}
catch (Exception)
{
wf.Regex = null;
}
}
msg = msg.Replace(wf.Search, wf.Replace);
}
}
// Determine the priority of this message
int priority = _chatterGroupManager.GetPriorityFor(groups); // + m.SubscribedMonthCount * (m.IsSubscriber ? 10 : 5);
// Determine voice selected.
string voiceSelected = _user.DefaultTTSVoice;
if (_user.VoicesSelected?.ContainsKey(chatterId) == true)
{
var voiceId = _user.VoicesSelected[chatterId];
if (_user.VoicesAvailable.TryGetValue(voiceId, out string? voiceName) && voiceName != null)
{
if (_user.VoicesEnabled.Contains(voiceName) || chatterId == _user.OwnerId)
voiceSelected = voiceName;
}
}
// Determine additional voices used
var matches = _user.VoiceNameRegex?.Matches(msg).ToArray();
if (matches == null || matches.FirstOrDefault() == null || matches.First().Index < 0)
{
HandlePartialMessage(priority, voiceSelected, msg.Trim(), message);
return; // new MessageResult(MessageStatus.None, _user.TwitchUserId, chatterId, emotesUsed);
}
HandlePartialMessage(priority, voiceSelected, msg.Substring(0, matches.First().Index).Trim(), message);
foreach (Match match in matches)
{
var m = match.Groups[2].ToString();
if (string.IsNullOrWhiteSpace(m))
continue;
var voice = match.Groups[1].ToString();
voice = voice[0].ToString().ToUpper() + voice.Substring(1).ToLower();
HandlePartialMessage(priority, voice, m.Trim(), message);
}
if (tasks.Any())
await Task.WhenAll(tasks);
}
private void HandlePartialMessage(int priority, string voice, string message, ChannelChatMessage e)
{
if (string.IsNullOrWhiteSpace(message))
return;
var parts = _sfxRegex.Split(message);
var chatterId = long.Parse(e.ChatterUserId);
var broadcasterId = long.Parse(e.BroadcasterUserId);
var badgesString = string.Join(", ", e.Badges.Select(b => b.SetId + '|' + b.Id + '=' + b.Info));
if (parts.Length == 1)
{
_logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {message}; Reward Id: {e.ChannelPointsCustomRewardId}; {badgesString}");
_player.Add(new TTSMessage()
{
Voice = voice,
Message = message,
Timestamp = DateTime.UtcNow,
RoomId = broadcasterId,
ChatterId = chatterId,
MessageId = e.MessageId,
Badges = e.Badges,
Priority = priority
});
return;
}
var sfxMatches = _sfxRegex.Matches(message);
var sfxStart = sfxMatches.FirstOrDefault()?.Index ?? message.Length;
for (var i = 0; i < sfxMatches.Count; i++)
{
var sfxMatch = sfxMatches[i];
var sfxName = sfxMatch.Groups[1]?.ToString()?.ToLower();
if (!File.Exists("sfx/" + sfxName + ".mp3"))
{
parts[i * 2 + 2] = parts[i * 2] + " (" + parts[i * 2 + 1] + ")" + parts[i * 2 + 2];
continue;
}
if (!string.IsNullOrWhiteSpace(parts[i * 2]))
{
_logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {parts[i * 2]}; {badgesString}");
_player.Add(new TTSMessage()
{
Voice = voice,
Message = parts[i * 2],
Timestamp = DateTime.UtcNow,
RoomId = broadcasterId,
ChatterId = chatterId,
MessageId = e.MessageId,
Badges = e.Badges,
Priority = priority
});
}
_logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; SFX: {sfxName}; {badgesString}");
_player.Add(new TTSMessage()
{
Voice = voice,
File = $"sfx/{sfxName}.mp3",
Timestamp = DateTime.UtcNow,
RoomId = broadcasterId,
ChatterId = chatterId,
MessageId = e.MessageId,
Badges = e.Badges,
Priority = priority
});
}
if (!string.IsNullOrWhiteSpace(parts.Last()))
{
_logger.Information($"Username: {e.ChatterUserLogin}; User ID: {e.ChatterUserId}; Voice: {voice}; Priority: {priority}; Message: {parts.Last()}; {badgesString}");
_player.Add(new TTSMessage()
{
Voice = voice,
Message = parts.Last(),
Timestamp = DateTime.UtcNow,
RoomId = broadcasterId,
ChatterId = chatterId,
MessageId = e.MessageId,
Badges = e.Badges,
Priority = priority
});
}
}
private string GetGroupNameByBadgeName(string badgeName)
{
if (badgeName == "subscriber")
return "subscribers";
if (badgeName == "moderator")
return "moderators";
return badgeName.ToLower();
await _reader.Execute(sender, message);
}
}
}

5
Twitch/Socket/Handlers/ChannelResubscriptionHandler.cs
View File

@ -1,4 +1,5 @@
using Serilog;
using TwitchChatTTS.Chat.Messaging;
using TwitchChatTTS.Twitch.Redemptions;
using TwitchChatTTS.Twitch.Socket.Messages;
@ -8,11 +9,13 @@ namespace TwitchChatTTS.Twitch.Socket.Handlers
{
public string Name => "channel.subscription.message";
private readonly ChatMessageReader _reader;
private readonly IRedemptionManager _redemptionManager;
private readonly ILogger _logger;
public ChannelResubscriptionHandler(IRedemptionManager redemptionManager, ILogger logger)
public ChannelResubscriptionHandler(ChatMessageReader reader, IRedemptionManager redemptionManager, ILogger logger)
{
_reader = reader;
_redemptionManager = redemptionManager;
_logger = logger;
}

2
User.cs
View File

@ -43,7 +43,7 @@ namespace TwitchChatTTS
return null;
var enabledVoicesString = string.Join("|", VoicesAvailable.Where(v => VoicesEnabled == null || !VoicesEnabled.Any() || VoicesEnabled.Contains(v.Value)).Select(v => v.Value));
return new Regex($@"\b({enabledVoicesString})\:(.*?)(?=\Z|\b(?:{enabledVoicesString})\:)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
return new Regex($@"(?:\A|\s)(?<voice>{enabledVoicesString}):(?<message>.*?)(?=\Z|\s(?:{enabledVoicesString})\:)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
}
}
}