using System.Runtime.InteropServices; using System.Web; using CommonSocketLibrary.Abstract; using CommonSocketLibrary.Common; using HermesSocketLibrary.Socket.Data; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; using NAudio.Wave.SampleProviders; using TwitchChatTTS.Seven; using TwitchLib.Client.Events; using TwitchChatTTS.Twitch.Redemptions; using org.mariuszgromada.math.mxparser; using TwitchChatTTS.Hermes.Socket; using TwitchChatTTS.Chat.Groups.Permissions; using TwitchChatTTS.Chat.Groups; namespace TwitchChatTTS { public class TTS : IHostedService { public const int MAJOR_VERSION = 3; public const int MINOR_VERSION = 8; private readonly User _user; private readonly HermesApiClient _hermesApiClient; private readonly SevenApiClient _sevenApiClient; private readonly RedemptionManager _redemptionManager; private readonly IChatterGroupManager _chatterGroupManager; private readonly IGroupPermissionManager _permissionManager; private readonly Configuration _configuration; private readonly TTSPlayer _player; private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; public TTS( User user, HermesApiClient hermesApiClient, SevenApiClient sevenApiClient, RedemptionManager redemptionManager, IChatterGroupManager chatterGroupManager, IGroupPermissionManager permissionManager, Configuration configuration, TTSPlayer player, IServiceProvider serviceProvider, ILogger logger ) { _user = user; _hermesApiClient = hermesApiClient; _sevenApiClient = sevenApiClient; _redemptionManager = redemptionManager; _chatterGroupManager = chatterGroupManager; _permissionManager = permissionManager; _configuration = configuration; _player = player; _serviceProvider = serviceProvider; _logger = logger; } public async Task StartAsync(CancellationToken cancellationToken) { Console.Title = "TTS - Twitch Chat"; License.iConfirmCommercialUse("abcdef"); if (string.IsNullOrWhiteSpace(_configuration.Hermes.Token)) { _logger.Error("Hermes API token not set in the configuration file."); return; } var hermesVersion = await _hermesApiClient.GetLatestTTSVersion(); if (hermesVersion == null) { _logger.Warning("Failed to fetch latest TTS version. Skipping version check."); } else if (hermesVersion.MajorVersion > TTS.MAJOR_VERSION || hermesVersion.MajorVersion == TTS.MAJOR_VERSION && hermesVersion.MinorVersion > TTS.MINOR_VERSION) { _logger.Information($"A new update for TTS is avaiable! Version {hermesVersion.MajorVersion}.{hermesVersion.MinorVersion} is available at {hermesVersion.Download}"); var changes = hermesVersion.Changelog.Split("\n"); if (changes != null && changes.Any()) _logger.Information("Changelogs:\n - " + string.Join("\n - ", changes) + "\n\n"); await Task.Delay(15 * 1000); } try { await FetchUserData(_user, _hermesApiClient, _sevenApiClient); } catch (Exception ex) { _logger.Error(ex, "Failed to initialize properly."); await Task.Delay(30 * 1000); } var twitchapiclient = await InitializeTwitchApiClient(_user.TwitchUsername, _user.TwitchUserId.ToString()); if (twitchapiclient == null) { await Task.Delay(30 * 1000); return; } var emoteSet = await _sevenApiClient.FetchChannelEmoteSet(_user.TwitchUserId.ToString()); _user.SevenEmoteSetId = emoteSet.Id; await InitializeEmotes(_sevenApiClient, emoteSet); await InitializeHermesWebsocket(); await InitializeSevenTv(); await InitializeObs(); AudioPlaybackEngine.Instance.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) => { if (e.SampleProvider == _player.Playing) { _player.Playing = null; } }); Task.Run(async () => { while (true) { try { if (cancellationToken.IsCancellationRequested) { _logger.Warning("TTS Buffer - Cancellation requested."); return; } var m = _player.ReceiveBuffer(); if (m == null) { await Task.Delay(200); continue; } string url = $"https://api.streamelements.com/kappa/v2/speech?voice={m.Voice}&text={HttpUtility.UrlEncode(m.Message)}"; var sound = new NetworkWavSound(url); var provider = new CachedWavProvider(sound); var data = AudioPlaybackEngine.Instance.ConvertSound(provider); var resampled = new WdlResamplingSampleProvider(data, AudioPlaybackEngine.Instance.SampleRate); _logger.Verbose("Fetched TTS audio data."); m.Audio = resampled; _player.Ready(m); } catch (COMException e) { _logger.Error(e, "Failed to send request for TTS [HResult: " + e.HResult + "]"); } catch (Exception e) { _logger.Error(e, "Failed to send request for TTS."); } } }); Task.Run(async () => { while (true) { try { if (cancellationToken.IsCancellationRequested) { _logger.Warning("TTS Queue - Cancellation requested."); return; } while (_player.IsEmpty() || _player.Playing != null) { await Task.Delay(200); continue; } var m = _player.ReceiveReady(); if (m == null) { continue; } if (!string.IsNullOrWhiteSpace(m.File) && File.Exists(m.File)) { _logger.Debug("Playing audio file via TTS: " + m.File); AudioPlaybackEngine.Instance.PlaySound(m.File); continue; } _logger.Debug("Playing message via TTS: " + m.Message); if (m.Audio != null) { _player.Playing = m.Audio; AudioPlaybackEngine.Instance.AddMixerInput(m.Audio); } } catch (Exception e) { _logger.Error(e, "Failed to play a TTS audio message"); } } }); _logger.Information("Twitch websocket client connecting..."); await twitchapiclient.Connect(); } public async Task StopAsync(CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) _logger.Warning("Application has stopped due to cancellation token."); else _logger.Warning("Application has stopped."); } private async Task FetchUserData(User user, HermesApiClient hermes, SevenApiClient seven) { var hermesAccount = await hermes.FetchHermesAccountDetails(); if (hermesAccount == null) throw new Exception("Cannot connect to Hermes. Ensure your token is valid."); user.HermesUserId = hermesAccount.Id; user.HermesUsername = hermesAccount.Username; user.TwitchUsername = hermesAccount.Username; var twitchBotToken = await hermes.FetchTwitchBotToken(); user.TwitchUserId = long.Parse(twitchBotToken.BroadcasterId); _logger.Information($"Username: {user.TwitchUsername} [id: {user.TwitchUserId}]"); user.DefaultTTSVoice = await hermes.FetchTTSDefaultVoice(); _logger.Information("TTS Default Voice: " + user.DefaultTTSVoice); var wordFilters = await hermes.FetchTTSWordFilters(); user.RegexFilters = wordFilters.ToList(); _logger.Information($"{user.RegexFilters.Count()} TTS word filters."); var usernameFilters = await hermes.FetchTTSUsernameFilters(); user.ChatterFilters = usernameFilters.ToDictionary(e => e.Username, e => e); _logger.Information($"{user.ChatterFilters.Where(f => f.Value.Tag == "blacklisted").Count()} username(s) have been blocked."); _logger.Information($"{user.ChatterFilters.Where(f => f.Value.Tag == "priority").Count()} user(s) have been prioritized."); var voicesSelected = await hermes.FetchTTSChatterSelectedVoices(); user.VoicesSelected = voicesSelected.ToDictionary(s => s.ChatterId, s => s.Voice); _logger.Information($"{user.VoicesSelected.Count} TTS voices have been selected for specific chatters."); var voicesEnabled = await hermes.FetchTTSEnabledVoices(); if (voicesEnabled == null || !voicesEnabled.Any()) user.VoicesEnabled = new HashSet(["Brian"]); else user.VoicesEnabled = new HashSet(voicesEnabled.Select(v => v)); _logger.Information($"{user.VoicesEnabled.Count} TTS voices have been enabled."); var defaultedChatters = voicesSelected.Where(item => item.Voice == null || !user.VoicesEnabled.Contains(item.Voice)); if (defaultedChatters.Any()) _logger.Information($"{defaultedChatters.Count()} chatter(s) will have their TTS voice set to default due to having selected a disabled TTS voice."); var redemptionActions = await hermes.FetchRedeemableActions(); var redemptions = await hermes.FetchRedemptions(); _redemptionManager.Initialize(redemptions, redemptionActions.ToDictionary(a => a.Name, a => a)); _logger.Information($"Redemption Manager has been initialized with {redemptionActions.Count()} actions & {redemptions.Count()} redemptions."); _chatterGroupManager.Clear(); _permissionManager.Clear(); var groups = await hermes.FetchGroups(); var groupsById = groups.ToDictionary(g => g.Id, g => g); foreach (var group in groups) _chatterGroupManager.Add(group); _logger.Information($"{groups.Count()} groups have been loaded."); var groupChatters = await hermes.FetchGroupChatters(); _logger.Debug($"{groupChatters.Count()} group users have been fetched."); var permissions = await hermes.FetchGroupPermissions(); foreach (var permission in permissions) { _logger.Debug($"Adding group permission [permission id: {permission.Id}][group id: {permission.GroupId}][path: {permission.Path}][allow: {permission.Allow?.ToString() ?? "null"}]"); if (!groupsById.TryGetValue(permission.GroupId, out var group)) { _logger.Warning($"Failed to find group by id [permission id: {permission.Id}][group id: {permission.GroupId}][path: {permission.Path}]"); continue; } var path = $"{group.Name}.{permission.Path}"; _permissionManager.Set(path, permission.Allow); _logger.Debug($"Added group permission [id: {permission.Id}][group id: {permission.GroupId}][path: {permission.Path}]"); } _logger.Information($"{permissions.Count()} group permissions have been loaded."); foreach (var chatter in groupChatters) if (groupsById.TryGetValue(chatter.GroupId, out var group)) _chatterGroupManager.Add(chatter.ChatterId, group.Name); _logger.Information($"Users in each group have been loaded."); } private async Task InitializeHermesWebsocket() { try { _logger.Information("Initializing hermes websocket client."); var hermesClient = _serviceProvider.GetRequiredKeyedService>("hermes"); var url = $"wss://{HermesSocketClient.BASE_URL}"; _logger.Debug($"Attempting to connect to {url}"); await hermesClient.ConnectAsync(url); hermesClient.Connected = true; await hermesClient.Send(1, new HermesLoginMessage() { ApiKey = _configuration.Hermes!.Token!, MajorVersion = TTS.MAJOR_VERSION, MinorVersion = TTS.MINOR_VERSION, }); } catch (Exception) { _logger.Warning("Connecting to hermes failed. Skipping hermes websockets."); } } private async Task InitializeSevenTv() { try { _logger.Information("Initializing 7tv websocket client."); var sevenClient = _serviceProvider.GetRequiredKeyedService>("7tv"); if (string.IsNullOrWhiteSpace(_user.SevenEmoteSetId)) { _logger.Warning("Could not fetch 7tv emotes."); return; } var url = $"{SevenApiClient.WEBSOCKET_URL}@emote_set.*"; _logger.Debug($"Attempting to connect to {url}"); await sevenClient.ConnectAsync($"{url}"); } catch (Exception) { _logger.Warning("Connecting to 7tv failed. Skipping 7tv websockets."); } } private async Task InitializeObs() { if (_configuration.Obs == null || string.IsNullOrWhiteSpace(_configuration.Obs.Host) || !_configuration.Obs.Port.HasValue || _configuration.Obs.Port.Value < 0) { _logger.Warning("Lacking OBS connection info. Skipping OBS websockets."); return; } try { var obsClient = _serviceProvider.GetRequiredKeyedService>("obs"); var url = $"ws://{_configuration.Obs.Host.Trim()}:{_configuration.Obs.Port}"; _logger.Debug($"Initializing OBS websocket client. Attempting to connect to {url}"); await obsClient.ConnectAsync(url); } catch (Exception) { _logger.Warning("Connecting to obs failed. Skipping obs websockets."); } } private async Task InitializeTwitchApiClient(string username, string broadcasterId) { _logger.Debug("Initializing twitch client."); var twitchapiclient = _serviceProvider.GetRequiredService(); if (!await twitchapiclient.Authorize(broadcasterId)) { _logger.Error("Cannot connect to Twitch API."); return null; } var channels = _configuration.Twitch.Channels ?? [username]; _logger.Information("Twitch channels: " + string.Join(", ", channels)); twitchapiclient.InitializeClient(username, channels); twitchapiclient.InitializePublisher(); var handler = _serviceProvider.GetRequiredService(); twitchapiclient.AddOnNewMessageReceived(async (object? s, OnMessageReceivedArgs e) => { try { var result = await handler.Handle(e); if (result.Status != MessageStatus.None || result.Emotes == null || !result.Emotes.Any()) return; var ws = _serviceProvider.GetRequiredKeyedService>("hermes"); await ws.Send(8, new EmoteUsageMessage() { MessageId = e.ChatMessage.Id, DateTime = DateTime.UtcNow, BroadcasterId = result.BroadcasterId, ChatterId = result.ChatterId, Emotes = result.Emotes }); } catch (Exception ex) { _logger.Error(ex, "Unable to either execute a command or to send emote usage message."); } }); return twitchapiclient; } private async Task InitializeEmotes(SevenApiClient sevenapi, EmoteSet channelEmotes) { var emotes = _serviceProvider.GetRequiredService(); var globalEmotes = await sevenapi.FetchGlobalSevenEmotes(); if (channelEmotes != null && channelEmotes.Emotes.Any()) { _logger.Information($"Loaded {channelEmotes.Emotes.Count()} 7tv channel emotes."); foreach (var entry in channelEmotes.Emotes) emotes.Add(entry.Name, entry.Id); } if (globalEmotes != null && globalEmotes.Any()) { _logger.Information($"Loaded {globalEmotes.Count()} 7tv global emotes."); foreach (var entry in globalEmotes) emotes.Add(entry.Name, entry.Id); } } } }