From f0071cae8180eeeb0ea21a43069c71b13b321f08 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 12 Aug 2024 16:42:53 +0000 Subject: [PATCH] Using IHostingService for TTS listening and playing in background. --- Chat/Commands/SkipCommand.cs | 2 +- Chat/Messaging/ChatMessageReader.cs | 2 +- Chat/Speech/TTSPlayer.cs | 2 +- Startup.cs | 4 +- TTS.cs | 122 ++---------------- TTSEngine.cs | 71 ++++++++++ TTSListening.cs | 117 +++++++++++++++++ .../Handlers/ChannelChatClearHandler.cs | 2 +- .../Handlers/ChannelChatClearUserHandler.cs | 2 +- .../ChannelChatDeleteMessageHandler.cs | 2 +- 10 files changed, 208 insertions(+), 118 deletions(-) create mode 100644 TTSEngine.cs create mode 100644 TTSListening.cs diff --git a/Chat/Commands/SkipCommand.cs b/Chat/Commands/SkipCommand.cs index 8e23421..452efa1 100644 --- a/Chat/Commands/SkipCommand.cs +++ b/Chat/Commands/SkipCommand.cs @@ -1,5 +1,5 @@ using Serilog; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Hermes.Socket; using TwitchChatTTS.Twitch.Socket.Messages; using static TwitchChatTTS.Chat.Commands.TTSCommands; diff --git a/Chat/Messaging/ChatMessageReader.cs b/Chat/Messaging/ChatMessageReader.cs index 97c91f0..afcda83 100644 --- a/Chat/Messaging/ChatMessageReader.cs +++ b/Chat/Messaging/ChatMessageReader.cs @@ -7,7 +7,7 @@ using TwitchChatTTS.Chat.Commands; using TwitchChatTTS.Chat.Emotes; using TwitchChatTTS.Chat.Groups; using TwitchChatTTS.Chat.Groups.Permissions; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Hermes.Socket; using TwitchChatTTS.OBS.Socket; using TwitchChatTTS.Twitch.Socket; diff --git a/Chat/Speech/TTSPlayer.cs b/Chat/Speech/TTSPlayer.cs index efc0afd..305d62d 100644 --- a/Chat/Speech/TTSPlayer.cs +++ b/Chat/Speech/TTSPlayer.cs @@ -1,6 +1,6 @@ using NAudio.Wave; -namespace TwitchChatTTS.Chat.Soeech +namespace TwitchChatTTS.Chat.Speech { public class TTSPlayer { diff --git a/Startup.cs b/Startup.cs index 03f16e3..9c445e4 100644 --- a/Startup.cs +++ b/Startup.cs @@ -29,7 +29,7 @@ using TwitchChatTTS.Twitch.Socket; using TwitchChatTTS.Twitch.Socket.Messages; using TwitchChatTTS.Twitch.Socket.Handlers; using CommonSocketLibrary.Backoff; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Chat.Messaging; // dotnet publish -r linux-x64 -p:PublishSingleFile=true --self-contained true @@ -156,5 +156,7 @@ s.AddKeyedSingleton, HermesMessageTypeMana s.AddKeyedSingleton, HermesSocketClient>("hermes"); s.AddHostedService(); +s.AddHostedService(); +s.AddHostedService(); using IHost host = builder.Build(); await host.RunAsync(); \ No newline at end of file diff --git a/TTS.cs b/TTS.cs index 6e201b9..43392b3 100644 --- a/TTS.cs +++ b/TTS.cs @@ -15,7 +15,7 @@ using TwitchChatTTS.Twitch.Socket.Messages; using TwitchChatTTS.Twitch.Socket; using TwitchChatTTS.Chat.Commands; using System.Text; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using NAudio.Wave; namespace TwitchChatTTS @@ -127,6 +127,14 @@ namespace TwitchChatTTS return; } + _playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) => + { + if (_player.Playing?.Audio == e.SampleProvider) + { + _player.Playing = null; + } + }); + try { await _twitch.Connect(); @@ -147,123 +155,15 @@ namespace TwitchChatTTS await InitializeEmotes(_sevenApiClient, emoteSet); await InitializeSevenTv(); await InitializeObs(); - - _playback.AddOnMixerInputEnded((object? s, SampleProviderEventArgs e) => - { - if (_player.Playing?.Audio == e.SampleProvider) - { - _player.Playing = null; - } - }); - - Task.Run(async () => - { - while (true) - { - try - { - if (cancellationToken.IsCancellationRequested) - { - _logger.Warning("TTS Buffer - Cancellation requested."); - return; - } - - var group = _player.ReceiveBuffer(); - if (group == null) - { - await Task.Delay(200); - continue; - } - - Task.Run(() => - { - var list = new List(); - foreach (var message in group.Messages) - { - if (string.IsNullOrEmpty(message.Message)) - { - using (var reader2 = new AudioFileReader(message.File)) - { - list.Add(_playback.ConvertSound(reader2.ToWaveProvider())); - } - continue; - } - - try - { - string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}"; - var nws = new NetworkWavSound(url); - var provider = new CachedWavProvider(nws); - var data = _playback.ConvertSound(provider); - var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate); - list.Add(resampled); - } - catch (Exception e) - { - _logger.Error(e, "Failed to fetch TTS message for "); - } - } - - var merged = new ConcatenatingSampleProvider(list); - group.Audio = merged; - _player.Ready(group); - }); - } - catch (COMException e) - { - _logger.Error(e, "Failed to send request for TTS [HResult: " + e.HResult + "]"); - } - catch (Exception e) - { - _logger.Error(e, "Failed to send request for TTS."); - } - } - }); - - Task.Run(async () => - { - while (true) - { - try - { - if (cancellationToken.IsCancellationRequested) - { - _logger.Warning("TTS Queue - Cancellation requested."); - return; - } - while (_player.IsEmpty() || _player.Playing != null) - { - await Task.Delay(200); - continue; - } - var g = _player.ReceiveReady(); - if (g == null) - { - continue; - } - - var audio = g.Audio; - - if (audio != null) - { - _player.Playing = g; - _playback.AddMixerInput(audio); - } - } - catch (Exception e) - { - _logger.Error(e, "Failed to play a TTS audio message"); - } - } - }); } - public async Task StopAsync(CancellationToken cancellationToken) + public Task StopAsync(CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) _logger.Warning("Application has stopped due to cancellation token."); else _logger.Warning("Application has stopped."); + return Task.CompletedTask; } private async Task InitializeHermesWebsocket() diff --git a/TTSEngine.cs b/TTSEngine.cs new file mode 100644 index 0000000..b993470 --- /dev/null +++ b/TTSEngine.cs @@ -0,0 +1,71 @@ +using System.Runtime.InteropServices; +using System.Web; +using Microsoft.Extensions.Hosting; +using NAudio.Wave; +using NAudio.Wave.SampleProviders; +using Serilog; +using TwitchChatTTS.Chat.Speech; + +namespace TwitchChatTTS +{ + public class TTSEngine : IHostedService + { + private readonly AudioPlaybackEngine _playback; + private readonly TTSPlayer _player; + private readonly ILogger _logger; + + + public TTSEngine(AudioPlaybackEngine playback, TTSPlayer player, ILogger logger) + { + _playback = playback; + _player = player; + _logger = logger; + } + + + public Task StartAsync(CancellationToken cancellationToken) + { + Task.Run(async () => + { + while (true) + { + try + { + if (cancellationToken.IsCancellationRequested) + { + _logger.Warning("TTS Engine - Cancellation requested."); + return; + } + while (_player.IsEmpty() || _player.Playing != null) + { + await Task.Delay(200, cancellationToken); + continue; + } + + var messageData = _player.ReceiveReady(); + if (messageData == null) + continue; + + if (messageData.Audio != null) + { + _player.Playing = messageData; + _playback.AddMixerInput(messageData.Audio); + string message = string.Join(" ", messageData.Messages.Select(m => m.File == null ? m.Message : '(' + m.File + ')')); + _logger.Debug($"Playing TTS message [message: {message}][chatter id: {messageData.ChatterId}][priority: {messageData.Priority}][message id: {messageData.MessageId}][broadcaster id: {messageData.RoomId}]"); + } + } + catch (Exception e) + { + _logger.Error(e, "Failed to play a TTS audio message"); + } + } + }); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/TTSListening.cs b/TTSListening.cs new file mode 100644 index 0000000..bd57610 --- /dev/null +++ b/TTSListening.cs @@ -0,0 +1,117 @@ +using System.Runtime.InteropServices; +using System.Web; +using Microsoft.Extensions.Hosting; +using NAudio.Wave; +using NAudio.Wave.SampleProviders; +using Serilog; +using TwitchChatTTS.Chat.Speech; + +namespace TwitchChatTTS +{ + public class TTSListening : IHostedService + { + private readonly AudioPlaybackEngine _playback; + private readonly TTSPlayer _player; + private readonly ILogger _logger; + + + public TTSListening(AudioPlaybackEngine playback, TTSPlayer player, ILogger logger) + { + _playback = playback; + _player = player; + _logger = logger; + } + + + public Task StartAsync(CancellationToken cancellationToken) + { + Task.Run(async () => + { + while (true) + { + try + { + if (cancellationToken.IsCancellationRequested) + { + _logger.Warning("TTS Listening - Cancellation requested."); + return; + } + + var group = _player.ReceiveBuffer(); + if (group == null) + { + await Task.Delay(200, cancellationToken); + continue; + } + + if (cancellationToken.IsCancellationRequested) + { + _logger.Warning("TTS Listening - Cancellation requested."); + return; + } + + FetchMasterAudio(group); + } + catch (COMException e) + { + _logger.Error(e, "Failed to send request for TTS [HResult: " + e.HResult + "]"); + } + catch (Exception e) + { + _logger.Error(e, "Failed to send request for TTS."); + } + } + }); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + private Task FetchMasterAudio(TTSGroupedMessage group) + { + return Task.Run(() => + { + var list = new List(); + foreach (var message in group.Messages) + { + if (string.IsNullOrEmpty(message.Message)) + { + using (var reader = new AudioFileReader(message.File)) + { + var data = _playback.ConvertSound(reader.ToWaveProvider()); + var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate); + list.Add(resampled); + } + continue; + } + if (string.IsNullOrEmpty(message.Voice)) + { + _logger.Error($"No voice has been selected for this message [message: {message.Message}]"); + continue; + } + + try + { + string url = $"https://api.streamelements.com/kappa/v2/speech?voice={message.Voice}&text={HttpUtility.UrlEncode(message.Message.Trim())}"; + var nws = new NetworkWavSound(url); + var provider = new CachedWavProvider(nws); + var data = _playback.ConvertSound(provider); + var resampled = new WdlResamplingSampleProvider(data, _playback.SampleRate); + list.Add(resampled); + } + catch (Exception e) + { + _logger.Error(e, "Failed to fetch TTS message for "); + } + } + + var merged = new ConcatenatingSampleProvider(list); + group.Audio = merged; + _player.Ready(group); + }); + } + } +} \ No newline at end of file diff --git a/Twitch/Socket/Handlers/ChannelChatClearHandler.cs b/Twitch/Socket/Handlers/ChannelChatClearHandler.cs index 241d108..0a03f12 100644 --- a/Twitch/Socket/Handlers/ChannelChatClearHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatClearHandler.cs @@ -1,5 +1,5 @@ using Serilog; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers diff --git a/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs b/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs index 05b3dcf..3031084 100644 --- a/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatClearUserHandler.cs @@ -1,5 +1,5 @@ using Serilog; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers diff --git a/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs b/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs index 691445b..b909b25 100644 --- a/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs +++ b/Twitch/Socket/Handlers/ChannelChatDeleteMessageHandler.cs @@ -1,5 +1,5 @@ using Serilog; -using TwitchChatTTS.Chat.Soeech; +using TwitchChatTTS.Chat.Speech; using TwitchChatTTS.Twitch.Socket.Messages; namespace TwitchChatTTS.Twitch.Socket.Handlers