Add loopback, fix mono and leak

assets/opensb/scripts/opensb/universeclient/voicemanager.lua

@@ -19,7 +19,7 @@ local LINE_COLOR = {50, 210, 255, 255}
 local FONT_DIRECTIVES = "?border=1;333;3337?border=1;333;3330"
 local NAME_PREFIX = "^noshadow,white,set;"
 
-local function dbToLoudness(db) return 2 ^ (db / 6) end
+local function dbToLoudness(db) return 2 ^ (db / 8) end
 
 local canvas
 

source/client/StarClientApplication.cpp

@@ -892,6 +892,7 @@ void ClientApplication::updateRunning() {
         }
         if (auto mainPlayer = m_universeClient->mainPlayer()) {
           auto localSpeaker = m_voice->localSpeaker();
+          localSpeaker->position = mainPlayer->position();
           localSpeaker->entityId = mainPlayer->entityId();
           localSpeaker->name = mainPlayer->name();
         }

source/frontend/StarVoice.cpp

@@ -63,11 +63,17 @@ float getAudioLoudness(int16_t* data, size_t samples, float volume = 1.0f) {
 struct VoiceAudioStream {
   // TODO: This should really be a ring buffer instead.
   std::queue<int16_t> samples;
-	SDL_AudioStream* sdlAudioStream;
+	SDL_AudioStream* sdlAudioStreamMono;
+	SDL_AudioStream* sdlAudioStreamStereo;
   Mutex mutex;
 
-	VoiceAudioStream() : sdlAudioStream(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100)) {};
+	VoiceAudioStream()
-	~VoiceAudioStream() { SDL_FreeAudioStream(sdlAudioStream); }
+		: sdlAudioStreamMono  (SDL_NewAudioStream(AUDIO_S16, 1, 48000, AUDIO_S16SYS, 1, 44100))
+	  , sdlAudioStreamStereo(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100)) {};
+	~VoiceAudioStream() {
+		SDL_FreeAudioStream(sdlAudioStreamMono);
+		SDL_FreeAudioStream(sdlAudioStreamStereo);
+	}
 
 	inline int16_t take() {
 		int16_t sample = 0;
@@ -78,11 +84,12 @@ struct VoiceAudioStream {
 		return sample;
 	}
 
-	size_t resample(int16_t* in, size_t inSamples, std::vector<int16_t>& out) {
+	size_t resample(int16_t* in, size_t inSamples, std::vector<int16_t>& out, bool mono) {
-		SDL_AudioStreamPut(sdlAudioStream, in, inSamples * sizeof(int16_t));
+		SDL_AudioStream* stream = mono ? sdlAudioStreamMono : sdlAudioStreamStereo;
-		if (int available = SDL_AudioStreamAvailable(sdlAudioStream)) {
+		SDL_AudioStreamPut(stream, in, inSamples * sizeof(int16_t));
+		if (int available = SDL_AudioStreamAvailable(stream)) {
 			out.resize(available / 2);
-			SDL_AudioStreamGet(sdlAudioStream, out.data(), available);
+			SDL_AudioStreamGet(stream, out.data(), available);
 			return available;
 		}
 		return 0;
@@ -172,6 +179,9 @@ void Voice::loadJson(Json const& config) {
   m_inputVolume  = config.getFloat("inputVolume", m_inputVolume);
   m_outputVolume = config.getFloat("outputVolume",   m_outputVolume);
 	
+	if (change(m_loopBack, config.getBool("loopBack", m_loopBack)))
+		m_clientSpeaker->playing = false;
+
 	if (auto inputMode = config.optString("inputMode")) {
 		if (change(m_inputMode, VoiceInputModeNames.getLeft(*inputMode)))
 			m_lastInputTime = 0;
@@ -273,10 +283,14 @@ void Voice::readAudioData(uint8_t* stream, int len) {
 		}
 	}
 
-	if (active && !m_clientSpeaker->playing)
+	if (!m_loopBack) {
-		m_clientSpeaker->lastPlayTime = now;
+		if (active && !m_clientSpeaker->playing)
+			m_clientSpeaker->lastPlayTime = now;
 
-	if (!(m_clientSpeaker->playing = active))
+		m_clientSpeaker->playing = active;
+	}
+
+	if (!active)
 		return;
 
 	MutexLocker captureLock(m_captureMutex);
@@ -311,7 +325,6 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
 			VoiceAudioStream* audio = speaker->audioStream.get();
 			MutexLocker audioLock(audio->mutex);
 			if (!audio->samples.empty()) {
-				SDL_AudioStream* sdlStream = audio->sdlAudioStream;
 				if (!speaker->muted) {
 					mix = true;
 					for (size_t i = 0; i != samples; ++i)
@@ -440,6 +453,8 @@ int Voice::send(DataStreamBuffer& out, size_t budget) {
 	}
 
 	m_lastSentTime = Time::monotonicMilliseconds();
+	if (m_loopBack)
+		receive(m_clientSpeaker, { out.ptr(), out.size() });
 	return 1;
 }
 
@@ -472,30 +487,26 @@ bool Voice::receive(SpeakerPtr speaker, std::string_view view) {
 			if (samples < 0)
 				throw VoiceException(strf("Decoder error: {}", opus_strerror(samples)), false);
 
-			size_t decodeBufferSize = samples * sizeof(opus_int16) * (size_t)channels;
+			m_decodeBuffer.resize(samples * (size_t)channels);
-			opus_int16* decodeBuffer = (opus_int16*)malloc(decodeBufferSize);
 
-			int decodedSamples = opus_decode(decoder, opusData, opusLength, decodeBuffer, decodeBufferSize, 0);
+			int decodedSamples = opus_decode(decoder, opusData, opusLength, m_decodeBuffer.data(), m_decodeBuffer.size() * sizeof(int16_t), 0);
 			if (decodedSamples <= 0) {
-				free(decodeBuffer);
 				if (decodedSamples < 0)
 				  throw VoiceException(strf("Decoder error: {}", opus_strerror(samples)), false);
 				return true;
 			}
 
-			decodedSamples *= channels;
+			//Logger::info("Voice: decoded Opus chunk {} bytes -> {} samples", opusLength, decodedSamples * channels);
-			//Logger::info("Voice: decoded Opus chunk {} bytes -> {} samples", opusLength, decodedSamples);
+
+			speaker->audioStream->resample(m_decodeBuffer.data(), (size_t)decodedSamples * channels, m_resampleBuffer, mono);
 
 			{
-				std::vector<int16_t> resamBuffer(decodedSamples, 0);
-				speaker->audioStream->resample(decodeBuffer, decodedSamples, resamBuffer);
-
 			  MutexLocker lock(speaker->audioStream->mutex);
 				auto& samples = speaker->audioStream->samples;
 
 				auto now = Time::monotonicMilliseconds();
 				if (now - speaker->lastReceiveTime < 1000) {
-					auto limit = ((size_t)speaker->minimumPlaySamples + 22050) * (size_t)channels;
+					auto limit = (size_t)speaker->minimumPlaySamples + 22050;
 					if (samples.size() > limit) { // skip ahead if we're getting too far
 						for (size_t i = samples.size(); i >= limit; --i)
 							samples.pop();
@@ -507,13 +518,13 @@ bool Voice::receive(SpeakerPtr speaker, std::string_view view) {
 				speaker->lastReceiveTime = now;
 
 				if (mono) {
-					for (int16_t sample : resamBuffer) {
+					for (int16_t sample : m_resampleBuffer) {
 						samples.push(sample);
 						samples.push(sample);
 					}
 				}
 				else {
-					for (int16_t sample : resamBuffer)
+					for (int16_t sample : m_resampleBuffer)
 						samples.push(sample);
 				}
 			}
@@ -589,8 +600,7 @@ void Voice::closeDevice() {
 }
 
 bool Voice::playSpeaker(SpeakerPtr const& speaker, int channels) {
-	unsigned int minSamples = speaker->minimumPlaySamples * channels;
+	if (speaker->playing || speaker->audioStream->samples.size() < speaker->minimumPlaySamples)
-	if (speaker->playing || speaker->audioStream->samples.size() < minSamples)
 		return false;
 
 	if (!speaker->playing) {
@@ -632,7 +642,8 @@ void Voice::thread() {
 						samples[i] *= m_inputVolume;
 				}
 
-				m_clientSpeaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
+				if (!m_loopBack)
+				  m_clientSpeaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
 
 				if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) {
 					if (encodedSize == 1)

source/frontend/StarVoice.hpp

@@ -180,6 +180,7 @@ private:
   int64_t m_nextSaveTime = 0;
   bool m_enabled = true;
   bool m_inputEnabled = false;
+  bool m_loopBack = false;
 
   int m_deviceChannels = 1;
   bool m_deviceOpen = false;
@@ -192,6 +193,9 @@ private:
   ConditionVariable m_threadCond;
   atomic<bool> m_stopThread;
 
+  std::vector<int16_t> m_decodeBuffer;
+  std::vector<int16_t> m_resampleBuffer;
+
   ApplicationControllerPtr m_applicationController;
 
   struct EncodedChunk {