Provide speaker info to HUD indicators

assets/opensb/scripts/universeClient/opensb/voice_manager.lua

@@ -107,7 +107,6 @@ local function drawSpeakerBar(mouse, pos, speaker, i)
 		--end
 	end
 end
-local speakersTime = {}
 
 local function simulateSpeakers()
 	local speakers = {}
@@ -127,11 +126,10 @@ local function drawIndicators()
 	canvas:clear()
 	local screenSize = canvas:size()
 	local mousePosition = canvas:mousePosition()
-	sb.setLogMap("mousePosition", sb.printJson(mousePosition))
 	local basePos = {screenSize[1] - 350, 50}
 
 	-- sort it ourselves for now
-	local speakersRemaining, speakersSorted = {}, {}
+	local speakersRemaining, speakers = {}, {}
 	local hoveredSpeakerId = nil
 	if hoveredSpeaker then
 		if not mouseOverSpeaker(mousePosition, hoveredSpeakerPosition, 16) then
@@ -141,63 +139,32 @@ local function drawIndicators()
 		end
 	end
 
-	--local speakers = voice.speakers()
+	local speakerCount = 0
-	local speakers = { -- just testing before implementing voice lua functions
-		{
-			speakerId = 1,
-			entityId = -65536,
-			loudness = -96 + math.random() * 96,
-			muted = false,
-			name = "theres a pipe bomb up my ass"
-		}
-	}
-
-	local sortI = 0
 	local now = os.clock()
-	for i, speaker in pairs(speakers) do
+	for i, speaker in pairs(voice.speakers()) do
 		local speakerId = speaker.speakerId
 		speakersRemaining[speakerId] = true
-		local t = speakersTime[speakerId]
-		if not t then
-			t = now
-			speakersTime[speakerId] = t
-		end
-		speaker.startTime = t
 		if speakerId == hoveredSpeakerId then
 			hoveredSpeaker = speaker
 		else
-			sortI = sortI + 1
+			speakerCount = speakerCount + 1
-			speakersSorted[sortI] = speaker
+			speakers[speakerCount] = speaker
 		end
 	end
 
-	for i, v in pairs(speakersTime) do
-		if not speakersRemaining[i] then
-			speakersTime[i] = nil
-		end
-	end
-
-	table.sort(speakersSorted, function(a, b)
-		if a.startTime == b.startTime then
-			return a.speakerId < b.speakerId
-		else
-			return a.startTime < b.startTime
-		end
-	end)
-
 	if hoveredSpeaker then
-		local len = #speakersSorted
+		local len = #speakers
 		if hoveredSpeakerIndex > len then
 			for i = len + 1, hoveredSpeakerIndex - 1 do
-				speakersSorted[i] = false
+				speakers[i] = false
 			end
-			speakersSorted[hoveredSpeakerIndex] = hoveredSpeaker
+			speakers[hoveredSpeakerIndex] = hoveredSpeaker
 		else
-			table.insert(speakersSorted, hoveredSpeakerIndex, hoveredSpeaker)
+			table.insert(speakers, hoveredSpeakerIndex, hoveredSpeaker)
 		end
 	end
 
-	for i, v in pairs(speakersSorted) do
+	for i, v in pairs(speakers) do
 		if v then
 			local entityId = v.entityId
 			local loudness = v.loudness

source/client/StarClientApplication.cpp

@@ -20,6 +20,7 @@
 
 #include "StarInterfaceLuaBindings.hpp"
 #include "StarInputLuaBindings.hpp"
+#include "StarVoiceLuaBindings.hpp"
 
 namespace Star {
 
@@ -496,6 +497,7 @@ void ClientApplication::changeState(MainAppState newState) {
     m_statistics = make_shared<Statistics>(m_root->toStoragePath("player"), appController()->statisticsService());
     m_universeClient = make_shared<UniverseClient>(m_playerStorage, m_statistics);
     m_universeClient->setLuaCallbacks("input", LuaBindings::makeInputCallbacks());
+    m_universeClient->setLuaCallbacks("voice", LuaBindings::makeVoiceCallbacks(m_voice.get()));
 
     m_mainMixer->setUniverseClient(m_universeClient);
     m_titleScreen = make_shared<TitleScreen>(m_playerStorage, m_mainMixer->mixer());
@@ -888,6 +890,11 @@ void ClientApplication::updateRunning() {
           auto broadcast = strf("data\0voice\0{}{}"s, signatureView, audioDataView);
           worldClient->sendSecretBroadcast(broadcast, true);
         }
+        if (auto mainPlayer = m_universeClient->mainPlayer()) {
+          auto localSpeaker = m_voice->localSpeaker();
+          localSpeaker->entityId = mainPlayer->entityId();
+          localSpeaker->name = mainPlayer->name();
+        }
         m_voice->setLocalSpeaker(worldClient->connection());
       }
       worldClient->setInteractiveHighlightMode(isActionTaken(InterfaceAction::ShowLabels));

source/frontend/CMakeLists.txt

@@ -57,6 +57,7 @@ SET (star_frontend_HEADERS
     StarTeleportDialog.hpp
     StarWireInterface.hpp
     StarVoice.hpp
+    StarVoiceLuaBindings.hpp
   )
 
 SET (star_frontend_SOURCES
@@ -106,6 +107,7 @@ SET (star_frontend_SOURCES
     StarTeleportDialog.cpp
     StarWireInterface.cpp
     StarVoice.cpp
+    StarVoiceLuaBindings.cpp
   )
 
 ADD_LIBRARY (star_frontend OBJECT ${star_frontend_SOURCES} ${star_frontend_HEADERS})

source/frontend/StarVoice.cpp

@@ -28,13 +28,13 @@ EnumMap<VoiceChannelMode> const VoiceChannelModeNames{
   {VoiceChannelMode::Stereo, "Stereo"}
 };
 
-float getAudioChunkLoudness(int16_t* data, size_t samples) {
+inline float getAudioChunkLoudness(int16_t* data, size_t samples, float volume) {
 	if (!samples)
 		return 0.f;
 
 	double rms = 0.;
 	for (size_t i = 0; i != samples; ++i) {
-		float sample = (float)data[i] / 32767.f;
+		float sample = ((float)data[i] / 32767.f) * volume;
 		rms += (double)(sample * sample);
 	}
 
@@ -46,12 +46,12 @@ float getAudioChunkLoudness(int16_t* data, size_t samples) {
 		return -127.f;
 }
 
-float getAudioLoudness(int16_t* data, size_t samples) {
+float getAudioLoudness(int16_t* data, size_t samples, float volume = 1.0f) {
 	constexpr size_t CHUNK_SIZE = 50;
 
 	float highest = -127.f;
 	for (size_t i = 0; i < samples; i += CHUNK_SIZE) {
-		float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i);
+		float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i, volume);
 		if (level > highest)
       highest = level;
 	}
@@ -192,6 +192,10 @@ Voice::SpeakerPtr Voice::setLocalSpeaker(SpeakerId speakerId) {
   return m_speakers.insert(m_speakerId, m_clientSpeaker).first->second;
 }
 
+Voice::SpeakerPtr Voice::localSpeaker() {
+	return m_clientSpeaker;
+}
+
 Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
   if (m_speakerId == speakerId)
     return m_clientSpeaker;
@@ -203,28 +207,47 @@ Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
   }
 }
 
-void Voice::readAudioData(uint8_t* stream, int len) {
+List<Voice::SpeakerPtr> Voice::speakers(bool onlyPlaying) {
-	auto now = Time::monotonicMilliseconds();
+	List<SpeakerPtr> result;
-	if (!m_encoder || m_inputMode == VoiceInputMode::PushToTalk && now > m_lastInputTime)
-		return;
 
-	// Stop encoding if 2048 bytes have been encoded and not taken by the game thread yet
+	auto sorter = [](SpeakerPtr const& a, SpeakerPtr const& b) -> bool {
-	if (m_encodedChunksLength > 2048)
+		if (a->lastPlayTime != b->lastPlayTime)
-		return;
+			return a->lastPlayTime < b->lastPlayTime;
+		else
+			return a->speakerId < b->speakerId;
+	};
 
-	size_t sampleCount = len / 2;
+	for (auto& p : m_speakers) {
-	float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
+		if (!onlyPlaying || p.second->playing)
-	m_clientSpeaker->decibelLevel = decibels;
+			result.insertSorted(p.second, sorter);
-
-	bool active = true;
-
-	if (m_inputMode == VoiceInputMode::VoiceActivity) {
-		if (decibels > m_threshold)
-			m_lastThresholdTime = now;
-		active = now - m_lastThresholdTime < 50;
 	}
 
-	bool added = false;
+	return result;
+}
+
+void Voice::readAudioData(uint8_t* stream, int len) {
+	auto now = Time::monotonicMilliseconds();
+	bool active = m_encoder && m_encodedChunksLength < 2048
+       && (m_inputMode == VoiceInputMode::VoiceActivity || now < m_lastInputTime);
+
+	size_t sampleCount = len / 2;
+
+	if (active) {
+		float volume = m_inputVolume;
+		float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
+
+		if (m_inputMode == VoiceInputMode::VoiceActivity) {
+			if (decibels > m_threshold)
+				m_lastThresholdTime = now;
+			active = now - m_lastThresholdTime < 50;
+		}
+	}
+
+	if (active && !m_clientSpeaker->playing)
+		m_clientSpeaker->lastPlayTime = now;
+
+	if (!(m_clientSpeaker->playing = active))
+		return;
 
 	MutexLocker captureLock(m_captureMutex);
 	if (active) {
@@ -232,7 +255,7 @@ void Voice::readAudioData(uint8_t* stream, int len) {
 		auto data = (opus_int16*)malloc(len);
 		memcpy(data, stream, len);
 		m_capturedChunks.emplace(data, sampleCount); // takes ownership
-		added = true;
+		m_threadCond.signal();
 	}
 	else { // Clear out any residual data so they don't manifest at the start of the next encode, whenever that is
 		while (!m_capturedChunks.empty())
@@ -240,9 +263,6 @@ void Voice::readAudioData(uint8_t* stream, int len) {
 
 		m_capturedChunksFrames = 0;
 	}
-
-	if (added)
-		m_threadCond.signal();
 }
 
 void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
@@ -493,9 +513,12 @@ bool Voice::playSpeaker(SpeakerPtr const& speaker, int channels) {
 	if (speaker->playing || speaker->audioStream->samples.size() < minSamples)
 		return false;
 
-	speaker->playing = true;
+	if (!speaker->playing) {
-	MutexLocker lock(m_activeSpeakersMutex);
+		speaker->lastPlayTime = Time::monotonicMilliseconds();
-	m_activeSpeakers.insert(speaker);
+		speaker->playing = true;
+		MutexLocker lock(m_activeSpeakersMutex);
+		m_activeSpeakers.insert(speaker);
+	}
 	return true;
 }
 
@@ -529,6 +552,8 @@ void Voice::thread() {
 						samples[i] *= m_inputVolume;
 				}
 
+				m_clientSpeaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
+
 				if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) {
 					if (encodedSize == 1)
 						continue;

source/frontend/StarVoice.hpp

@@ -81,9 +81,10 @@ public:
     Mutex mutex;
 
     int64_t lastReceiveTime = 0;
+    int64_t lastPlayTime = 0;
 
     atomic<bool> muted = false;
-    atomic<bool> playing = false;
+    atomic<bool> playing = 0;
     atomic<float> decibelLevel = 0.0f;
     atomic<Array<float, 2>> channelVolumes = Array<float, 2>::filled(1.0f);
 
@@ -118,7 +119,9 @@ public:
 
   // Sets the local speaker ID and returns the local speaker. Must be called upon loading into a world.
   SpeakerPtr setLocalSpeaker(SpeakerId speakerId);
+  SpeakerPtr localSpeaker();
   SpeakerPtr speaker(SpeakerId speakerId);
+  List<Voice::SpeakerPtr> speakers(bool onlyPlaying);
 
   // Called when receiving input audio data from SDL, on its own thread.
   void readAudioData(uint8_t* stream, int len);

source/frontend/StarVoiceLuaBindings.cpp

@@ -0,0 +1,29 @@
+#include "StarVoiceLuaBindings.hpp"
+#include "StarVoice.hpp"
+
+namespace Star {
+
+LuaCallbacks LuaBindings::makeVoiceCallbacks(Voice* voice) {
+  LuaCallbacks callbacks;
+
+  callbacks.registerCallback("speakers", [voice](Maybe<bool> onlyPlaying) -> List<Json> {
+    List<Json> list;
+
+    for (auto& speaker : voice->speakers(onlyPlaying.value(true))) {
+      list.append(JsonObject{
+        {"speakerId", speaker->speakerId },
+        {"entityId",  speaker->entityId },
+        {"name",      speaker->name },
+        {"playing",   (bool)speaker->playing },
+        {"muted",     (bool)speaker->muted },
+        {"loudness",  (float)speaker->decibelLevel },
+      });
+    }
+
+    return list;
+  });
+
+  return callbacks;
+}
+
+}

source/frontend/StarVoiceLuaBindings.hpp

@@ -0,0 +1,16 @@
+#ifndef STAR_VOICE_LUA_BINDINGS_HPP
+#define STAR_VOICE_LUA_BINDINGS_HPP
+
+#include "StarLua.hpp"
+
+namespace Star {
+
+STAR_CLASS(Voice);
+
+namespace LuaBindings {
+  LuaCallbacks makeVoiceCallbacks(Voice* voice);
+}
+
+}
+
+#endif