Provide speaker info to HUD indicators

This commit is contained in:
Kae 2023-07-18 17:36:51 +10:00
parent 34bb0b5422
commit 6e1d29fe86
7 changed files with 122 additions and 73 deletions

View File

@ -107,7 +107,6 @@ local function drawSpeakerBar(mouse, pos, speaker, i)
--end --end
end end
end end
local speakersTime = {}
local function simulateSpeakers() local function simulateSpeakers()
local speakers = {} local speakers = {}
@ -127,11 +126,10 @@ local function drawIndicators()
canvas:clear() canvas:clear()
local screenSize = canvas:size() local screenSize = canvas:size()
local mousePosition = canvas:mousePosition() local mousePosition = canvas:mousePosition()
sb.setLogMap("mousePosition", sb.printJson(mousePosition))
local basePos = {screenSize[1] - 350, 50} local basePos = {screenSize[1] - 350, 50}
-- sort it ourselves for now -- sort it ourselves for now
local speakersRemaining, speakersSorted = {}, {} local speakersRemaining, speakers = {}, {}
local hoveredSpeakerId = nil local hoveredSpeakerId = nil
if hoveredSpeaker then if hoveredSpeaker then
if not mouseOverSpeaker(mousePosition, hoveredSpeakerPosition, 16) then if not mouseOverSpeaker(mousePosition, hoveredSpeakerPosition, 16) then
@ -141,63 +139,32 @@ local function drawIndicators()
end end
end end
--local speakers = voice.speakers() local speakerCount = 0
local speakers = { -- just testing before implementing voice lua functions
{
speakerId = 1,
entityId = -65536,
loudness = -96 + math.random() * 96,
muted = false,
name = "theres a pipe bomb up my ass"
}
}
local sortI = 0
local now = os.clock() local now = os.clock()
for i, speaker in pairs(speakers) do for i, speaker in pairs(voice.speakers()) do
local speakerId = speaker.speakerId local speakerId = speaker.speakerId
speakersRemaining[speakerId] = true speakersRemaining[speakerId] = true
local t = speakersTime[speakerId]
if not t then
t = now
speakersTime[speakerId] = t
end
speaker.startTime = t
if speakerId == hoveredSpeakerId then if speakerId == hoveredSpeakerId then
hoveredSpeaker = speaker hoveredSpeaker = speaker
else else
sortI = sortI + 1 speakerCount = speakerCount + 1
speakersSorted[sortI] = speaker speakers[speakerCount] = speaker
end end
end end
for i, v in pairs(speakersTime) do
if not speakersRemaining[i] then
speakersTime[i] = nil
end
end
table.sort(speakersSorted, function(a, b)
if a.startTime == b.startTime then
return a.speakerId < b.speakerId
else
return a.startTime < b.startTime
end
end)
if hoveredSpeaker then if hoveredSpeaker then
local len = #speakersSorted local len = #speakers
if hoveredSpeakerIndex > len then if hoveredSpeakerIndex > len then
for i = len + 1, hoveredSpeakerIndex - 1 do for i = len + 1, hoveredSpeakerIndex - 1 do
speakersSorted[i] = false speakers[i] = false
end end
speakersSorted[hoveredSpeakerIndex] = hoveredSpeaker speakers[hoveredSpeakerIndex] = hoveredSpeaker
else else
table.insert(speakersSorted, hoveredSpeakerIndex, hoveredSpeaker) table.insert(speakers, hoveredSpeakerIndex, hoveredSpeaker)
end end
end end
for i, v in pairs(speakersSorted) do for i, v in pairs(speakers) do
if v then if v then
local entityId = v.entityId local entityId = v.entityId
local loudness = v.loudness local loudness = v.loudness

View File

@ -20,6 +20,7 @@
#include "StarInterfaceLuaBindings.hpp" #include "StarInterfaceLuaBindings.hpp"
#include "StarInputLuaBindings.hpp" #include "StarInputLuaBindings.hpp"
#include "StarVoiceLuaBindings.hpp"
namespace Star { namespace Star {
@ -496,6 +497,7 @@ void ClientApplication::changeState(MainAppState newState) {
m_statistics = make_shared<Statistics>(m_root->toStoragePath("player"), appController()->statisticsService()); m_statistics = make_shared<Statistics>(m_root->toStoragePath("player"), appController()->statisticsService());
m_universeClient = make_shared<UniverseClient>(m_playerStorage, m_statistics); m_universeClient = make_shared<UniverseClient>(m_playerStorage, m_statistics);
m_universeClient->setLuaCallbacks("input", LuaBindings::makeInputCallbacks()); m_universeClient->setLuaCallbacks("input", LuaBindings::makeInputCallbacks());
m_universeClient->setLuaCallbacks("voice", LuaBindings::makeVoiceCallbacks(m_voice.get()));
m_mainMixer->setUniverseClient(m_universeClient); m_mainMixer->setUniverseClient(m_universeClient);
m_titleScreen = make_shared<TitleScreen>(m_playerStorage, m_mainMixer->mixer()); m_titleScreen = make_shared<TitleScreen>(m_playerStorage, m_mainMixer->mixer());
@ -888,6 +890,11 @@ void ClientApplication::updateRunning() {
auto broadcast = strf("data\0voice\0{}{}"s, signatureView, audioDataView); auto broadcast = strf("data\0voice\0{}{}"s, signatureView, audioDataView);
worldClient->sendSecretBroadcast(broadcast, true); worldClient->sendSecretBroadcast(broadcast, true);
} }
if (auto mainPlayer = m_universeClient->mainPlayer()) {
auto localSpeaker = m_voice->localSpeaker();
localSpeaker->entityId = mainPlayer->entityId();
localSpeaker->name = mainPlayer->name();
}
m_voice->setLocalSpeaker(worldClient->connection()); m_voice->setLocalSpeaker(worldClient->connection());
} }
worldClient->setInteractiveHighlightMode(isActionTaken(InterfaceAction::ShowLabels)); worldClient->setInteractiveHighlightMode(isActionTaken(InterfaceAction::ShowLabels));

View File

@ -57,6 +57,7 @@ SET (star_frontend_HEADERS
StarTeleportDialog.hpp StarTeleportDialog.hpp
StarWireInterface.hpp StarWireInterface.hpp
StarVoice.hpp StarVoice.hpp
StarVoiceLuaBindings.hpp
) )
SET (star_frontend_SOURCES SET (star_frontend_SOURCES
@ -106,6 +107,7 @@ SET (star_frontend_SOURCES
StarTeleportDialog.cpp StarTeleportDialog.cpp
StarWireInterface.cpp StarWireInterface.cpp
StarVoice.cpp StarVoice.cpp
StarVoiceLuaBindings.cpp
) )
ADD_LIBRARY (star_frontend OBJECT ${star_frontend_SOURCES} ${star_frontend_HEADERS}) ADD_LIBRARY (star_frontend OBJECT ${star_frontend_SOURCES} ${star_frontend_HEADERS})

View File

@ -28,13 +28,13 @@ EnumMap<VoiceChannelMode> const VoiceChannelModeNames{
{VoiceChannelMode::Stereo, "Stereo"} {VoiceChannelMode::Stereo, "Stereo"}
}; };
float getAudioChunkLoudness(int16_t* data, size_t samples) { inline float getAudioChunkLoudness(int16_t* data, size_t samples, float volume) {
if (!samples) if (!samples)
return 0.f; return 0.f;
double rms = 0.; double rms = 0.;
for (size_t i = 0; i != samples; ++i) { for (size_t i = 0; i != samples; ++i) {
float sample = (float)data[i] / 32767.f; float sample = ((float)data[i] / 32767.f) * volume;
rms += (double)(sample * sample); rms += (double)(sample * sample);
} }
@ -46,12 +46,12 @@ float getAudioChunkLoudness(int16_t* data, size_t samples) {
return -127.f; return -127.f;
} }
float getAudioLoudness(int16_t* data, size_t samples) { float getAudioLoudness(int16_t* data, size_t samples, float volume = 1.0f) {
constexpr size_t CHUNK_SIZE = 50; constexpr size_t CHUNK_SIZE = 50;
float highest = -127.f; float highest = -127.f;
for (size_t i = 0; i < samples; i += CHUNK_SIZE) { for (size_t i = 0; i < samples; i += CHUNK_SIZE) {
float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i); float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i, volume);
if (level > highest) if (level > highest)
highest = level; highest = level;
} }
@ -192,6 +192,10 @@ Voice::SpeakerPtr Voice::setLocalSpeaker(SpeakerId speakerId) {
return m_speakers.insert(m_speakerId, m_clientSpeaker).first->second; return m_speakers.insert(m_speakerId, m_clientSpeaker).first->second;
} }
Voice::SpeakerPtr Voice::localSpeaker() {
return m_clientSpeaker;
}
Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) { Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
if (m_speakerId == speakerId) if (m_speakerId == speakerId)
return m_clientSpeaker; return m_clientSpeaker;
@ -203,28 +207,47 @@ Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
} }
} }
List<Voice::SpeakerPtr> Voice::speakers(bool onlyPlaying) {
List<SpeakerPtr> result;
auto sorter = [](SpeakerPtr const& a, SpeakerPtr const& b) -> bool {
if (a->lastPlayTime != b->lastPlayTime)
return a->lastPlayTime < b->lastPlayTime;
else
return a->speakerId < b->speakerId;
};
for (auto& p : m_speakers) {
if (!onlyPlaying || p.second->playing)
result.insertSorted(p.second, sorter);
}
return result;
}
void Voice::readAudioData(uint8_t* stream, int len) { void Voice::readAudioData(uint8_t* stream, int len) {
auto now = Time::monotonicMilliseconds(); auto now = Time::monotonicMilliseconds();
if (!m_encoder || m_inputMode == VoiceInputMode::PushToTalk && now > m_lastInputTime) bool active = m_encoder && m_encodedChunksLength < 2048
return; && (m_inputMode == VoiceInputMode::VoiceActivity || now < m_lastInputTime);
// Stop encoding if 2048 bytes have been encoded and not taken by the game thread yet
if (m_encodedChunksLength > 2048)
return;
size_t sampleCount = len / 2; size_t sampleCount = len / 2;
float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
m_clientSpeaker->decibelLevel = decibels;
bool active = true; if (active) {
float volume = m_inputVolume;
float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
if (m_inputMode == VoiceInputMode::VoiceActivity) { if (m_inputMode == VoiceInputMode::VoiceActivity) {
if (decibels > m_threshold) if (decibels > m_threshold)
m_lastThresholdTime = now; m_lastThresholdTime = now;
active = now - m_lastThresholdTime < 50; active = now - m_lastThresholdTime < 50;
} }
}
bool added = false; if (active && !m_clientSpeaker->playing)
m_clientSpeaker->lastPlayTime = now;
if (!(m_clientSpeaker->playing = active))
return;
MutexLocker captureLock(m_captureMutex); MutexLocker captureLock(m_captureMutex);
if (active) { if (active) {
@ -232,7 +255,7 @@ void Voice::readAudioData(uint8_t* stream, int len) {
auto data = (opus_int16*)malloc(len); auto data = (opus_int16*)malloc(len);
memcpy(data, stream, len); memcpy(data, stream, len);
m_capturedChunks.emplace(data, sampleCount); // takes ownership m_capturedChunks.emplace(data, sampleCount); // takes ownership
added = true; m_threadCond.signal();
} }
else { // Clear out any residual data so they don't manifest at the start of the next encode, whenever that is else { // Clear out any residual data so they don't manifest at the start of the next encode, whenever that is
while (!m_capturedChunks.empty()) while (!m_capturedChunks.empty())
@ -240,9 +263,6 @@ void Voice::readAudioData(uint8_t* stream, int len) {
m_capturedChunksFrames = 0; m_capturedChunksFrames = 0;
} }
if (added)
m_threadCond.signal();
} }
void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) { void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
@ -493,9 +513,12 @@ bool Voice::playSpeaker(SpeakerPtr const& speaker, int channels) {
if (speaker->playing || speaker->audioStream->samples.size() < minSamples) if (speaker->playing || speaker->audioStream->samples.size() < minSamples)
return false; return false;
if (!speaker->playing) {
speaker->lastPlayTime = Time::monotonicMilliseconds();
speaker->playing = true; speaker->playing = true;
MutexLocker lock(m_activeSpeakersMutex); MutexLocker lock(m_activeSpeakersMutex);
m_activeSpeakers.insert(speaker); m_activeSpeakers.insert(speaker);
}
return true; return true;
} }
@ -529,6 +552,8 @@ void Voice::thread() {
samples[i] *= m_inputVolume; samples[i] *= m_inputVolume;
} }
m_clientSpeaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) { if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) {
if (encodedSize == 1) if (encodedSize == 1)
continue; continue;

View File

@ -81,9 +81,10 @@ public:
Mutex mutex; Mutex mutex;
int64_t lastReceiveTime = 0; int64_t lastReceiveTime = 0;
int64_t lastPlayTime = 0;
atomic<bool> muted = false; atomic<bool> muted = false;
atomic<bool> playing = false; atomic<bool> playing = 0;
atomic<float> decibelLevel = 0.0f; atomic<float> decibelLevel = 0.0f;
atomic<Array<float, 2>> channelVolumes = Array<float, 2>::filled(1.0f); atomic<Array<float, 2>> channelVolumes = Array<float, 2>::filled(1.0f);
@ -118,7 +119,9 @@ public:
// Sets the local speaker ID and returns the local speaker. Must be called upon loading into a world. // Sets the local speaker ID and returns the local speaker. Must be called upon loading into a world.
SpeakerPtr setLocalSpeaker(SpeakerId speakerId); SpeakerPtr setLocalSpeaker(SpeakerId speakerId);
SpeakerPtr localSpeaker();
SpeakerPtr speaker(SpeakerId speakerId); SpeakerPtr speaker(SpeakerId speakerId);
List<Voice::SpeakerPtr> speakers(bool onlyPlaying);
// Called when receiving input audio data from SDL, on its own thread. // Called when receiving input audio data from SDL, on its own thread.
void readAudioData(uint8_t* stream, int len); void readAudioData(uint8_t* stream, int len);

View File

@ -0,0 +1,29 @@
#include "StarVoiceLuaBindings.hpp"
#include "StarVoice.hpp"
namespace Star {
LuaCallbacks LuaBindings::makeVoiceCallbacks(Voice* voice) {
LuaCallbacks callbacks;
callbacks.registerCallback("speakers", [voice](Maybe<bool> onlyPlaying) -> List<Json> {
List<Json> list;
for (auto& speaker : voice->speakers(onlyPlaying.value(true))) {
list.append(JsonObject{
{"speakerId", speaker->speakerId },
{"entityId", speaker->entityId },
{"name", speaker->name },
{"playing", (bool)speaker->playing },
{"muted", (bool)speaker->muted },
{"loudness", (float)speaker->decibelLevel },
});
}
return list;
});
return callbacks;
}
}

View File

@ -0,0 +1,16 @@
#ifndef STAR_VOICE_LUA_BINDINGS_HPP
#define STAR_VOICE_LUA_BINDINGS_HPP
#include "StarLua.hpp"
namespace Star {
STAR_CLASS(Voice);
namespace LuaBindings {
LuaCallbacks makeVoiceCallbacks(Voice* voice);
}
}
#endif