Provide speaker info to HUD indicators
This commit is contained in:
parent
34bb0b5422
commit
6e1d29fe86
@ -107,7 +107,6 @@ local function drawSpeakerBar(mouse, pos, speaker, i)
|
|||||||
--end
|
--end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
local speakersTime = {}
|
|
||||||
|
|
||||||
local function simulateSpeakers()
|
local function simulateSpeakers()
|
||||||
local speakers = {}
|
local speakers = {}
|
||||||
@ -127,11 +126,10 @@ local function drawIndicators()
|
|||||||
canvas:clear()
|
canvas:clear()
|
||||||
local screenSize = canvas:size()
|
local screenSize = canvas:size()
|
||||||
local mousePosition = canvas:mousePosition()
|
local mousePosition = canvas:mousePosition()
|
||||||
sb.setLogMap("mousePosition", sb.printJson(mousePosition))
|
|
||||||
local basePos = {screenSize[1] - 350, 50}
|
local basePos = {screenSize[1] - 350, 50}
|
||||||
|
|
||||||
-- sort it ourselves for now
|
-- sort it ourselves for now
|
||||||
local speakersRemaining, speakersSorted = {}, {}
|
local speakersRemaining, speakers = {}, {}
|
||||||
local hoveredSpeakerId = nil
|
local hoveredSpeakerId = nil
|
||||||
if hoveredSpeaker then
|
if hoveredSpeaker then
|
||||||
if not mouseOverSpeaker(mousePosition, hoveredSpeakerPosition, 16) then
|
if not mouseOverSpeaker(mousePosition, hoveredSpeakerPosition, 16) then
|
||||||
@ -141,63 +139,32 @@ local function drawIndicators()
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
--local speakers = voice.speakers()
|
local speakerCount = 0
|
||||||
local speakers = { -- just testing before implementing voice lua functions
|
|
||||||
{
|
|
||||||
speakerId = 1,
|
|
||||||
entityId = -65536,
|
|
||||||
loudness = -96 + math.random() * 96,
|
|
||||||
muted = false,
|
|
||||||
name = "theres a pipe bomb up my ass"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
local sortI = 0
|
|
||||||
local now = os.clock()
|
local now = os.clock()
|
||||||
for i, speaker in pairs(speakers) do
|
for i, speaker in pairs(voice.speakers()) do
|
||||||
local speakerId = speaker.speakerId
|
local speakerId = speaker.speakerId
|
||||||
speakersRemaining[speakerId] = true
|
speakersRemaining[speakerId] = true
|
||||||
local t = speakersTime[speakerId]
|
|
||||||
if not t then
|
|
||||||
t = now
|
|
||||||
speakersTime[speakerId] = t
|
|
||||||
end
|
|
||||||
speaker.startTime = t
|
|
||||||
if speakerId == hoveredSpeakerId then
|
if speakerId == hoveredSpeakerId then
|
||||||
hoveredSpeaker = speaker
|
hoveredSpeaker = speaker
|
||||||
else
|
else
|
||||||
sortI = sortI + 1
|
speakerCount = speakerCount + 1
|
||||||
speakersSorted[sortI] = speaker
|
speakers[speakerCount] = speaker
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
for i, v in pairs(speakersTime) do
|
|
||||||
if not speakersRemaining[i] then
|
|
||||||
speakersTime[i] = nil
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
table.sort(speakersSorted, function(a, b)
|
|
||||||
if a.startTime == b.startTime then
|
|
||||||
return a.speakerId < b.speakerId
|
|
||||||
else
|
|
||||||
return a.startTime < b.startTime
|
|
||||||
end
|
|
||||||
end)
|
|
||||||
|
|
||||||
if hoveredSpeaker then
|
if hoveredSpeaker then
|
||||||
local len = #speakersSorted
|
local len = #speakers
|
||||||
if hoveredSpeakerIndex > len then
|
if hoveredSpeakerIndex > len then
|
||||||
for i = len + 1, hoveredSpeakerIndex - 1 do
|
for i = len + 1, hoveredSpeakerIndex - 1 do
|
||||||
speakersSorted[i] = false
|
speakers[i] = false
|
||||||
end
|
end
|
||||||
speakersSorted[hoveredSpeakerIndex] = hoveredSpeaker
|
speakers[hoveredSpeakerIndex] = hoveredSpeaker
|
||||||
else
|
else
|
||||||
table.insert(speakersSorted, hoveredSpeakerIndex, hoveredSpeaker)
|
table.insert(speakers, hoveredSpeakerIndex, hoveredSpeaker)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
for i, v in pairs(speakersSorted) do
|
for i, v in pairs(speakers) do
|
||||||
if v then
|
if v then
|
||||||
local entityId = v.entityId
|
local entityId = v.entityId
|
||||||
local loudness = v.loudness
|
local loudness = v.loudness
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
|
|
||||||
#include "StarInterfaceLuaBindings.hpp"
|
#include "StarInterfaceLuaBindings.hpp"
|
||||||
#include "StarInputLuaBindings.hpp"
|
#include "StarInputLuaBindings.hpp"
|
||||||
|
#include "StarVoiceLuaBindings.hpp"
|
||||||
|
|
||||||
namespace Star {
|
namespace Star {
|
||||||
|
|
||||||
@ -496,6 +497,7 @@ void ClientApplication::changeState(MainAppState newState) {
|
|||||||
m_statistics = make_shared<Statistics>(m_root->toStoragePath("player"), appController()->statisticsService());
|
m_statistics = make_shared<Statistics>(m_root->toStoragePath("player"), appController()->statisticsService());
|
||||||
m_universeClient = make_shared<UniverseClient>(m_playerStorage, m_statistics);
|
m_universeClient = make_shared<UniverseClient>(m_playerStorage, m_statistics);
|
||||||
m_universeClient->setLuaCallbacks("input", LuaBindings::makeInputCallbacks());
|
m_universeClient->setLuaCallbacks("input", LuaBindings::makeInputCallbacks());
|
||||||
|
m_universeClient->setLuaCallbacks("voice", LuaBindings::makeVoiceCallbacks(m_voice.get()));
|
||||||
|
|
||||||
m_mainMixer->setUniverseClient(m_universeClient);
|
m_mainMixer->setUniverseClient(m_universeClient);
|
||||||
m_titleScreen = make_shared<TitleScreen>(m_playerStorage, m_mainMixer->mixer());
|
m_titleScreen = make_shared<TitleScreen>(m_playerStorage, m_mainMixer->mixer());
|
||||||
@ -888,6 +890,11 @@ void ClientApplication::updateRunning() {
|
|||||||
auto broadcast = strf("data\0voice\0{}{}"s, signatureView, audioDataView);
|
auto broadcast = strf("data\0voice\0{}{}"s, signatureView, audioDataView);
|
||||||
worldClient->sendSecretBroadcast(broadcast, true);
|
worldClient->sendSecretBroadcast(broadcast, true);
|
||||||
}
|
}
|
||||||
|
if (auto mainPlayer = m_universeClient->mainPlayer()) {
|
||||||
|
auto localSpeaker = m_voice->localSpeaker();
|
||||||
|
localSpeaker->entityId = mainPlayer->entityId();
|
||||||
|
localSpeaker->name = mainPlayer->name();
|
||||||
|
}
|
||||||
m_voice->setLocalSpeaker(worldClient->connection());
|
m_voice->setLocalSpeaker(worldClient->connection());
|
||||||
}
|
}
|
||||||
worldClient->setInteractiveHighlightMode(isActionTaken(InterfaceAction::ShowLabels));
|
worldClient->setInteractiveHighlightMode(isActionTaken(InterfaceAction::ShowLabels));
|
||||||
|
@ -57,6 +57,7 @@ SET (star_frontend_HEADERS
|
|||||||
StarTeleportDialog.hpp
|
StarTeleportDialog.hpp
|
||||||
StarWireInterface.hpp
|
StarWireInterface.hpp
|
||||||
StarVoice.hpp
|
StarVoice.hpp
|
||||||
|
StarVoiceLuaBindings.hpp
|
||||||
)
|
)
|
||||||
|
|
||||||
SET (star_frontend_SOURCES
|
SET (star_frontend_SOURCES
|
||||||
@ -106,6 +107,7 @@ SET (star_frontend_SOURCES
|
|||||||
StarTeleportDialog.cpp
|
StarTeleportDialog.cpp
|
||||||
StarWireInterface.cpp
|
StarWireInterface.cpp
|
||||||
StarVoice.cpp
|
StarVoice.cpp
|
||||||
|
StarVoiceLuaBindings.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
ADD_LIBRARY (star_frontend OBJECT ${star_frontend_SOURCES} ${star_frontend_HEADERS})
|
ADD_LIBRARY (star_frontend OBJECT ${star_frontend_SOURCES} ${star_frontend_HEADERS})
|
||||||
|
@ -28,13 +28,13 @@ EnumMap<VoiceChannelMode> const VoiceChannelModeNames{
|
|||||||
{VoiceChannelMode::Stereo, "Stereo"}
|
{VoiceChannelMode::Stereo, "Stereo"}
|
||||||
};
|
};
|
||||||
|
|
||||||
float getAudioChunkLoudness(int16_t* data, size_t samples) {
|
inline float getAudioChunkLoudness(int16_t* data, size_t samples, float volume) {
|
||||||
if (!samples)
|
if (!samples)
|
||||||
return 0.f;
|
return 0.f;
|
||||||
|
|
||||||
double rms = 0.;
|
double rms = 0.;
|
||||||
for (size_t i = 0; i != samples; ++i) {
|
for (size_t i = 0; i != samples; ++i) {
|
||||||
float sample = (float)data[i] / 32767.f;
|
float sample = ((float)data[i] / 32767.f) * volume;
|
||||||
rms += (double)(sample * sample);
|
rms += (double)(sample * sample);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -46,12 +46,12 @@ float getAudioChunkLoudness(int16_t* data, size_t samples) {
|
|||||||
return -127.f;
|
return -127.f;
|
||||||
}
|
}
|
||||||
|
|
||||||
float getAudioLoudness(int16_t* data, size_t samples) {
|
float getAudioLoudness(int16_t* data, size_t samples, float volume = 1.0f) {
|
||||||
constexpr size_t CHUNK_SIZE = 50;
|
constexpr size_t CHUNK_SIZE = 50;
|
||||||
|
|
||||||
float highest = -127.f;
|
float highest = -127.f;
|
||||||
for (size_t i = 0; i < samples; i += CHUNK_SIZE) {
|
for (size_t i = 0; i < samples; i += CHUNK_SIZE) {
|
||||||
float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i);
|
float level = getAudioChunkLoudness(data + i, std::min<size_t>(i + CHUNK_SIZE, samples) - i, volume);
|
||||||
if (level > highest)
|
if (level > highest)
|
||||||
highest = level;
|
highest = level;
|
||||||
}
|
}
|
||||||
@ -192,6 +192,10 @@ Voice::SpeakerPtr Voice::setLocalSpeaker(SpeakerId speakerId) {
|
|||||||
return m_speakers.insert(m_speakerId, m_clientSpeaker).first->second;
|
return m_speakers.insert(m_speakerId, m_clientSpeaker).first->second;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Voice::SpeakerPtr Voice::localSpeaker() {
|
||||||
|
return m_clientSpeaker;
|
||||||
|
}
|
||||||
|
|
||||||
Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
|
Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
|
||||||
if (m_speakerId == speakerId)
|
if (m_speakerId == speakerId)
|
||||||
return m_clientSpeaker;
|
return m_clientSpeaker;
|
||||||
@ -203,28 +207,47 @@ Voice::SpeakerPtr Voice::speaker(SpeakerId speakerId) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
List<Voice::SpeakerPtr> Voice::speakers(bool onlyPlaying) {
|
||||||
|
List<SpeakerPtr> result;
|
||||||
|
|
||||||
|
auto sorter = [](SpeakerPtr const& a, SpeakerPtr const& b) -> bool {
|
||||||
|
if (a->lastPlayTime != b->lastPlayTime)
|
||||||
|
return a->lastPlayTime < b->lastPlayTime;
|
||||||
|
else
|
||||||
|
return a->speakerId < b->speakerId;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (auto& p : m_speakers) {
|
||||||
|
if (!onlyPlaying || p.second->playing)
|
||||||
|
result.insertSorted(p.second, sorter);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
void Voice::readAudioData(uint8_t* stream, int len) {
|
void Voice::readAudioData(uint8_t* stream, int len) {
|
||||||
auto now = Time::monotonicMilliseconds();
|
auto now = Time::monotonicMilliseconds();
|
||||||
if (!m_encoder || m_inputMode == VoiceInputMode::PushToTalk && now > m_lastInputTime)
|
bool active = m_encoder && m_encodedChunksLength < 2048
|
||||||
return;
|
&& (m_inputMode == VoiceInputMode::VoiceActivity || now < m_lastInputTime);
|
||||||
|
|
||||||
// Stop encoding if 2048 bytes have been encoded and not taken by the game thread yet
|
|
||||||
if (m_encodedChunksLength > 2048)
|
|
||||||
return;
|
|
||||||
|
|
||||||
size_t sampleCount = len / 2;
|
size_t sampleCount = len / 2;
|
||||||
float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
|
|
||||||
m_clientSpeaker->decibelLevel = decibels;
|
|
||||||
|
|
||||||
bool active = true;
|
if (active) {
|
||||||
|
float volume = m_inputVolume;
|
||||||
|
float decibels = getAudioLoudness((int16_t*)stream, sampleCount);
|
||||||
|
|
||||||
if (m_inputMode == VoiceInputMode::VoiceActivity) {
|
if (m_inputMode == VoiceInputMode::VoiceActivity) {
|
||||||
if (decibels > m_threshold)
|
if (decibels > m_threshold)
|
||||||
m_lastThresholdTime = now;
|
m_lastThresholdTime = now;
|
||||||
active = now - m_lastThresholdTime < 50;
|
active = now - m_lastThresholdTime < 50;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
bool added = false;
|
if (active && !m_clientSpeaker->playing)
|
||||||
|
m_clientSpeaker->lastPlayTime = now;
|
||||||
|
|
||||||
|
if (!(m_clientSpeaker->playing = active))
|
||||||
|
return;
|
||||||
|
|
||||||
MutexLocker captureLock(m_captureMutex);
|
MutexLocker captureLock(m_captureMutex);
|
||||||
if (active) {
|
if (active) {
|
||||||
@ -232,7 +255,7 @@ void Voice::readAudioData(uint8_t* stream, int len) {
|
|||||||
auto data = (opus_int16*)malloc(len);
|
auto data = (opus_int16*)malloc(len);
|
||||||
memcpy(data, stream, len);
|
memcpy(data, stream, len);
|
||||||
m_capturedChunks.emplace(data, sampleCount); // takes ownership
|
m_capturedChunks.emplace(data, sampleCount); // takes ownership
|
||||||
added = true;
|
m_threadCond.signal();
|
||||||
}
|
}
|
||||||
else { // Clear out any residual data so they don't manifest at the start of the next encode, whenever that is
|
else { // Clear out any residual data so they don't manifest at the start of the next encode, whenever that is
|
||||||
while (!m_capturedChunks.empty())
|
while (!m_capturedChunks.empty())
|
||||||
@ -240,9 +263,6 @@ void Voice::readAudioData(uint8_t* stream, int len) {
|
|||||||
|
|
||||||
m_capturedChunksFrames = 0;
|
m_capturedChunksFrames = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (added)
|
|
||||||
m_threadCond.signal();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
|
void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
|
||||||
@ -493,9 +513,12 @@ bool Voice::playSpeaker(SpeakerPtr const& speaker, int channels) {
|
|||||||
if (speaker->playing || speaker->audioStream->samples.size() < minSamples)
|
if (speaker->playing || speaker->audioStream->samples.size() < minSamples)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
if (!speaker->playing) {
|
||||||
|
speaker->lastPlayTime = Time::monotonicMilliseconds();
|
||||||
speaker->playing = true;
|
speaker->playing = true;
|
||||||
MutexLocker lock(m_activeSpeakersMutex);
|
MutexLocker lock(m_activeSpeakersMutex);
|
||||||
m_activeSpeakers.insert(speaker);
|
m_activeSpeakers.insert(speaker);
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -529,6 +552,8 @@ void Voice::thread() {
|
|||||||
samples[i] *= m_inputVolume;
|
samples[i] *= m_inputVolume;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
m_clientSpeaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
|
||||||
|
|
||||||
if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) {
|
if (int encodedSize = opus_encode(m_encoder.get(), samples.data(), VOICE_FRAME_SIZE, (unsigned char*)encoded.ptr(), encoded.size())) {
|
||||||
if (encodedSize == 1)
|
if (encodedSize == 1)
|
||||||
continue;
|
continue;
|
||||||
|
@ -81,9 +81,10 @@ public:
|
|||||||
Mutex mutex;
|
Mutex mutex;
|
||||||
|
|
||||||
int64_t lastReceiveTime = 0;
|
int64_t lastReceiveTime = 0;
|
||||||
|
int64_t lastPlayTime = 0;
|
||||||
|
|
||||||
atomic<bool> muted = false;
|
atomic<bool> muted = false;
|
||||||
atomic<bool> playing = false;
|
atomic<bool> playing = 0;
|
||||||
atomic<float> decibelLevel = 0.0f;
|
atomic<float> decibelLevel = 0.0f;
|
||||||
atomic<Array<float, 2>> channelVolumes = Array<float, 2>::filled(1.0f);
|
atomic<Array<float, 2>> channelVolumes = Array<float, 2>::filled(1.0f);
|
||||||
|
|
||||||
@ -118,7 +119,9 @@ public:
|
|||||||
|
|
||||||
// Sets the local speaker ID and returns the local speaker. Must be called upon loading into a world.
|
// Sets the local speaker ID and returns the local speaker. Must be called upon loading into a world.
|
||||||
SpeakerPtr setLocalSpeaker(SpeakerId speakerId);
|
SpeakerPtr setLocalSpeaker(SpeakerId speakerId);
|
||||||
|
SpeakerPtr localSpeaker();
|
||||||
SpeakerPtr speaker(SpeakerId speakerId);
|
SpeakerPtr speaker(SpeakerId speakerId);
|
||||||
|
List<Voice::SpeakerPtr> speakers(bool onlyPlaying);
|
||||||
|
|
||||||
// Called when receiving input audio data from SDL, on its own thread.
|
// Called when receiving input audio data from SDL, on its own thread.
|
||||||
void readAudioData(uint8_t* stream, int len);
|
void readAudioData(uint8_t* stream, int len);
|
||||||
|
29
source/frontend/StarVoiceLuaBindings.cpp
Normal file
29
source/frontend/StarVoiceLuaBindings.cpp
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
#include "StarVoiceLuaBindings.hpp"
|
||||||
|
#include "StarVoice.hpp"
|
||||||
|
|
||||||
|
namespace Star {
|
||||||
|
|
||||||
|
LuaCallbacks LuaBindings::makeVoiceCallbacks(Voice* voice) {
|
||||||
|
LuaCallbacks callbacks;
|
||||||
|
|
||||||
|
callbacks.registerCallback("speakers", [voice](Maybe<bool> onlyPlaying) -> List<Json> {
|
||||||
|
List<Json> list;
|
||||||
|
|
||||||
|
for (auto& speaker : voice->speakers(onlyPlaying.value(true))) {
|
||||||
|
list.append(JsonObject{
|
||||||
|
{"speakerId", speaker->speakerId },
|
||||||
|
{"entityId", speaker->entityId },
|
||||||
|
{"name", speaker->name },
|
||||||
|
{"playing", (bool)speaker->playing },
|
||||||
|
{"muted", (bool)speaker->muted },
|
||||||
|
{"loudness", (float)speaker->decibelLevel },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return list;
|
||||||
|
});
|
||||||
|
|
||||||
|
return callbacks;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
16
source/frontend/StarVoiceLuaBindings.hpp
Normal file
16
source/frontend/StarVoiceLuaBindings.hpp
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
#ifndef STAR_VOICE_LUA_BINDINGS_HPP
|
||||||
|
#define STAR_VOICE_LUA_BINDINGS_HPP
|
||||||
|
|
||||||
|
#include "StarLua.hpp"
|
||||||
|
|
||||||
|
namespace Star {
|
||||||
|
|
||||||
|
STAR_CLASS(Voice);
|
||||||
|
|
||||||
|
namespace LuaBindings {
|
||||||
|
LuaCallbacks makeVoiceCallbacks(Voice* voice);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
Loading…
Reference in New Issue
Block a user