evert/osb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support receiving SE voice data, resample per-speaker again because of positional delay

Browse Source
This commit is contained in:
Kae 2023-07-16 20:44:15 +10:00
parent 4e44a4ed75
commit da098c7b48
3 changed files with 95 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
source/frontend/StarVoice.cpp
View File

@ -61,9 +61,31 @@ float getAudioLoudness(int16_t* data, size_t samples) {
struct VoiceAudioStream {
// TODO: This should really be a ring buffer instead.
std::vector<int16_t> samples;
std::queue<int16_t> samples;
SDL_AudioStream* sdlAudioStream;
Mutex mutex;
VoiceAudioStream() : sdlAudioStream(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100)) {};
~VoiceAudioStream() { SDL_FreeAudioStream(sdlAudioStream); }
inline int16_t take() {
int16_t sample = 0;
if (!samples.empty()) {
sample = samples.front();
samples.pop();
}
return sample;
}
size_t resample(int16_t* in, size_t inSamples, std::vector<int16_t>& out) {
SDL_AudioStreamPut(sdlAudioStream, in, inSamples * sizeof(int16_t));
if (int available = SDL_AudioStreamAvailable(sdlAudioStream)) {
out.resize(available / 2);
SDL_AudioStreamGet(sdlAudioStream, out.data(), available);
return available;
}
return 0;
}
};
Voice::Speaker::Speaker(SpeakerId id)
@ -224,11 +246,11 @@ void Voice::readAudioData(uint8_t* stream, int len) {
}
void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
static std::vector<int16_t> finalBuffer;
static std::vector<int32_t> voiceBuffer;
static std::vector<int16_t> resampled;
size_t samples = frameCount * channels;
resampled.resize(samples, 0);
static std::vector<int16_t> finalBuffer, speakerBuffer;
static std::vector<int32_t> sharedBuffer; //int32 to reduce clipping
speakerBuffer.resize(samples);
sharedBuffer.resize(samples);
bool mix = false;
{
@ -239,17 +261,21 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
VoiceAudioStream* audio = speaker->audioStream.get();
MutexLocker audioLock(audio->mutex);
if (!audio->samples.empty()) {
std::vector<int16_t> samples = move(audio->samples);
audioLock.unlock();
speaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
SDL_AudioStream* sdlStream = audio->sdlAudioStream;
if (!speaker->muted) {
mix = true;
if (voiceBuffer.size() < samples.size())
voiceBuffer.resize(samples.size(), 0);
for (size_t i = 0; i != samples; ++i)
speakerBuffer[i] = audio->take();
speaker->decibelLevel = getAudioLoudness(speakerBuffer.data(), samples);
auto channelVolumes = speaker->channelVolumes.load();
for (size_t i = 0; i != samples.size(); ++i)
voiceBuffer[i] += (int32_t)(samples[i]) * channelVolumes[i % 2];
for (size_t i = 0; i != samples; ++i)
sharedBuffer[i] += (int32_t)(speakerBuffer[i]) * channelVolumes[i % 2];
}
else {
for (size_t i = 0; i != samples; ++i)
audio->take();
}
++it;
}
@ -260,26 +286,16 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
}
}
static std::unique_ptr<SDL_AudioStream, void(*)(SDL_AudioStream*)> audioStream
(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100), SDL_FreeAudioStream);
if (mix) {
finalBuffer.resize(voiceBuffer.size(), 0);
finalBuffer.resize(sharedBuffer.size(), 0);
float vol = m_outputVolume;
for (size_t i = 0; i != voiceBuffer.size(); ++i)
finalBuffer[i] = (int16_t)clamp<int>(voiceBuffer[i] * vol, INT16_MIN, INT16_MAX);
for (size_t i = 0; i != sharedBuffer.size(); ++i)
finalBuffer[i] = (int16_t)clamp<int>(sharedBuffer[i] * vol, INT16_MIN, INT16_MAX);
SDL_AudioStreamPut(audioStream.get(), finalBuffer.data(), finalBuffer.size() * sizeof(int16_t));
SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)finalBuffer.data(), AUDIO_S16, finalBuffer.size() * sizeof(int16_t), SDL_MIX_MAXVOLUME);
memset(sharedBuffer.data(), 0, sharedBuffer.size() * sizeof(int32_t));
}
if (size_t available = min<size_t>(samples * sizeof(int16_t), SDL_AudioStreamAvailable(audioStream.get()))) {
SDL_AudioStreamGet(audioStream.get(), resampled.data(), available);
SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)resampled.data(), AUDIO_S16, samples * sizeof(int16_t), SDL_MIX_MAXVOLUME);
}
resampled.clear();
voiceBuffer.clear();
}
void Voice::update(PositionalAttenuationFunction positionalAttenuationFunction) {
@ -378,17 +394,35 @@ bool Voice::receive(SpeakerPtr speaker, std::string_view view) {
//Logger::info("Voice: decoded Opus chunk {} bytes -> {} samples", opusLength, decodedSamples);
{
MutexLocker lock(speaker->audioStream->mutex);
std::vector<int16_t> resamBuffer(decodedSamples, 0);
speaker->audioStream->resample(decodeBuffer, decodedSamples, resamBuffer);
MutexLocker lock(speaker->audioStream->mutex);
auto& samples = speaker->audioStream->samples;
if (mono) {
size_t prevSize = samples.size();
samples.resize(prevSize + (size_t)decodedSamples * 2);
int16_t* data = samples.data() + prevSize;
for (int i = 0; i != decodedSamples; ++i)
*data++ = *data++ = decodeBuffer[i];
auto now = Time::monotonicMilliseconds();
if (now - speaker->lastReceiveTime < 1000) {
auto limit = ((size_t)speaker->minimumPlaySamples + 22050) * (size_t)channels;
if (samples.size() > limit) { // skip ahead if we're getting too far
for (size_t i = samples.size(); i >= limit; --i)
samples.pop();
}
}
else
samples.insert(samples.end(), decodeBuffer, decodeBuffer + decodedSamples);
samples = std::queue<int16_t>();
speaker->lastReceiveTime = now;
if (mono) {
for (int16_t sample : resamBuffer) {
samples.push(sample);
samples.push(sample);
}
}
else {
for (int16_t sample : resamBuffer)
samples.push(sample);
}
}
playSpeaker(speaker, channels);
}

2
source/frontend/StarVoice.hpp
View File

@ -80,6 +80,8 @@ public:
VoiceAudioStreamPtr audioStream;
Mutex mutex;
int64_t lastReceiveTime = 0;
atomic<bool> muted = false;
atomic<bool> playing = false;
atomic<float> decibelLevel = 0.0f;

31
source/game/StarWorldClient.cpp
View File

@ -795,20 +795,18 @@ void WorldClient::handleIncomingPackets(List<PacketPtr> const& packets) {
m_damageManager->pushRemoteDamageRequest(damage->remoteDamageRequest);
} else if (auto damage = as<DamageNotificationPacket>(packet)) {
auto& materialKind = damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8();
const size_t prefixSize = SECRET_BROADCAST_PREFIX.size();
const size_t signatureSize = Curve25519::SignatureSize;
const size_t dataSize = prefixSize + signatureSize;
std::string_view view(damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8());
static const size_t FULL_SIZE = SECRET_BROADCAST_PREFIX.size() + Curve25519::SignatureSize;
static const std::string LEGACY_VOICE_PREFIX = "data\0voice\0"s;
if (materialKind.size() >= dataSize && materialKind.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) {
if (view.size() >= FULL_SIZE && view.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) {
// this is actually a secret broadcast!!
if (auto player = m_entityMap->get<Player>(damage->remoteDamageNotification.sourceEntityId)) {
if (auto publicKey = player->getSecretPropertyView(SECRET_BROADCAST_PUBLIC_KEY)) {
if (publicKey->utf8Size() == Curve25519::PublicKeySize) {
std::string_view broadcast(materialKind);
auto signature = broadcast.substr(prefixSize, signatureSize);
auto signature = view.substr(SECRET_BROADCAST_PREFIX.size(), Curve25519::SignatureSize);
auto rawBroadcast = broadcast.substr(dataSize);
auto rawBroadcast = view.substr(FULL_SIZE);
if (Curve25519::verify(
(uint8_t const*)signature.data(),
(uint8_t const*)publicKey->utf8Ptr(),
@ -821,6 +819,23 @@ void WorldClient::handleIncomingPackets(List<PacketPtr> const& packets) {
}
}
}
else if (view.size() > 75 && view.rfind(LEGACY_VOICE_PREFIX, 0) != NPos) {
// this is a StarExtensions voice packet
// (remove this and stop transmitting like this once most SE features are ported over)
if (auto player = m_entityMap->get<Player>(damage->remoteDamageNotification.sourceEntityId)) {
if (auto publicKey = player->effectsAnimator()->globalTagPtr("\0SE_VOICE_SIGNING_KEY"s)) {
auto rawData = view.substr(75);
if (m_broadcastCallback && Curve25519::verify(
(uint8_t const*)view.data() + LEGACY_VOICE_PREFIX.size(),
(uint8_t const*)publicKey->utf8Ptr(),
(void*)rawData.data(),
rawData.size()
)) {
m_broadcastCallback(player, "Voice\0"s + rawData);
}
}
}
}
else {
m_damageManager->pushRemoteDamageNotification(damage->remoteDamageNotification);
}