From da098c7b4812408d1316b14b3b3f46d2ec7dce04 Mon Sep 17 00:00:00 2001 From: Kae <80987908+Novaenia@users.noreply.github.com> Date: Sun, 16 Jul 2023 20:44:15 +1000 Subject: [PATCH] Support receiving SE voice data, resample per-speaker again because of positional delay --- source/frontend/StarVoice.cpp | 106 +++++++++++++++++++++----------- source/frontend/StarVoice.hpp | 2 + source/game/StarWorldClient.cpp | 31 +++++++--- 3 files changed, 95 insertions(+), 44 deletions(-) diff --git a/source/frontend/StarVoice.cpp b/source/frontend/StarVoice.cpp index c424d30..4c0f4be 100644 --- a/source/frontend/StarVoice.cpp +++ b/source/frontend/StarVoice.cpp @@ -61,9 +61,31 @@ float getAudioLoudness(int16_t* data, size_t samples) { struct VoiceAudioStream { // TODO: This should really be a ring buffer instead. - std::vector samples; - + std::queue samples; + SDL_AudioStream* sdlAudioStream; Mutex mutex; + + VoiceAudioStream() : sdlAudioStream(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100)) {}; + ~VoiceAudioStream() { SDL_FreeAudioStream(sdlAudioStream); } + + inline int16_t take() { + int16_t sample = 0; + if (!samples.empty()) { + sample = samples.front(); + samples.pop(); + } + return sample; + } + + size_t resample(int16_t* in, size_t inSamples, std::vector& out) { + SDL_AudioStreamPut(sdlAudioStream, in, inSamples * sizeof(int16_t)); + if (int available = SDL_AudioStreamAvailable(sdlAudioStream)) { + out.resize(available / 2); + SDL_AudioStreamGet(sdlAudioStream, out.data(), available); + return available; + } + return 0; + } }; Voice::Speaker::Speaker(SpeakerId id) @@ -224,11 +246,11 @@ void Voice::readAudioData(uint8_t* stream, int len) { } void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) { - static std::vector finalBuffer; - static std::vector voiceBuffer; - static std::vector resampled; size_t samples = frameCount * channels; - resampled.resize(samples, 0); + static std::vector finalBuffer, speakerBuffer; + static std::vector sharedBuffer; //int32 to reduce clipping + speakerBuffer.resize(samples); + sharedBuffer.resize(samples); bool mix = false; { @@ -239,17 +261,21 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) { VoiceAudioStream* audio = speaker->audioStream.get(); MutexLocker audioLock(audio->mutex); if (!audio->samples.empty()) { - std::vector samples = move(audio->samples); - audioLock.unlock(); - speaker->decibelLevel = getAudioLoudness(samples.data(), samples.size()); + SDL_AudioStream* sdlStream = audio->sdlAudioStream; if (!speaker->muted) { mix = true; - if (voiceBuffer.size() < samples.size()) - voiceBuffer.resize(samples.size(), 0); + for (size_t i = 0; i != samples; ++i) + speakerBuffer[i] = audio->take(); + speaker->decibelLevel = getAudioLoudness(speakerBuffer.data(), samples); auto channelVolumes = speaker->channelVolumes.load(); - for (size_t i = 0; i != samples.size(); ++i) - voiceBuffer[i] += (int32_t)(samples[i]) * channelVolumes[i % 2]; + + for (size_t i = 0; i != samples; ++i) + sharedBuffer[i] += (int32_t)(speakerBuffer[i]) * channelVolumes[i % 2]; + } + else { + for (size_t i = 0; i != samples; ++i) + audio->take(); } ++it; } @@ -260,26 +286,16 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) { } } - static std::unique_ptr audioStream - (SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100), SDL_FreeAudioStream); - if (mix) { - finalBuffer.resize(voiceBuffer.size(), 0); + finalBuffer.resize(sharedBuffer.size(), 0); float vol = m_outputVolume; - for (size_t i = 0; i != voiceBuffer.size(); ++i) - finalBuffer[i] = (int16_t)clamp(voiceBuffer[i] * vol, INT16_MIN, INT16_MAX); + for (size_t i = 0; i != sharedBuffer.size(); ++i) + finalBuffer[i] = (int16_t)clamp(sharedBuffer[i] * vol, INT16_MIN, INT16_MAX); - SDL_AudioStreamPut(audioStream.get(), finalBuffer.data(), finalBuffer.size() * sizeof(int16_t)); + SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)finalBuffer.data(), AUDIO_S16, finalBuffer.size() * sizeof(int16_t), SDL_MIX_MAXVOLUME); + memset(sharedBuffer.data(), 0, sharedBuffer.size() * sizeof(int32_t)); } - - if (size_t available = min(samples * sizeof(int16_t), SDL_AudioStreamAvailable(audioStream.get()))) { - SDL_AudioStreamGet(audioStream.get(), resampled.data(), available); - SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)resampled.data(), AUDIO_S16, samples * sizeof(int16_t), SDL_MIX_MAXVOLUME); - } - - resampled.clear(); - voiceBuffer.clear(); } void Voice::update(PositionalAttenuationFunction positionalAttenuationFunction) { @@ -378,17 +394,35 @@ bool Voice::receive(SpeakerPtr speaker, std::string_view view) { //Logger::info("Voice: decoded Opus chunk {} bytes -> {} samples", opusLength, decodedSamples); { - MutexLocker lock(speaker->audioStream->mutex); + std::vector resamBuffer(decodedSamples, 0); + speaker->audioStream->resample(decodeBuffer, decodedSamples, resamBuffer); + + MutexLocker lock(speaker->audioStream->mutex); auto& samples = speaker->audioStream->samples; - if (mono) { - size_t prevSize = samples.size(); - samples.resize(prevSize + (size_t)decodedSamples * 2); - int16_t* data = samples.data() + prevSize; - for (int i = 0; i != decodedSamples; ++i) - *data++ = *data++ = decodeBuffer[i]; + + auto now = Time::monotonicMilliseconds(); + if (now - speaker->lastReceiveTime < 1000) { + auto limit = ((size_t)speaker->minimumPlaySamples + 22050) * (size_t)channels; + if (samples.size() > limit) { // skip ahead if we're getting too far + for (size_t i = samples.size(); i >= limit; --i) + samples.pop(); + } } else - samples.insert(samples.end(), decodeBuffer, decodeBuffer + decodedSamples); + samples = std::queue(); + + speaker->lastReceiveTime = now; + + if (mono) { + for (int16_t sample : resamBuffer) { + samples.push(sample); + samples.push(sample); + } + } + else { + for (int16_t sample : resamBuffer) + samples.push(sample); + } } playSpeaker(speaker, channels); } diff --git a/source/frontend/StarVoice.hpp b/source/frontend/StarVoice.hpp index 94ef8ac..3b95235 100644 --- a/source/frontend/StarVoice.hpp +++ b/source/frontend/StarVoice.hpp @@ -80,6 +80,8 @@ public: VoiceAudioStreamPtr audioStream; Mutex mutex; + int64_t lastReceiveTime = 0; + atomic muted = false; atomic playing = false; atomic decibelLevel = 0.0f; diff --git a/source/game/StarWorldClient.cpp b/source/game/StarWorldClient.cpp index 272b5df..54d4651 100644 --- a/source/game/StarWorldClient.cpp +++ b/source/game/StarWorldClient.cpp @@ -795,20 +795,18 @@ void WorldClient::handleIncomingPackets(List const& packets) { m_damageManager->pushRemoteDamageRequest(damage->remoteDamageRequest); } else if (auto damage = as(packet)) { - auto& materialKind = damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8(); - const size_t prefixSize = SECRET_BROADCAST_PREFIX.size(); - const size_t signatureSize = Curve25519::SignatureSize; - const size_t dataSize = prefixSize + signatureSize; + std::string_view view(damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8()); + static const size_t FULL_SIZE = SECRET_BROADCAST_PREFIX.size() + Curve25519::SignatureSize; + static const std::string LEGACY_VOICE_PREFIX = "data\0voice\0"s; - if (materialKind.size() >= dataSize && materialKind.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) { + if (view.size() >= FULL_SIZE && view.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) { // this is actually a secret broadcast!! if (auto player = m_entityMap->get(damage->remoteDamageNotification.sourceEntityId)) { if (auto publicKey = player->getSecretPropertyView(SECRET_BROADCAST_PUBLIC_KEY)) { if (publicKey->utf8Size() == Curve25519::PublicKeySize) { - std::string_view broadcast(materialKind); - auto signature = broadcast.substr(prefixSize, signatureSize); + auto signature = view.substr(SECRET_BROADCAST_PREFIX.size(), Curve25519::SignatureSize); - auto rawBroadcast = broadcast.substr(dataSize); + auto rawBroadcast = view.substr(FULL_SIZE); if (Curve25519::verify( (uint8_t const*)signature.data(), (uint8_t const*)publicKey->utf8Ptr(), @@ -821,6 +819,23 @@ void WorldClient::handleIncomingPackets(List const& packets) { } } } + else if (view.size() > 75 && view.rfind(LEGACY_VOICE_PREFIX, 0) != NPos) { + // this is a StarExtensions voice packet + // (remove this and stop transmitting like this once most SE features are ported over) + if (auto player = m_entityMap->get(damage->remoteDamageNotification.sourceEntityId)) { + if (auto publicKey = player->effectsAnimator()->globalTagPtr("\0SE_VOICE_SIGNING_KEY"s)) { + auto rawData = view.substr(75); + if (m_broadcastCallback && Curve25519::verify( + (uint8_t const*)view.data() + LEGACY_VOICE_PREFIX.size(), + (uint8_t const*)publicKey->utf8Ptr(), + (void*)rawData.data(), + rawData.size() + )) { + m_broadcastCallback(player, "Voice\0"s + rawData); + } + } + } + } else { m_damageManager->pushRemoteDamageNotification(damage->remoteDamageNotification); }