Support receiving SE voice data, resample per-speaker again because of positional delay

source/frontend/StarVoice.cpp

@@ -61,9 +61,31 @@ float getAudioLoudness(int16_t* data, size_t samples) {
 
 struct VoiceAudioStream {
   // TODO: This should really be a ring buffer instead.
-  std::vector<int16_t> samples;
+  std::queue<int16_t> samples;
-
+	SDL_AudioStream* sdlAudioStream;
   Mutex mutex;
+
+	VoiceAudioStream() : sdlAudioStream(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100)) {};
+	~VoiceAudioStream() { SDL_FreeAudioStream(sdlAudioStream); }
+
+	inline int16_t take() {
+		int16_t sample = 0;
+		if (!samples.empty()) {
+			sample = samples.front();
+			samples.pop();
+		}
+		return sample;
+	}
+
+	size_t resample(int16_t* in, size_t inSamples, std::vector<int16_t>& out) {
+		SDL_AudioStreamPut(sdlAudioStream, in, inSamples * sizeof(int16_t));
+		if (int available = SDL_AudioStreamAvailable(sdlAudioStream)) {
+			out.resize(available / 2);
+			SDL_AudioStreamGet(sdlAudioStream, out.data(), available);
+			return available;
+		}
+		return 0;
+	}
 };
 
 Voice::Speaker::Speaker(SpeakerId id)
@@ -224,11 +246,11 @@ void Voice::readAudioData(uint8_t* stream, int len) {
 }
 
 void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
-	static std::vector<int16_t> finalBuffer;
-	static std::vector<int32_t> voiceBuffer;
-	static std::vector<int16_t> resampled;
 	size_t samples = frameCount * channels;
-	resampled.resize(samples, 0);
+	static std::vector<int16_t> finalBuffer, speakerBuffer;
+	static std::vector<int32_t> sharedBuffer; //int32 to reduce clipping
+	speakerBuffer.resize(samples);
+	sharedBuffer.resize(samples);
 
 	bool mix = false;
 	{
@@ -239,17 +261,21 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
 			VoiceAudioStream* audio = speaker->audioStream.get();
 			MutexLocker audioLock(audio->mutex);
 			if (!audio->samples.empty()) {
-				std::vector<int16_t> samples = move(audio->samples);
+				SDL_AudioStream* sdlStream = audio->sdlAudioStream;
-				audioLock.unlock();
-				speaker->decibelLevel = getAudioLoudness(samples.data(), samples.size());
 				if (!speaker->muted) {
 					mix = true;
-					if (voiceBuffer.size() < samples.size())
+					for (size_t i = 0; i != samples; ++i)
-						voiceBuffer.resize(samples.size(), 0);
+						speakerBuffer[i] = audio->take();
 
+					speaker->decibelLevel = getAudioLoudness(speakerBuffer.data(), samples);
 					auto channelVolumes = speaker->channelVolumes.load();
-					for (size_t i = 0; i != samples.size(); ++i)
+
-						voiceBuffer[i] += (int32_t)(samples[i]) * channelVolumes[i % 2];
+					for (size_t i = 0; i != samples; ++i)
+						sharedBuffer[i] += (int32_t)(speakerBuffer[i]) * channelVolumes[i % 2];
+				}
+				else {
+					for (size_t i = 0; i != samples; ++i)
+						audio->take();
 				}
 				++it;
 			}
@@ -260,26 +286,16 @@ void Voice::mix(int16_t* buffer, size_t frameCount, unsigned channels) {
 		}
 	}
 
-	static std::unique_ptr<SDL_AudioStream, void(*)(SDL_AudioStream*)> audioStream
-	(SDL_NewAudioStream(AUDIO_S16, 2, 48000, AUDIO_S16SYS, 2, 44100), SDL_FreeAudioStream);
-
 	if (mix) {
-		finalBuffer.resize(voiceBuffer.size(), 0);
+		finalBuffer.resize(sharedBuffer.size(), 0);
 
 		float vol = m_outputVolume;
-		for (size_t i = 0; i != voiceBuffer.size(); ++i)
+		for (size_t i = 0; i != sharedBuffer.size(); ++i)
-			finalBuffer[i] = (int16_t)clamp<int>(voiceBuffer[i] * vol, INT16_MIN, INT16_MAX);
+			finalBuffer[i] = (int16_t)clamp<int>(sharedBuffer[i] * vol, INT16_MIN, INT16_MAX);
 
-		SDL_AudioStreamPut(audioStream.get(), finalBuffer.data(), finalBuffer.size() * sizeof(int16_t));
+		SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)finalBuffer.data(), AUDIO_S16, finalBuffer.size() * sizeof(int16_t), SDL_MIX_MAXVOLUME);
+		memset(sharedBuffer.data(), 0, sharedBuffer.size() * sizeof(int32_t));
 	}
-
-	if (size_t available = min<size_t>(samples * sizeof(int16_t), SDL_AudioStreamAvailable(audioStream.get()))) {
-		SDL_AudioStreamGet(audioStream.get(), resampled.data(), available);
-		SDL_MixAudioFormat((Uint8*)buffer, (Uint8*)resampled.data(), AUDIO_S16, samples * sizeof(int16_t), SDL_MIX_MAXVOLUME);
-	}
-
-	resampled.clear();
-	voiceBuffer.clear();
 }
 
 void Voice::update(PositionalAttenuationFunction positionalAttenuationFunction) {
@@ -378,17 +394,35 @@ bool Voice::receive(SpeakerPtr speaker, std::string_view view) {
 			//Logger::info("Voice: decoded Opus chunk {} bytes -> {} samples", opusLength, decodedSamples);
 
 			{
+				std::vector<int16_t> resamBuffer(decodedSamples, 0);
+				speaker->audioStream->resample(decodeBuffer, decodedSamples, resamBuffer);
+
 			  MutexLocker lock(speaker->audioStream->mutex);
 				auto& samples = speaker->audioStream->samples;
-				if (mono) {
+
-					size_t prevSize = samples.size();
+				auto now = Time::monotonicMilliseconds();
-					samples.resize(prevSize + (size_t)decodedSamples * 2);
+				if (now - speaker->lastReceiveTime < 1000) {
-					int16_t* data = samples.data() + prevSize;
+					auto limit = ((size_t)speaker->minimumPlaySamples + 22050) * (size_t)channels;
-					for (int i = 0; i != decodedSamples; ++i)
+					if (samples.size() > limit) { // skip ahead if we're getting too far
-						*data++ = *data++ = decodeBuffer[i];
+						for (size_t i = samples.size(); i >= limit; --i)
+							samples.pop();
+					}
 				}
 				else
-					samples.insert(samples.end(), decodeBuffer, decodeBuffer + decodedSamples);
+					samples = std::queue<int16_t>();
+
+				speaker->lastReceiveTime = now;
+
+				if (mono) {
+					for (int16_t sample : resamBuffer) {
+						samples.push(sample);
+						samples.push(sample);
+					}
+				}
+				else {
+					for (int16_t sample : resamBuffer)
+						samples.push(sample);
+				}
 			}
 			playSpeaker(speaker, channels);
 		}

source/frontend/StarVoice.hpp

@@ -80,6 +80,8 @@ public:
     VoiceAudioStreamPtr audioStream;
     Mutex mutex;
 
+    int64_t lastReceiveTime = 0;
+
     atomic<bool> muted = false;
     atomic<bool> playing = false;
     atomic<float> decibelLevel = 0.0f;

source/game/StarWorldClient.cpp

@@ -795,20 +795,18 @@ void WorldClient::handleIncomingPackets(List<PacketPtr> const& packets) {
       m_damageManager->pushRemoteDamageRequest(damage->remoteDamageRequest);
 
     } else if (auto damage = as<DamageNotificationPacket>(packet)) {
-      auto& materialKind = damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8();
+      std::string_view view(damage->remoteDamageNotification.damageNotification.targetMaterialKind.utf8());
-      const size_t prefixSize = SECRET_BROADCAST_PREFIX.size();
+      static const size_t FULL_SIZE = SECRET_BROADCAST_PREFIX.size() + Curve25519::SignatureSize;
-      const size_t signatureSize = Curve25519::SignatureSize;
+      static const std::string LEGACY_VOICE_PREFIX = "data\0voice\0"s;
-      const size_t dataSize = prefixSize + signatureSize;
 
-      if (materialKind.size() >= dataSize && materialKind.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) {
+      if (view.size() >= FULL_SIZE && view.rfind(SECRET_BROADCAST_PREFIX, 0) != NPos) {
         // this is actually a secret broadcast!!
         if (auto player = m_entityMap->get<Player>(damage->remoteDamageNotification.sourceEntityId)) {
           if (auto publicKey = player->getSecretPropertyView(SECRET_BROADCAST_PUBLIC_KEY)) {
             if (publicKey->utf8Size() == Curve25519::PublicKeySize) {
-              std::string_view broadcast(materialKind);
+              auto signature = view.substr(SECRET_BROADCAST_PREFIX.size(), Curve25519::SignatureSize);
-              auto signature = broadcast.substr(prefixSize, signatureSize);
 
-              auto rawBroadcast = broadcast.substr(dataSize);
+              auto rawBroadcast = view.substr(FULL_SIZE);
               if (Curve25519::verify(
                 (uint8_t const*)signature.data(),
                 (uint8_t const*)publicKey->utf8Ptr(),
@@ -821,6 +819,23 @@ void WorldClient::handleIncomingPackets(List<PacketPtr> const& packets) {
           }
         }
       }
+      else if (view.size() > 75 && view.rfind(LEGACY_VOICE_PREFIX, 0) != NPos) {
+        // this is a StarExtensions voice packet
+        // (remove this and stop transmitting like this once most SE features are ported over)
+        if (auto player = m_entityMap->get<Player>(damage->remoteDamageNotification.sourceEntityId)) {
+          if (auto publicKey = player->effectsAnimator()->globalTagPtr("\0SE_VOICE_SIGNING_KEY"s)) {
+            auto rawData = view.substr(75);
+            if (m_broadcastCallback && Curve25519::verify(
+              (uint8_t const*)view.data() + LEGACY_VOICE_PREFIX.size(),
+              (uint8_t const*)publicKey->utf8Ptr(),
+              (void*)rawData.data(),
+                     rawData.size()
+            )) {
+              m_broadcastCallback(player, "Voice\0"s + rawData);
+            }
+          }
+        }
+      }
       else {
         m_damageManager->pushRemoteDamageNotification(damage->remoteDamageNotification);
       }