obs-webrtc: Add Simulcast Support

Author: Sean-Der

UI/data/locale/en-US.ini

@@ -956,6 +956,7 @@ Basic.Settings.Stream.StreamSettingsWarning="Open Settings"
 Basic.Settings.Stream.MissingUrlAndApiKey="URL and Stream Key are missing.\n\nOpen settings to enter the URL and Stream Key in the 'stream' tab."
 Basic.Settings.Stream.MissingUrl="Stream URL is missing.\n\nOpen settings to enter the URL in the 'Stream' tab."
 Basic.Settings.Stream.MissingStreamKey="Stream key is missing.\n\nOpen settings to enter the stream key in the 'Stream' tab."
+Basic.Settings.Stream.UseSimulcast="Use Simulcast"
 Basic.Settings.Stream.IgnoreRecommended="Ignore streaming service setting recommendations"
 Basic.Settings.Stream.IgnoreRecommended.Warn.Title="Override Recommended Settings"
 Basic.Settings.Stream.IgnoreRecommended.Warn.Text="Warning: Ignoring the service's limitations may result in degraded stream quality or prevent you from streaming.\n\nContinue?"

UI/forms/OBSBasicSettings.ui

@@ -1926,6 +1926,13 @@
                    </property>
                   </widget>
                  </item>
+                 <item row="4" column="1">
+                  <widget class="QCheckBox" name="useSimulcast">
+                   <property name="text">
+                    <string>Basic.Settings.Stream.UseSimulcast</string>
+                   </property>
+                  </widget>
+                 </item>
                 </layout>
                </widget>
               </item>

UI/window-basic-main-outputs.cpp

@@ -588,6 +588,10 @@ void SimpleOutput::LoadStreamingPreset_Lossy(const char *encoderId)
 	if (!videoStreaming)
 		throw "Failed to create video streaming encoder (simple output)";
 	obs_encoder_release(videoStreaming);
+
+	if (config_get_bool(main->Config(), "Stream1", "UseSimulcast")) {
+		CreateSimulcastEncoders(encoderId);
+	}
 }
 
 /* mistakes have been made to lead us to this. */
@@ -890,9 +894,14 @@ void SimpleOutput::Update()
 	default:
 		obs_encoder_set_preferred_video_format(videoStreaming,
 						       VIDEO_FORMAT_NV12);
+		for (auto enc : simulcastEncoders)
+			obs_encoder_set_preferred_video_format(
+				enc, VIDEO_FORMAT_NV12);
 	}
 
 	obs_encoder_update(videoStreaming, videoSettings);
+	SimulcastEncodersUpdate(videoSettings, videoBitrate);
+
 	obs_encoder_update(audioStreaming, audioSettings);
 	obs_encoder_update(audioArchive, audioSettings);
 }
@@ -1204,6 +1213,9 @@ SimpleOutput::SetupStreaming(obs_service_t *service,
 		}
 
 		obs_output_set_video_encoder(streamOutput, videoStreaming);
+		for (size_t i = 0; i < simulcastEncoders.size(); i++)
+			obs_output_set_video_encoder2(
+				streamOutput, simulcastEncoders[i], i + 1);
 		obs_output_set_audio_encoder(streamOutput, audioStreaming, 0);
 		obs_output_set_service(streamOutput, service);
 		return true;
@@ -1747,6 +1759,10 @@ AdvancedOutput::AdvancedOutput(OBSBasic *main_) : BasicOutputHandler(main_)
 		      "(advanced output)";
 	obs_encoder_release(videoStreaming);
 
+	if (config_get_bool(main->Config(), "Stream1", "UseSimulcast")) {
+		CreateSimulcastEncoders(streamEncoder);
+	}
+
 	const char *rate_control = obs_data_get_string(
 		useStreamEncoder ? streamEncSettings : recordEncSettings,
 		"rate_control");
@@ -1867,6 +1883,8 @@ void AdvancedOutput::UpdateStreamSettings()
 	}
 
 	obs_encoder_update(videoStreaming, settings);
+	SimulcastEncodersUpdate(settings,
+				obs_data_get_int(settings, "bitrate"));
 }
 
 inline void AdvancedOutput::UpdateRecordingSettings()
@@ -2351,6 +2369,9 @@ AdvancedOutput::SetupStreaming(obs_service_t *service,
 		}
 
 		obs_output_set_video_encoder(streamOutput, videoStreaming);
+		for (size_t i = 0; i < simulcastEncoders.size(); i++)
+			obs_output_set_video_encoder2(
+				streamOutput, simulcastEncoders[i], i + 1);
 		obs_output_set_audio_encoder(streamOutput, streamAudioEnc, 0);
 
 		if (!is_multitrack_output) {
@@ -2900,3 +2921,52 @@ BasicOutputHandler *CreateAdvancedOutputHandler(OBSBasic *main)
 {
 	return new AdvancedOutput(main);
 }
+
+void BasicOutputHandler::CreateSimulcastEncoders(const char *encoderId)
+{
+	int rescaleFilter =
+		config_get_int(main->Config(), "AdvOut", "RescaleFilter");
+	if (rescaleFilter == OBS_SCALE_DISABLE) {
+		rescaleFilter = OBS_SCALE_BICUBIC;
+	}
+
+	std::string encoder_name = "simulcast_0";
+	for (auto i = 0; i < 2; i++) {
+		uint32_t width = video_output_get_width(obs_get_video()) /
+				 (1.5 + (.5 * i));
+		width -= width % 2;
+
+		uint32_t height = video_output_get_height(obs_get_video()) /
+				  (1.5 + (.5 * i));
+		height -= height % 2;
+
+		encoder_name[encoder_name.size() - 1] = to_string(i).at(0);
+		auto simulcast_encoder = obs_video_encoder_create(
+			encoderId, encoder_name.c_str(), nullptr, nullptr);
+
+		if (simulcast_encoder) {
+			obs_encoder_set_video(simulcast_encoder,
+					      obs_get_video());
+			obs_encoder_set_scaled_size(simulcast_encoder, width,
+						    height);
+			obs_encoder_set_gpu_scale_type(
+				simulcast_encoder,
+				(obs_scale_type)rescaleFilter);
+			simulcastEncoders.push_back(simulcast_encoder);
+			obs_encoder_release(simulcast_encoder);
+		} else {
+			blog(LOG_WARNING,
+			     "Failed to create video streaming simulcast encoders (BasicOutputHandler)");
+		}
+	}
+}
+
+void BasicOutputHandler::SimulcastEncodersUpdate(obs_data_t *videoSettings,
+						 int videoBitrate)
+{
+	for (size_t i = 0; i < simulcastEncoders.size(); i++) {
+		obs_data_set_int(videoSettings, "bitrate",
+				 videoBitrate / (2 * (i + 1)));
+		obs_encoder_update(simulcastEncoders[i], videoSettings);
+	}
+}

UI/window-basic-main-outputs.hpp

@@ -38,6 +38,8 @@ struct BasicOutputHandler {
 	obs_scene_t *vCamSourceScene = nullptr;
 	obs_sceneitem_t *vCamSourceSceneItem = nullptr;
 
+	std::vector<OBSEncoder> simulcastEncoders;
+
 	std::string outputType;
 	std::string lastError;
 
@@ -105,6 +107,9 @@ struct BasicOutputHandler {
 		size_t main_audio_mixer, std::optional<size_t> vod_track_mixer,
 		std::function<void(std::optional<bool>)> continuation);
 	OBSDataAutoRelease GenerateMultitrackVideoStreamDumpConfig();
+	void CreateSimulcastEncoders(const char *encoderId);
+	void SimulcastEncodersUpdate(obs_data_t *videoSettings,
+				     int videoBitrate);
 };
 
 BasicOutputHandler *CreateSimpleOutputHandler(OBSBasic *main);

UI/window-basic-settings-stream.cpp

@@ -112,6 +112,8 @@ void OBSBasicSettings::LoadStream1Settings()
 {
 	bool ignoreRecommended =
 		config_get_bool(main->Config(), "Stream1", "IgnoreRecommended");
+	bool useSimulcast =
+		config_get_bool(main->Config(), "Stream1", "UseSimulcast");
 
 	obs_service_t *service_obj = main->GetService();
 	const char *type = obs_service_get_type(service_obj);
@@ -224,10 +226,13 @@ void OBSBasicSettings::LoadStream1Settings()
 	if (use_custom_server)
 		ui->serviceCustomServer->setText(server);
 
-	if (is_whip)
+	if (is_whip) {
 		ui->key->setText(bearer_token);
-	else
+		ui->useSimulcast->show();
+	} else {
 		ui->key->setText(key);
+		ui->useSimulcast->hide();
+	}
 
 	ServiceChanged(true);
 
@@ -241,6 +246,7 @@ void OBSBasicSettings::LoadStream1Settings()
 	ui->streamPage->setEnabled(!streamActive);
 
 	ui->ignoreRecommended->setChecked(ignoreRecommended);
+	ui->useSimulcast->setChecked(useSimulcast);
 
 	loading = false;
 
@@ -365,6 +371,10 @@ void OBSBasicSettings::SaveStream1Settings()
 
 	SaveCheckBox(ui->ignoreRecommended, "Stream1", "IgnoreRecommended");
 
+	auto oldSimulcastSetting =
+		config_get_bool(main->Config(), "Stream1", "UseSimulcast");
+	SaveCheckBox(ui->useSimulcast, "Stream1", "UseSimulcast");
+
 	auto oldMultitrackVideoSetting = config_get_bool(
 		main->Config(), "Stream1", "EnableMultitrackVideo");
 
@@ -404,7 +414,9 @@ void OBSBasicSettings::SaveStream1Settings()
 	SaveText(ui->multitrackVideoConfigOverride, "Stream1",
 		 "MultitrackVideoConfigOverride");
 
-	if (oldMultitrackVideoSetting != ui->enableMultitrackVideo->isChecked())
+	if (oldMultitrackVideoSetting !=
+		    ui->enableMultitrackVideo->isChecked() ||
+	    oldSimulcastSetting != ui->useSimulcast->isChecked())
 		main->ResetOutputs();
 
 	SwapMultiTrack(QT_TO_UTF8(protocol));
@@ -661,6 +673,12 @@ void OBSBasicSettings::on_service_currentIndexChanged(int idx)
 	} else {
 		SwapMultiTrack(QT_TO_UTF8(protocol));
 	}
+
+	if (IsWHIP()) {
+		ui->useSimulcast->show();
+	} else {
+		ui->useSimulcast->hide();
+	}
 }
 
 void OBSBasicSettings::on_customServer_textChanged(const QString &)

UI/window-basic-settings.cpp

@@ -420,6 +420,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
 	HookWidget(ui->authUsername,         EDIT_CHANGED,   STREAM1_CHANGED);
 	HookWidget(ui->authPw,               EDIT_CHANGED,   STREAM1_CHANGED);
 	HookWidget(ui->ignoreRecommended,    CHECK_CHANGED,  STREAM1_CHANGED);
+	HookWidget(ui->useSimulcast,         CHECK_CHANGED,  STREAM1_CHANGED);
 	HookWidget(ui->enableMultitrackVideo,      CHECK_CHANGED,  STREAM1_CHANGED);
 	HookWidget(ui->multitrackVideoMaximumAggregateBitrateAuto, CHECK_CHANGED,  STREAM1_CHANGED);
 	HookWidget(ui->multitrackVideoMaximumAggregateBitrate,     SCROLL_CHANGED, STREAM1_CHANGED);

plugins/obs-webrtc/whip-output.cpp

@@ -24,6 +24,14 @@ const uint8_t video_payload_type = 96;
 // ~3 seconds of 8.5 Megabit video
 const int video_nack_buffer_size = 4000;
 
+const std::string rtpHeaderExtUriMid = "urn:ietf:params:rtp-hdrext:sdes:mid";
+const std::string rtpHeaderExtUriRid =
+	"urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id";
+
+const std::string highRid = "h";
+const std::string medRid = "m";
+const std::string lowRid = "l";
+
 WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output)
 	: output(output),
 	  endpoint_url(),
@@ -39,8 +47,7 @@ WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output)
 	  total_bytes_sent(0),
 	  connect_time_ms(0),
 	  start_time_ns(0),
-	  last_audio_timestamp(0),
-	  last_video_timestamp(0)
+	  last_audio_timestamp(0)
 {
 }
 
@@ -57,6 +64,27 @@ bool WHIPOutput::Start()
 {
 	std::lock_guard<std::mutex> l(start_stop_mutex);
 
+	for (size_t idx = 0; idx < 5; idx++) {
+		auto encoder = obs_output_get_video_encoder2(output, idx);
+		if (encoder == nullptr) {
+			break;
+		}
+
+		auto v = std::make_shared<videoLayerState>();
+		if (idx == 0) {
+			v->ssrc = base_ssrc + 1;
+			v->rid = highRid;
+		} else if (idx == 1) {
+			v->ssrc = base_ssrc + 2;
+			v->rid = medRid;
+		} else if (idx == 2) {
+			v->ssrc = base_ssrc + 3;
+			v->rid = lowRid;
+		}
+
+		videoLayerStates[obs_encoder_get_width(encoder)] = v;
+	}
+
 	if (!obs_output_can_begin_data_capture(output, 0))
 		return false;
 	if (!obs_output_initialize_encoders(output, 0))
@@ -92,10 +120,28 @@ void WHIPOutput::Data(struct encoder_packet *packet)
 		     audio_sr_reporter);
 		last_audio_timestamp = packet->dts_usec;
 	} else if (video_track && packet->type == OBS_ENCODER_VIDEO) {
-		int64_t duration = packet->dts_usec - last_video_timestamp;
+		auto rtp_config = video_sr_reporter->rtpConfig;
+		auto videoLayerState =
+			videoLayerStates[obs_encoder_get_width(packet->encoder)];
+		if (videoLayerState == nullptr) {
+			Stop(false);
+			obs_output_signal_stop(output, OBS_OUTPUT_ENCODE_ERROR);
+			return;
+		}
+
+		rtp_config->sequenceNumber = videoLayerState->sequenceNumber;
+		rtp_config->ssrc = videoLayerState->ssrc;
+		rtp_config->rid = videoLayerState->rid;
+		rtp_config->timestamp = videoLayerState->rtpTimestamp;
+		int64_t duration =
+			packet->dts_usec - videoLayerState->lastVideoTimestamp;
+
 		Send(packet->data, packet->size, duration, video_track,
 		     video_sr_reporter);
-		last_video_timestamp = packet->dts_usec;
+
+		videoLayerState->sequenceNumber = rtp_config->sequenceNumber;
+		videoLayerState->lastVideoTimestamp = packet->dts_usec;
+		videoLayerState->rtpTimestamp = rtp_config->timestamp;
 	}
 }
 
@@ -151,9 +197,24 @@ void WHIPOutput::ConfigureVideoTrack(std::string media_stream_id,
 	video_description.addSSRC(ssrc, cname, media_stream_id,
 				  media_stream_track_id);
 
+	video_description.addExtMap(
+		rtc::Description::Entry::ExtMap(1, rtpHeaderExtUriMid));
+	video_description.addExtMap(
+		rtc::Description::Entry::ExtMap(2, rtpHeaderExtUriRid));
+
+	if (videoLayerStates.size() >= 2) {
+		for (auto i = videoLayerStates.rbegin();
+		     i != videoLayerStates.rend(); i++) {
+			video_description.addRid(i->second->rid);
+		}
+	}
+
 	auto rtp_config = std::make_shared<rtc::RtpPacketizationConfig>(
 		ssrc, cname, video_payload_type,
 		rtc::H264RtpPacketizer::defaultClockRate);
+	rtp_config->midId = 1;
+	rtp_config->ridId = 2;
+	rtp_config->mid = video_mid;
 
 	const obs_encoder_t *encoder = obs_output_get_video_encoder2(output, 0);
 	if (!encoder)
@@ -653,7 +714,7 @@ void WHIPOutput::StopThread(bool signal)
 	connect_time_ms = 0;
 	start_time_ns = 0;
 	last_audio_timestamp = 0;
-	last_video_timestamp = 0;
+	videoLayerStates.clear();
 }
 
 void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration,
@@ -697,7 +758,8 @@ void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration,
 
 void register_whip_output()
 {
-	const uint32_t base_flags = OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE;
+	const uint32_t base_flags = OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE |
+				    OBS_OUTPUT_MULTI_TRACK_AV;
 
 	const char *audio_codecs = "opus";
 #ifdef ENABLE_HEVC

plugins/obs-webrtc/whip-output.h

@@ -13,6 +13,14 @@
 
 #include <rtc/rtc.hpp>
 
+struct videoLayerState {
+	uint16_t sequenceNumber;
+	uint32_t rtpTimestamp;
+	int64_t lastVideoTimestamp;
+	uint32_t ssrc;
+	std::string rid;
+};
+
 class WHIPOutput {
 public:
 	WHIPOutput(obs_data_t *settings, obs_output_t *output);
@@ -62,11 +70,12 @@ class WHIPOutput {
 	std::shared_ptr<rtc::RtcpSrReporter> audio_sr_reporter;
 	std::shared_ptr<rtc::RtcpSrReporter> video_sr_reporter;
 
+	std::map<uint32_t, std::shared_ptr<videoLayerState>> videoLayerStates;
+
 	std::atomic<size_t> total_bytes_sent;
 	std::atomic<int> connect_time_ms;
 	int64_t start_time_ns;
 	int64_t last_audio_timestamp;
-	int64_t last_video_timestamp;
 };
 
 void register_whip_output();