WebRTC 的音频数据编码及发送控制管线

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
-----------------------------     --------------------------     ---------------------------
|                           |     |                        | ==> | webrtc::AudioProcessing |
| webrtc::AudioDeviceModule | ==> | webrtc::AudioTransport |     ---------------------------
|                           |     |                        |          ｜｜
-----------------------------     --------------------------          ｜｜
                                                                      ｜｜
                                          +=+=========================+=+
                                          | |
                                           /
                                           |
                    -----------------------------------------------     ---------------------
                    | webrtc::AudioSender/webrtc::AudioSendStream | ==> | webrtc::Transport |
                    -----------------------------------------------     ---------------------
                                                                                ｜｜
                                                                                 /
                                                 -------------------------------------------
                                                 | cricket::MediaChannel::NetworkInterface |
                                                 -------------------------------------------

1
2
3
4
5
6
7
-----------------------------     ---------------------------     ---------------------------     -------------------------------------------
|                           |     |                         |     | webrtc::AudioSender/    |     |                                         |
| webrtc::AudioDeviceModule | ==> | webrtc::AudioProcessing | ==> | webrtc::AudioSendStream | ==> | cricket::MediaChannel::NetworkInterface |
|                           |     |                         |     |                         |     |                                         |
-----------------------------     ---------------------------     ---------------------------     -------------------------------------------
                                                 

1
2
3
4
5
6
7
#0  webrtc::AudioTransportImpl::UpdateAudioSenders(std::vector<webrtc::AudioSender*, std::allocator<webrtc::AudioSender*> >, int, unsigned long) ()
    at webrtc/audio/audio_transport_impl.cc:262
#1  webrtc::internal::AudioState::UpdateAudioTransportWithSendingStreams() () at webrtc/audio/audio_state.cc:172
#2  webrtc::internal::AudioState::AddSendingStream(webrtc::AudioSendStream*, int, unsigned long) ()
    at webrtc/audio/audio_state.cc:100
#3  webrtc::internal::AudioSendStream::Start() () at webrtc/audio/audio_send_stream.cc:370

1
2
3
4
5
#0  webrtc::voe::(anonymous namespace)::ChannelSend::ChannelSend(webrtc::Clock*, webrtc::TaskQueueFactory*, webrtc::Transport*, webrtc::RtcpRttStats*, webrtc::RtcEventLog*, webrtc::FrameEncryptorInterface*, webrtc::CryptoOptions const&, bool, int, unsigned int, rtc::scoped_refptr<webrtc::FrameTransformerInterface>, webrtc::TransportFeedbackObserver*) () at webrtc/audio/channel_send.cc:450
#2  webrtc::voe::CreateChannelSend(webrtc::Clock*, webrtc::TaskQueueFactory*, webrtc::Transport*, webrtc::RtcpRttStats*, webrtc::RtcEventLog*, webrtc::FrameEncryptorInterface*, webrtc::CryptoOptions const&, bool, int, unsigned int, rtc::scoped_refptr<webrtc::FrameTransformerInterface>, webrtc::TransportFeedbackObserver*) () at webrtc/audio/channel_send.cc:953
#3  webrtc::internal::AudioSendStream::AudioSendStream(webrtc::Clock*, webrtc::AudioSendStream::Config const&, rtc::scoped_refptr<webrtc::AudioState> const&, webrtc::TaskQueueFactory*, webrtc::RtpTransportControllerSendInterface*, webrtc::BitrateAllocatorInterface*, webrtc::RtcEventLog*, webrtc::RtcpRttStats*, absl::optional<webrtc::RtpState> const&) () at webrtc/audio/audio_send_stream.cc:118
#4  webrtc::internal::Call::CreateAudioSendStream(webrtc::AudioSendStream::Config const&) () at webrtc/call/call.cc:897

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
ChannelSend::ChannelSend(
    Clock* clock,
    TaskQueueFactory* task_queue_factory,
    Transport* rtp_transport,
    RtcpRttStats* rtcp_rtt_stats,
    RtcEventLog* rtc_event_log,
    FrameEncryptorInterface* frame_encryptor,
    const webrtc::CryptoOptions& crypto_options,
    bool extmap_allow_mixed,
    int rtcp_report_interval_ms,
    uint32_t ssrc,
    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
    TransportFeedbackObserver* feedback_observer)
    : ssrc_(ssrc),
      event_log_(rtc_event_log),
      _timeStamp(0),  // This is just an offset, RTP module will add it's own
                      // random offset
      input_mute_(false),
      previous_frame_muted_(false),
      _includeAudioLevelIndication(false),
      rtcp_observer_(new VoERtcpObserver(this)),
      feedback_observer_(feedback_observer),
      rtp_packet_pacer_proxy_(new RtpPacketSenderProxy()),
      retransmission_rate_limiter_(
          new RateLimiter(clock, kMaxRetransmissionWindowMs)),
      frame_encryptor_(frame_encryptor),
      crypto_options_(crypto_options),
      encoder_queue_(task_queue_factory->CreateTaskQueue(
          "AudioEncoder",
          TaskQueueFactory::Priority::NORMAL)),
      fixing_timestamp_stall_(
          !field_trial::IsDisabled("WebRTC-Audio-FixTimestampStall")) {
  audio_coding_.reset(AudioCodingModule::Create(AudioCodingModule::Config()));

  RtpRtcpInterface::Configuration configuration;
  configuration.bandwidth_callback = rtcp_observer_.get();
  configuration.transport_feedback_callback = feedback_observer_;
  configuration.clock = (clock ? clock : Clock::GetRealTimeClock());
  configuration.audio = true;
  configuration.outgoing_transport = rtp_transport;

  configuration.paced_sender = rtp_packet_pacer_proxy_.get();

  configuration.event_log = event_log_;
  configuration.rtt_stats = rtcp_rtt_stats;
  configuration.retransmission_rate_limiter =
      retransmission_rate_limiter_.get();
  configuration.extmap_allow_mixed = extmap_allow_mixed;
  configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;
  configuration.rtcp_packet_type_counter_observer = this;

  configuration.local_media_ssrc = ssrc;

  rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(configuration);
  rtp_rtcp_->SetSendingMediaStatus(false);

  rtp_sender_audio_ = std::make_unique<RTPSenderAudio>(configuration.clock,
                                                       rtp_rtcp_->RtpSender());

  // Ensure that RTCP is enabled by default for the created channel.
  rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound);

  int error = audio_coding_->RegisterTransportCallback(this);
  RTC_DCHECK_EQ(0, error);
  if (frame_transformer)
    InitFrameTransformerDelegate(std::move(frame_transformer));
}

1
2
3
4
5
6
7
#0  webrtc::voe::(anonymous namespace)::ChannelSend::RegisterSenderCongestionControlObjects(webrtc::RtpTransportControllerSendInterface*, webrtc::RtcpBandwidthObserver*) () at webrtc/audio/channel_send.cc:713
#1  webrtc::internal::AudioSendStream::ConfigureStream(webrtc::AudioSendStream::Config const&, bool) ()
    at webrtc/audio/audio_send_stream.cc:303
#2  webrtc::internal::AudioSendStream::AudioSendStream(webrtc::Clock*, webrtc::AudioSendStream::Config const&, rtc::scoped_refptr<webrtc::AudioState> const&, webrtc::TaskQueueFactory*, webrtc::RtpTransportControllerSendInterface*, webrtc::BitrateAllocatorInterface*, webrtc::RtcEventLog*, absl::optional<webrtc::RtpState> const&, std::unique_ptr<webrtc::voe::ChannelSendInterface, std::default_delete<webrtc::voe::ChannelSendInterface> >) ()
    at webrtc/audio/audio_send_stream.cc:170
#3  webrtc::internal::AudioSendStream::AudioSendStream(webrtc::Clock*, webrtc::AudioSendStream::Config const&, rtc::scoped_refptr<webrtc::AudioState> const&, webrtc::TaskQueueFactory*, webrtc::RtpTransportControllerSendInterface*, webrtc::BitrateAllocatorInterface*, webrtc::RtcEventLog*, webrtc::RtcpRttStats*, absl::optional<webrtc::RtpState> const&) () at webrtc/audio/audio_send_stream.cc:110

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
void ChannelSend::RegisterSenderCongestionControlObjects(
    RtpTransportControllerSendInterface* transport,
    RtcpBandwidthObserver* bandwidth_observer) {
  RTC_DCHECK_RUN_ON(&worker_thread_checker_);
  RtpPacketSender* rtp_packet_pacer = transport->packet_sender();
  PacketRouter* packet_router = transport->packet_router();

  RTC_DCHECK(rtp_packet_pacer);
  RTC_DCHECK(packet_router);
  RTC_DCHECK(!packet_router_);
  rtcp_observer_->SetBandwidthObserver(bandwidth_observer);
  rtp_packet_pacer_proxy_->SetPacketPacer(rtp_packet_pacer);
  rtp_rtcp_->SetStorePacketsStatus(true, 600);
  packet_router_ = packet_router;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#0  webrtc::(anonymous namespace)::AudioCodingModuleImpl::ModifyEncoder(rtc::FunctionView<void (std::unique_ptr<webrtc::AudioEncoder, std::default_delete<webrtc::AudioEncoder> >*)>) () at webrtc/modules/audio_coding/acm2/audio_coding_module.cc:320
#1  webrtc::AudioCodingModule::SetEncoder(std::unique_ptr<webrtc::AudioEncoder, std::default_delete<webrtc::AudioEncoder> >) ()
    at webrtc/modules/audio_coding/include/audio_coding_module.h:96
#2  webrtc::voe::(anonymous namespace)::ChannelSend::SetEncoder(int, std::unique_ptr<webrtc::AudioEncoder, std::default_delete<webrtc::AudioEncoder> >) () at webrtc/audio/channel_send.cc:588
#3  webrtc::internal::AudioSendStream::SetupSendCodec(webrtc::AudioSendStream::Config const&) ()
    at webrtc/audio/audio_send_stream.cc:657
#4  webrtc::internal::AudioSendStream::ReconfigureSendCodec(webrtc::AudioSendStream::Config const&) ()
    at webrtc/audio/audio_send_stream.cc:688
#5  webrtc::internal::AudioSendStream::ConfigureStream(webrtc::AudioSendStream::Config const&, bool) ()
    at webrtc/audio/audio_send_stream.cc:323
#6  webrtc::internal::AudioSendStream::Reconfigure(webrtc::AudioSendStream::Config const&) ()
    at webrtc/audio/audio_send_stream.cc:195
#7  cricket::WebRtcVoiceMediaChannel::WebRtcAudioSendStream::ReconfigureAudioSendStream() ()
    at webrtc/media/engine/webrtc_voice_engine.cc:1181

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
// Apply current codec settings to a single voe::Channel used for sending.
bool AudioSendStream::SetupSendCodec(const Config& new_config) {
  RTC_DCHECK(new_config.send_codec_spec);
  const auto& spec = *new_config.send_codec_spec;

  RTC_DCHECK(new_config.encoder_factory);
  std::unique_ptr<AudioEncoder> encoder =
      new_config.encoder_factory->MakeAudioEncoder(
          spec.payload_type, spec.format, new_config.codec_pair_id);

  if (!encoder) {
    RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
                       << rtc::ToString(spec.format);
    return false;
  }

  // If a bitrate has been specified for the codec, use it over the
  // codec's default.
  if (spec.target_bitrate_bps) {
    encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
  }

  // Enable ANA if configured (currently only used by Opus).
  if (new_config.audio_network_adaptor_config) {
    if (encoder->EnableAudioNetworkAdaptor(
            *new_config.audio_network_adaptor_config, event_log_)) {
      RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
                       << new_config.rtp.ssrc;
    } else {
      RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
                       << new_config.rtp.ssrc;
    }
  }

  // Wrap the encoder in an AudioEncoderCNG, if VAD is enabled.
  if (spec.cng_payload_type) {
    AudioEncoderCngConfig cng_config;
    cng_config.num_channels = encoder->NumChannels();
    cng_config.payload_type = *spec.cng_payload_type;
    cng_config.speech_encoder = std::move(encoder);
    cng_config.vad_mode = Vad::kVadNormal;
    encoder = CreateComfortNoiseEncoder(std::move(cng_config));

    RegisterCngPayloadType(*spec.cng_payload_type,
                           new_config.send_codec_spec->format.clockrate_hz);
  }

  // Wrap the encoder in a RED encoder, if RED is enabled.
  if (spec.red_payload_type) {
    AudioEncoderCopyRed::Config red_config;
    red_config.payload_type = *spec.red_payload_type;
    red_config.speech_encoder = std::move(encoder);
    encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config));
  }

  // Set currently known overhead (used in ANA, opus only).
  // If overhead changes later, it will be updated in UpdateOverheadForEncoder.
  {
    MutexLock lock(&overhead_per_packet_lock_);
    size_t overhead = GetPerPacketOverheadBytes();
    if (overhead > 0) {
      encoder->OnReceivedOverhead(overhead);
    }
  }

  StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
  channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
                            std::move(encoder));

  return true;
}

1
2
3
#0  webrtc::voe::(anonymous namespace)::ChannelSend::StartSend() () at webrtc/audio/channel_send.cc:531
#1  webrtc::internal::AudioSendStream::Start() () at webrtc/audio/audio_send_stream.cc:368

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
void ChannelSend::StartSend() {
  RTC_DCHECK_RUN_ON(&worker_thread_checker_);
  RTC_DCHECK(!sending_);
  sending_ = true;

  RTC_DCHECK(packet_router_);
  packet_router_->AddSendRtpModule(rtp_rtcp_.get(), /*remb_candidate=*/false);
  rtp_rtcp_->SetSendingMediaStatus(true);
  int ret = rtp_rtcp_->SetSendingStatus(true);
  RTC_DCHECK_EQ(0, ret);

  // It is now OK to start processing on the encoder task queue.
  encoder_queue_.PostTask([this] {
    RTC_DCHECK_RUN_ON(&encoder_queue_);
    encoder_queue_is_active_ = true;
  });
}