FFmpeg
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
whip.c
Go to the documentation of this file.
1 /*
2  * WebRTC-HTTP ingestion protocol (WHIP) muxer
3  * Copyright (c) 2023 The FFmpeg Project
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavcodec/avcodec.h"
23 #include "libavcodec/codec_desc.h"
24 #include "libavcodec/h264.h"
25 #include "libavcodec/startcode.h"
26 #include "libavutil/base64.h"
27 #include "libavutil/bprint.h"
28 #include "libavutil/crc.h"
29 #include "libavutil/hmac.h"
30 #include "libavutil/intreadwrite.h"
31 #include "libavutil/lfg.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/mem.h"
34 #include "libavutil/random_seed.h"
35 #include "libavutil/time.h"
36 #include "avc.h"
37 #include "nal.h"
38 #include "avio_internal.h"
39 #include "http.h"
40 #include "internal.h"
41 #include "mux.h"
42 #include "network.h"
43 #include "srtp.h"
44 #include "tls.h"
45 
46 /**
47  * Maximum size limit of a Session Description Protocol (SDP),
48  * be it an offer or answer.
49  */
50 #define MAX_SDP_SIZE 8192
51 
52 /**
53  * The size of the Secure Real-time Transport Protocol (SRTP) master key material
54  * that is exported by Secure Sockets Layer (SSL) after a successful Datagram
55  * Transport Layer Security (DTLS) handshake. This material consists of a key
56  * of 16 bytes and a salt of 14 bytes.
57  */
58 #define DTLS_SRTP_KEY_LEN 16
59 #define DTLS_SRTP_SALT_LEN 14
60 
61 /**
62  * The maximum size of the Secure Real-time Transport Protocol (SRTP) HMAC checksum
63  * and padding that is appended to the end of the packet. To calculate the maximum
64  * size of the User Datagram Protocol (UDP) packet that can be sent out, subtract
65  * this size from the `pkt_size`.
66  */
67 #define DTLS_SRTP_CHECKSUM_LEN 16
68 
69 /**
70  * When sending ICE or DTLS messages, responses are received via UDP. However, the peer
71  * may not be ready and return EAGAIN, in which case we should wait for a short duration
72  * and retry reading.
73  * For instance, if we try to read from UDP and get EAGAIN, we sleep for 5ms and retry.
74  * This macro is used to limit the total duration in milliseconds (e.g., 50ms), so we
75  * will try at most 5 times.
76  * Keep in mind that this macro should have a minimum duration of 5 ms.
77  */
78 #define ICE_DTLS_READ_INTERVAL 50
79 
80 /* The magic cookie for Session Traversal Utilities for NAT (STUN) messages. */
81 #define STUN_MAGIC_COOKIE 0x2112A442
82 
83 /**
84  * The DTLS content type.
85  * See https://tools.ietf.org/html/rfc2246#section-6.2.1
86  * change_cipher_spec(20), alert(21), handshake(22), application_data(23)
87  */
88 #define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20
89 
90 /**
91  * The DTLS record layer header has a total size of 13 bytes, consisting of
92  * ContentType (1 byte), ProtocolVersion (2 bytes), Epoch (2 bytes),
93  * SequenceNumber (6 bytes), and Length (2 bytes).
94  * See https://datatracker.ietf.org/doc/html/rfc9147#section-4
95  */
96 #define DTLS_RECORD_LAYER_HEADER_LEN 13
97 
98 /**
99  * The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2.
100  * See https://datatracker.ietf.org/doc/html/rfc9147#name-the-dtls-record-layer
101  */
102 #define DTLS_VERSION_10 0xfeff
103 #define DTLS_VERSION_12 0xfefd
104 
105 /**
106  * Maximum size of the buffer for sending and receiving UDP packets.
107  * Please note that this size does not limit the size of the UDP packet that can be sent.
108  * To set the limit for packet size, modify the `pkt_size` parameter.
109  * For instance, it is possible to set the UDP buffer to 4096 to send or receive packets,
110  * but please keep in mind that the `pkt_size` option limits the packet size to 1400.
111  */
112 #define MAX_UDP_BUFFER_SIZE 4096
113 
114 /* Referring to Chrome's definition of RTP payload types. */
115 #define WHIP_RTP_PAYLOAD_TYPE_H264 106
116 #define WHIP_RTP_PAYLOAD_TYPE_OPUS 111
117 
118 /**
119  * The STUN message header, which is 20 bytes long, comprises the
120  * STUNMessageType (1B), MessageLength (2B), MagicCookie (4B),
121  * and TransactionID (12B).
122  * See https://datatracker.ietf.org/doc/html/rfc5389#section-6
123  */
124 #define ICE_STUN_HEADER_SIZE 20
125 
126 /**
127  * The RTP header is 12 bytes long, comprising the Version(1B), PT(1B),
128  * SequenceNumber(2B), Timestamp(4B), and SSRC(4B).
129  * See https://www.rfc-editor.org/rfc/rfc3550#section-5.1
130  */
131 #define WHIP_RTP_HEADER_SIZE 12
132 
133 /**
134  * For RTCP, PT is [128, 223] (or without marker [0, 95]). Literally, RTCP starts
135  * from 64 not 0, so PT is [192, 223] (or without marker [64, 95]), see "RTCP Control
136  * Packet Types (PT)" at
137  * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-parameters-4
138  *
139  * For RTP, the PT is [96, 127], or [224, 255] with marker. See "RTP Payload Types (PT)
140  * for standard audio and video encodings" at
141  * https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-parameters-1
142  */
143 #define WHIP_RTCP_PT_START 192
144 #define WHIP_RTCP_PT_END 223
145 
146 /**
147  * In the case of ICE-LITE, these fields are not used; instead, they are defined
148  * as constant values.
149  */
150 #define WHIP_SDP_SESSION_ID "4489045141692799359"
151 #define WHIP_SDP_CREATOR_IP "127.0.0.1"
152 
153 /* Calculate the elapsed time from starttime to endtime in milliseconds. */
154 #define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000)
155 
156 /* STUN Attribute, comprehension-required range (0x0000-0x7FFF) */
157 enum STUNAttr {
158  STUN_ATTR_USERNAME = 0x0006, /// shared secret response/bind request
159  STUN_ATTR_USE_CANDIDATE = 0x0025, /// bind request
160  STUN_ATTR_MESSAGE_INTEGRITY = 0x0008, /// bind request/response
161  STUN_ATTR_FINGERPRINT = 0x8028, /// rfc5389
162 };
163 
164 enum WHIPState {
166 
167  /* The initial state. */
169  /* The muxer has sent the offer to the peer. */
171  /* The muxer has received the answer from the peer. */
173  /**
174  * After parsing the answer received from the peer, the muxer negotiates the abilities
175  * in the offer that it generated.
176  */
178  /* The muxer has connected to the peer via UDP. */
180  /* The muxer has sent the ICE request to the peer. */
182  /* The muxer has received the ICE response from the peer. */
184  /* The muxer starts attempting the DTLS handshake. */
186  /* The muxer has finished the DTLS handshake with the peer. */
188  /* The muxer has finished the SRTP setup. */
190  /* The muxer is ready to send/receive media frames. */
192  /* The muxer is failed. */
194 };
195 
196 typedef struct WHIPContext {
198 
199  /* The state of the RTC connection. */
201  /* The callback return value for DTLS. */
202  int dtls_ret;
204 
205  /* Parameters for the input audio and video codecs. */
208 
209  /**
210  * The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet;
211  * therefore, it is essential to insert the SPS and PPS before each IDR frame
212  * in such cases.
213  */
215 
216  /* The random number generator. */
218 
219  /* The ICE username and pwd fragment generated by the muxer. */
221  char ice_pwd_local[33];
222  /* The SSRC of the audio and video stream, generated by the muxer. */
223  uint32_t audio_ssrc;
224  uint32_t video_ssrc;
225  /* The PT(Payload Type) of stream, generated by the muxer. */
228  /**
229  * This is the SDP offer generated by the muxer based on the codec parameters,
230  * DTLS, and ICE information.
231  */
232  char *sdp_offer;
233 
234  /* The ICE username and pwd from remote server. */
237  /**
238  * This represents the ICE candidate protocol, priority, host and port.
239  * Currently, we only support one candidate and choose the first UDP candidate.
240  * However, we plan to support multiple candidates in the future.
241  */
243  char *ice_host;
244  int ice_port;
245 
246  /* The SDP answer received from the WebRTC server. */
247  char *sdp_answer;
248  /* The resource URL returned in the Location header of WHIP HTTP response. */
250 
251  /* These variables represent timestamps used for calculating and tracking the cost. */
260 
261  /* The certificate and private key content used for DTLS hanshake */
264  /* The fingerprint of certificate, used in SDP offer. */
266  /**
267  * This represents the material used to build the SRTP master key. It is
268  * generated by DTLS and has the following layout:
269  * 16B 16B 14B 14B
270  * client_key | server_key | client_salt | server_salt
271  */
273 
274  char ssl_error_message[256];
275 
276  /* TODO: Use AVIOContext instead of URLContext */
278 
279  /* The SRTP send context, to encrypt outgoing packets. */
283  /* The SRTP receive context, to decrypt incoming packets. */
285 
286  /* The UDP transport is used for delivering ICE, DTLS and SRTP packets. */
288  /* The buffer for UDP transmission. */
290 
291  /* The timeout in milliseconds for ICE and DTLS handshake. */
293  /**
294  * The size of RTP packet, should generally be set to MTU.
295  * Note that pion requires a smaller value, for example, 1200.
296  */
297  int pkt_size;
298  /**
299  * The optional Bearer token for WHIP Authorization.
300  * See https://www.ietf.org/archive/id/draft-ietf-wish-whip-08.html#name-authentication-and-authoriz
301  */
303  /* The certificate and private key used for DTLS handshake. */
304  char* cert_file;
305  char* key_file;
306 } WHIPContext;
307 
308 /**
309  * Whether the packet is a DTLS packet.
310  */
311 static int is_dtls_packet(uint8_t *b, int size) {
312  uint16_t version = AV_RB16(&b[1]);
316 }
317 
318 
319 /**
320  * Get or Generate a self-signed certificate and private key for DTLS,
321  * fingerprint for SDP
322  */
324 {
325  int ret = 0;
326  WHIPContext *whip = s->priv_data;
327 
328  if (whip->cert_file && whip->key_file) {
329  /* Read the private key and certificate from the file. */
330  if ((ret = ff_ssl_read_key_cert(whip->key_file, whip->cert_file,
331  whip->key_buf, sizeof(whip->key_buf),
332  whip->cert_buf, sizeof(whip->cert_buf),
333  &whip->dtls_fingerprint)) < 0) {
334  av_log(s, AV_LOG_ERROR, "Failed to read DTLS certificate from cert=%s, key=%s\n",
335  whip->cert_file, whip->key_file);
336  return ret;
337  }
338  } else {
339  /* Generate a private key to ctx->dtls_pkey and self-signed certificate. */
340  if ((ret = ff_ssl_gen_key_cert(whip->key_buf, sizeof(whip->key_buf),
341  whip->cert_buf, sizeof(whip->cert_buf),
342  &whip->dtls_fingerprint)) < 0) {
343  av_log(s, AV_LOG_ERROR, "Failed to generate DTLS private key and certificate\n");
344  return ret;
345  }
346  }
347 
348  return ret;
349 }
350 
351 /**
352  * When DTLS state change.
353  */
354 static int dtls_context_on_state(AVFormatContext *s, const char* type, const char* desc)
355 {
356  int ret = 0;
357  WHIPContext *whip = s->priv_data;
358  int state = ff_dtls_state(whip->dtls_uc);
359 
360  if (state == DTLS_STATE_CLOSED) {
361  whip->dtls_closed = 1;
362  av_log(whip, AV_LOG_VERBOSE, "DTLS session closed, type=%s, desc=%s, elapsed=%dms\n",
363  type ? type : "", desc ? desc : "", ELAPSED(whip->whip_starttime, av_gettime()));
364  goto error;
365  }
366 
367  if (state == DTLS_STATE_FAILED) {
368  whip->state = WHIP_STATE_FAILED;
369  av_log(whip, AV_LOG_ERROR, "DTLS session failed, type=%s, desc=%s\n",
370  type ? type : "", desc ? desc : "");
371  whip->dtls_ret = AVERROR(EIO);
372  goto error;
373  }
374 
377  whip->whip_dtls_time = av_gettime();
378  av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
379  ELAPSED(whip->whip_starttime, av_gettime()));
380  return ret;
381  }
382 error:
383  return -1;
384 }
385 
387 {
388  WHIPContext *whip = s->priv_data;
389  /* reuse the udp created by whip */
390  ff_tls_set_external_socket(whip->dtls_uc, whip->udp);
391 
392  /* Make the socket non-blocking */
394  whip->dtls_uc->flags |= AVIO_FLAG_NONBLOCK;
395 
396  return 0;
397 }
398 
399 /**
400  * Initialize and check the options for the WebRTC muxer.
401  */
403 {
404  int ret, ideal_pkt_size = 532;
405  WHIPContext *whip = s->priv_data;
406  uint32_t seed;
407 
408  whip->whip_starttime = av_gettime();
409 
411  if (ret < 0) {
412  av_log(whip, AV_LOG_ERROR, "Failed to init certificate and key\n");
413  return ret;
414  }
415 
416  /* Initialize the random number generator. */
418  av_lfg_init(&whip->rnd, seed);
419 
420  if (whip->pkt_size < ideal_pkt_size)
421  av_log(whip, AV_LOG_WARNING, "pkt_size=%d(<%d) is too small, may cause packet loss\n",
422  whip->pkt_size, ideal_pkt_size);
423 
424  if (whip->state < WHIP_STATE_INIT)
425  whip->state = WHIP_STATE_INIT;
426  whip->whip_init_time = av_gettime();
427  av_log(whip, AV_LOG_VERBOSE, "Init state=%d, handshake_timeout=%dms, pkt_size=%d, seed=%d, elapsed=%dms\n",
428  whip->state, whip->handshake_timeout, whip->pkt_size, seed, ELAPSED(whip->whip_starttime, av_gettime()));
429 
430  return 0;
431 }
432 
433 /**
434  * When duplicating a stream, the demuxer has already set the extradata, profile, and
435  * level of the par. Keep in mind that this function will not be invoked since the
436  * profile and level are set.
437  *
438  * When utilizing an encoder, such as libx264, to encode a stream, the extradata in
439  * par->extradata contains the SPS, which includes profile and level information.
440  * However, the profile and level of par remain unspecified. Therefore, it is necessary
441  * to extract the profile and level data from the extradata and assign it to the par's
442  * profile and level. Keep in mind that AVFMT_GLOBALHEADER must be enabled; otherwise,
443  * the extradata will remain empty.
444  */
446 {
447  int ret = 0;
448  const uint8_t *r = par->extradata, *r1, *end = par->extradata + par->extradata_size;
449  H264SPS seq, *const sps = &seq;
450  uint32_t state;
451  WHIPContext *whip = s->priv_data;
452 
453  if (par->codec_id != AV_CODEC_ID_H264)
454  return ret;
455 
456  if (par->profile != AV_PROFILE_UNKNOWN && par->level != AV_LEVEL_UNKNOWN)
457  return ret;
458 
459  if (!par->extradata || par->extradata_size <= 0) {
460  av_log(whip, AV_LOG_ERROR, "Unable to parse profile from empty extradata=%p, size=%d\n",
461  par->extradata, par->extradata_size);
462  return AVERROR(EINVAL);
463  }
464 
465  while (1) {
466  r = avpriv_find_start_code(r, end, &state);
467  if (r >= end)
468  break;
469 
470  r1 = ff_nal_find_startcode(r, end);
471  if ((state & 0x1f) == H264_NAL_SPS) {
472  ret = ff_avc_decode_sps(sps, r, r1 - r);
473  if (ret < 0) {
474  av_log(whip, AV_LOG_ERROR, "Failed to decode SPS, state=%x, size=%d\n",
475  state, (int)(r1 - r));
476  return ret;
477  }
478 
479  av_log(whip, AV_LOG_VERBOSE, "Parse profile=%d, level=%d from SPS\n",
480  sps->profile_idc, sps->level_idc);
481  par->profile = sps->profile_idc;
482  par->level = sps->level_idc;
483  }
484 
485  r = r1;
486  }
487 
488  return ret;
489 }
490 
491 /**
492  * Parses video SPS/PPS from the extradata of codecpar and checks the codec.
493  * Currently only supports video(h264) and audio(opus). Note that only baseline
494  * and constrained baseline profiles of h264 are supported.
495  *
496  * If the profile is less than 0, the function considers the profile as baseline.
497  * It may need to parse the profile from SPS/PPS. This situation occurs when ingesting
498  * desktop and transcoding.
499  *
500  * @param s Pointer to the AVFormatContext
501  * @returns Returns 0 if successful or AVERROR_xxx in case of an error.
502  *
503  * TODO: FIXME: There is an issue with the timestamp of OPUS audio, especially when
504  * the input is an MP4 file. The timestamp deviates from the expected value of 960,
505  * causing Chrome to play the audio stream with noise. This problem can be replicated
506  * by transcoding a specific file into MP4 format and publishing it using the WHIP
507  * muxer. However, when directly transcoding and publishing through the WHIP muxer,
508  * the issue is not present, and the audio timestamp remains consistent. The root
509  * cause is still unknown, and this comment has been added to address this issue
510  * in the future. Further research is needed to resolve the problem.
511  */
513 {
514  int i, ret = 0;
515  WHIPContext *whip = s->priv_data;
516 
517  for (i = 0; i < s->nb_streams; i++) {
518  AVCodecParameters *par = s->streams[i]->codecpar;
520  switch (par->codec_type) {
521  case AVMEDIA_TYPE_VIDEO:
522  if (whip->video_par) {
523  av_log(whip, AV_LOG_ERROR, "Only one video stream is supported by RTC\n");
524  return AVERROR(EINVAL);
525  }
526  whip->video_par = par;
527 
528  if (par->codec_id != AV_CODEC_ID_H264) {
529  av_log(whip, AV_LOG_ERROR, "Unsupported video codec %s by RTC, choose h264\n",
530  desc ? desc->name : "unknown");
531  return AVERROR_PATCHWELCOME;
532  }
533 
534  if (par->video_delay > 0) {
535  av_log(whip, AV_LOG_ERROR, "Unsupported B frames by RTC\n");
536  return AVERROR_PATCHWELCOME;
537  }
538 
539  if ((ret = parse_profile_level(s, par)) < 0) {
540  av_log(whip, AV_LOG_ERROR, "Failed to parse SPS/PPS from extradata\n");
541  return AVERROR(EINVAL);
542  }
543 
544  if (par->profile == AV_PROFILE_UNKNOWN) {
545  av_log(whip, AV_LOG_WARNING, "No profile found in extradata, consider baseline\n");
546  return AVERROR(EINVAL);
547  }
548  if (par->level == AV_LEVEL_UNKNOWN) {
549  av_log(whip, AV_LOG_WARNING, "No level found in extradata, consider 3.1\n");
550  return AVERROR(EINVAL);
551  }
552  break;
553  case AVMEDIA_TYPE_AUDIO:
554  if (whip->audio_par) {
555  av_log(whip, AV_LOG_ERROR, "Only one audio stream is supported by RTC\n");
556  return AVERROR(EINVAL);
557  }
558  whip->audio_par = par;
559 
560  if (par->codec_id != AV_CODEC_ID_OPUS) {
561  av_log(whip, AV_LOG_ERROR, "Unsupported audio codec %s by RTC, choose opus\n",
562  desc ? desc->name : "unknown");
563  return AVERROR_PATCHWELCOME;
564  }
565 
566  if (par->ch_layout.nb_channels != 2) {
567  av_log(whip, AV_LOG_ERROR, "Unsupported audio channels %d by RTC, choose stereo\n",
568  par->ch_layout.nb_channels);
569  return AVERROR_PATCHWELCOME;
570  }
571 
572  if (par->sample_rate != 48000) {
573  av_log(whip, AV_LOG_ERROR, "Unsupported audio sample rate %d by RTC, choose 48000\n", par->sample_rate);
574  return AVERROR_PATCHWELCOME;
575  }
576  break;
577  default:
578  av_log(whip, AV_LOG_ERROR, "Codec type '%s' for stream %d is not supported by RTC\n",
580  return AVERROR_PATCHWELCOME;
581  }
582  }
583 
584  return ret;
585 }
586 
587 /**
588  * Generate SDP offer according to the codec parameters, DTLS and ICE information.
589  *
590  * Note that we don't use av_sdp_create to generate SDP offer because it doesn't
591  * support DTLS and ICE information.
592  *
593  * @return 0 if OK, AVERROR_xxx on error
594  */
596 {
597  int ret = 0, profile, level, profile_iop;
598  const char *acodec_name = NULL, *vcodec_name = NULL;
599  AVBPrint bp;
600  WHIPContext *whip = s->priv_data;
601 
602  /* To prevent a crash during cleanup, always initialize it. */
603  av_bprint_init(&bp, 1, MAX_SDP_SIZE);
604 
605  if (whip->sdp_offer) {
606  av_log(whip, AV_LOG_ERROR, "SDP offer is already set\n");
607  ret = AVERROR(EINVAL);
608  goto end;
609  }
610 
611  snprintf(whip->ice_ufrag_local, sizeof(whip->ice_ufrag_local), "%08x",
612  av_lfg_get(&whip->rnd));
613  snprintf(whip->ice_pwd_local, sizeof(whip->ice_pwd_local), "%08x%08x%08x%08x",
614  av_lfg_get(&whip->rnd), av_lfg_get(&whip->rnd), av_lfg_get(&whip->rnd),
615  av_lfg_get(&whip->rnd));
616 
617  whip->audio_ssrc = av_lfg_get(&whip->rnd);
618  whip->video_ssrc = av_lfg_get(&whip->rnd);
619 
622 
623  av_bprintf(&bp, ""
624  "v=0\r\n"
625  "o=FFmpeg %s 2 IN IP4 %s\r\n"
626  "s=FFmpegPublishSession\r\n"
627  "t=0 0\r\n"
628  "a=group:BUNDLE 0 1\r\n"
629  "a=extmap-allow-mixed\r\n"
630  "a=msid-semantic: WMS\r\n",
633 
634  if (whip->audio_par) {
635  if (whip->audio_par->codec_id == AV_CODEC_ID_OPUS)
636  acodec_name = "opus";
637 
638  av_bprintf(&bp, ""
639  "m=audio 9 UDP/TLS/RTP/SAVPF %u\r\n"
640  "c=IN IP4 0.0.0.0\r\n"
641  "a=ice-ufrag:%s\r\n"
642  "a=ice-pwd:%s\r\n"
643  "a=fingerprint:sha-256 %s\r\n"
644  "a=setup:passive\r\n"
645  "a=mid:0\r\n"
646  "a=sendonly\r\n"
647  "a=msid:FFmpeg audio\r\n"
648  "a=rtcp-mux\r\n"
649  "a=rtpmap:%u %s/%d/%d\r\n"
650  "a=ssrc:%u cname:FFmpeg\r\n"
651  "a=ssrc:%u msid:FFmpeg audio\r\n",
652  whip->audio_payload_type,
653  whip->ice_ufrag_local,
654  whip->ice_pwd_local,
655  whip->dtls_fingerprint,
656  whip->audio_payload_type,
657  acodec_name,
658  whip->audio_par->sample_rate,
660  whip->audio_ssrc,
661  whip->audio_ssrc);
662  }
663 
664  if (whip->video_par) {
665  profile_iop = profile = whip->video_par->profile;
666  level = whip->video_par->level;
667  if (whip->video_par->codec_id == AV_CODEC_ID_H264) {
668  vcodec_name = "H264";
669  profile_iop &= AV_PROFILE_H264_CONSTRAINED;
671  }
672 
673  av_bprintf(&bp, ""
674  "m=video 9 UDP/TLS/RTP/SAVPF %u\r\n"
675  "c=IN IP4 0.0.0.0\r\n"
676  "a=ice-ufrag:%s\r\n"
677  "a=ice-pwd:%s\r\n"
678  "a=fingerprint:sha-256 %s\r\n"
679  "a=setup:passive\r\n"
680  "a=mid:1\r\n"
681  "a=sendonly\r\n"
682  "a=msid:FFmpeg video\r\n"
683  "a=rtcp-mux\r\n"
684  "a=rtcp-rsize\r\n"
685  "a=rtpmap:%u %s/90000\r\n"
686  "a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n"
687  "a=ssrc:%u cname:FFmpeg\r\n"
688  "a=ssrc:%u msid:FFmpeg video\r\n",
689  whip->video_payload_type,
690  whip->ice_ufrag_local,
691  whip->ice_pwd_local,
692  whip->dtls_fingerprint,
693  whip->video_payload_type,
694  vcodec_name,
695  whip->video_payload_type,
696  profile,
697  profile_iop,
698  level,
699  whip->video_ssrc,
700  whip->video_ssrc);
701  }
702 
703  if (!av_bprint_is_complete(&bp)) {
704  av_log(whip, AV_LOG_ERROR, "Offer exceed max %d, %s\n", MAX_SDP_SIZE, bp.str);
705  ret = AVERROR(EIO);
706  goto end;
707  }
708 
709  whip->sdp_offer = av_strdup(bp.str);
710  if (!whip->sdp_offer) {
711  ret = AVERROR(ENOMEM);
712  goto end;
713  }
714 
715  if (whip->state < WHIP_STATE_OFFER)
716  whip->state = WHIP_STATE_OFFER;
717  whip->whip_offer_time = av_gettime();
718  av_log(whip, AV_LOG_VERBOSE, "Generated state=%d, offer: %s\n", whip->state, whip->sdp_offer);
719 
720 end:
721  av_bprint_finalize(&bp, NULL);
722  return ret;
723 }
724 
725 /**
726  * Exchange SDP offer with WebRTC peer to get the answer.
727  *
728  * @return 0 if OK, AVERROR_xxx on error
729  */
731 {
732  int ret;
733  char buf[MAX_URL_SIZE];
734  AVBPrint bp;
735  WHIPContext *whip = s->priv_data;
736  /* The URL context is an HTTP transport layer for the WHIP protocol. */
737  URLContext *whip_uc = NULL;
739  char *hex_data = NULL;
740  const char *proto_name = avio_find_protocol_name(s->url);
741 
742  /* To prevent a crash during cleanup, always initialize it. */
743  av_bprint_init(&bp, 1, MAX_SDP_SIZE);
744 
745  if (!av_strstart(proto_name, "http", NULL)) {
746  av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose http, url is %s\n",
747  proto_name, s->url);
748  ret = AVERROR(EINVAL);
749  goto end;
750  }
751 
752  if (!whip->sdp_offer || !strlen(whip->sdp_offer)) {
753  av_log(whip, AV_LOG_ERROR, "No offer to exchange\n");
754  ret = AVERROR(EINVAL);
755  goto end;
756  }
757 
758  ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\nContent-Type: application/sdp\r\n");
759  if (whip->authorization)
760  ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization);
761  if (ret <= 0 || ret >= sizeof(buf)) {
762  av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf);
763  ret = AVERROR(EINVAL);
764  goto end;
765  }
766 
767  av_dict_set(&opts, "headers", buf, 0);
768  av_dict_set_int(&opts, "chunked_post", 0, 0);
769 
770  hex_data = av_mallocz(2 * strlen(whip->sdp_offer) + 1);
771  if (!hex_data) {
772  ret = AVERROR(ENOMEM);
773  goto end;
774  }
775  ff_data_to_hex(hex_data, whip->sdp_offer, strlen(whip->sdp_offer), 0);
776  av_dict_set(&opts, "post_data", hex_data, 0);
777 
778  ret = ffurl_open_whitelist(&whip_uc, s->url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
779  &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
780  if (ret < 0) {
781  av_log(whip, AV_LOG_ERROR, "Failed to request url=%s, offer: %s\n", s->url, whip->sdp_offer);
782  goto end;
783  }
784 
785  if (ff_http_get_new_location(whip_uc)) {
787  if (!whip->whip_resource_url) {
788  ret = AVERROR(ENOMEM);
789  goto end;
790  }
791  }
792 
793  while (1) {
794  ret = ffurl_read(whip_uc, buf, sizeof(buf));
795  if (ret == AVERROR_EOF) {
796  /* Reset the error because we read all response as answer util EOF. */
797  ret = 0;
798  break;
799  }
800  if (ret <= 0) {
801  av_log(whip, AV_LOG_ERROR, "Failed to read response from url=%s, offer is %s, answer is %s\n",
802  s->url, whip->sdp_offer, whip->sdp_answer);
803  goto end;
804  }
805 
806  av_bprintf(&bp, "%.*s", ret, buf);
807  if (!av_bprint_is_complete(&bp)) {
808  av_log(whip, AV_LOG_ERROR, "Answer exceed max size %d, %.*s, %s\n", MAX_SDP_SIZE, ret, buf, bp.str);
809  ret = AVERROR(EIO);
810  goto end;
811  }
812  }
813 
814  if (!av_strstart(bp.str, "v=", NULL)) {
815  av_log(whip, AV_LOG_ERROR, "Invalid answer: %s\n", bp.str);
816  ret = AVERROR(EINVAL);
817  goto end;
818  }
819 
820  whip->sdp_answer = av_strdup(bp.str);
821  if (!whip->sdp_answer) {
822  ret = AVERROR(ENOMEM);
823  goto end;
824  }
825 
826  if (whip->state < WHIP_STATE_ANSWER)
827  whip->state = WHIP_STATE_ANSWER;
828  av_log(whip, AV_LOG_VERBOSE, "Got state=%d, answer: %s\n", whip->state, whip->sdp_answer);
829 
830 end:
831  ffurl_closep(&whip_uc);
832  av_bprint_finalize(&bp, NULL);
833  av_dict_free(&opts);
834  av_freep(&hex_data);
835  return ret;
836 }
837 
838 /**
839  * Parses the ICE ufrag, pwd, and candidates from the SDP answer.
840  *
841  * This function is used to extract the ICE ufrag, pwd, and candidates from the SDP answer.
842  * It returns an error if any of these fields is NULL. The function only uses the first
843  * candidate if there are multiple candidates. However, support for multiple candidates
844  * will be added in the future.
845  *
846  * @param s Pointer to the AVFormatContext
847  * @returns Returns 0 if successful or AVERROR_xxx if an error occurs.
848  */
850 {
851  int ret = 0;
852  AVIOContext *pb;
853  char line[MAX_URL_SIZE];
854  const char *ptr;
855  int i;
856  WHIPContext *whip = s->priv_data;
857 
858  if (!whip->sdp_answer || !strlen(whip->sdp_answer)) {
859  av_log(whip, AV_LOG_ERROR, "No answer to parse\n");
860  ret = AVERROR(EINVAL);
861  goto end;
862  }
863 
864  pb = avio_alloc_context(whip->sdp_answer, strlen(whip->sdp_answer), 0, NULL, NULL, NULL, NULL);
865  if (!pb)
866  return AVERROR(ENOMEM);
867 
868  for (i = 0; !avio_feof(pb); i++) {
869  ff_get_chomp_line(pb, line, sizeof(line));
870  if (av_strstart(line, "a=ice-ufrag:", &ptr) && !whip->ice_ufrag_remote) {
871  whip->ice_ufrag_remote = av_strdup(ptr);
872  if (!whip->ice_ufrag_remote) {
873  ret = AVERROR(ENOMEM);
874  goto end;
875  }
876  } else if (av_strstart(line, "a=ice-pwd:", &ptr) && !whip->ice_pwd_remote) {
877  whip->ice_pwd_remote = av_strdup(ptr);
878  if (!whip->ice_pwd_remote) {
879  ret = AVERROR(ENOMEM);
880  goto end;
881  }
882  } else if (av_strstart(line, "a=candidate:", &ptr) && !whip->ice_protocol) {
883  ptr = av_stristr(ptr, "udp");
884  if (ptr && av_stristr(ptr, "host")) {
885  char protocol[17], host[129];
886  int priority, port;
887  ret = sscanf(ptr, "%16s %d %128s %d typ host", protocol, &priority, host, &port);
888  if (ret != 4) {
889  av_log(whip, AV_LOG_ERROR, "Failed %d to parse line %d %s from %s\n",
890  ret, i, line, whip->sdp_answer);
891  ret = AVERROR(EIO);
892  goto end;
893  }
894 
895  if (av_strcasecmp(protocol, "udp")) {
896  av_log(whip, AV_LOG_ERROR, "Protocol %s is not supported by RTC, choose udp, line %d %s of %s\n",
897  protocol, i, line, whip->sdp_answer);
898  ret = AVERROR(EIO);
899  goto end;
900  }
901 
902  whip->ice_protocol = av_strdup(protocol);
903  whip->ice_host = av_strdup(host);
904  whip->ice_port = port;
905  if (!whip->ice_protocol || !whip->ice_host) {
906  ret = AVERROR(ENOMEM);
907  goto end;
908  }
909  }
910  }
911  }
912 
913  if (!whip->ice_pwd_remote || !strlen(whip->ice_pwd_remote)) {
914  av_log(whip, AV_LOG_ERROR, "No remote ice pwd parsed from %s\n", whip->sdp_answer);
915  ret = AVERROR(EINVAL);
916  goto end;
917  }
918 
919  if (!whip->ice_ufrag_remote || !strlen(whip->ice_ufrag_remote)) {
920  av_log(whip, AV_LOG_ERROR, "No remote ice ufrag parsed from %s\n", whip->sdp_answer);
921  ret = AVERROR(EINVAL);
922  goto end;
923  }
924 
925  if (!whip->ice_protocol || !whip->ice_host || !whip->ice_port) {
926  av_log(whip, AV_LOG_ERROR, "No ice candidate parsed from %s\n", whip->sdp_answer);
927  ret = AVERROR(EINVAL);
928  goto end;
929  }
930 
931  if (whip->state < WHIP_STATE_NEGOTIATED)
933  whip->whip_answer_time = av_gettime();
934  av_log(whip, AV_LOG_VERBOSE, "SDP state=%d, offer=%zuB, answer=%zuB, ufrag=%s, pwd=%zuB, transport=%s://%s:%d, elapsed=%dms\n",
935  whip->state, strlen(whip->sdp_offer), strlen(whip->sdp_answer), whip->ice_ufrag_remote, strlen(whip->ice_pwd_remote),
936  whip->ice_protocol, whip->ice_host, whip->ice_port, ELAPSED(whip->whip_starttime, av_gettime()));
937 
938 end:
939  avio_context_free(&pb);
940  return ret;
941 }
942 
943 /**
944  * Creates and marshals an ICE binding request packet.
945  *
946  * This function creates and marshals an ICE binding request packet. The function only
947  * generates the username attribute and does not include goog-network-info, ice-controlling,
948  * use-candidate, and priority. However, some of these attributes may be added in the future.
949  *
950  * @param s Pointer to the AVFormatContext
951  * @param buf Pointer to memory buffer to store the request packet
952  * @param buf_size Size of the memory buffer
953  * @param request_size Pointer to an integer that receives the size of the request packet
954  * @return Returns 0 if successful or AVERROR_xxx if an error occurs.
955  */
956 static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, int *request_size)
957 {
958  int ret, size, crc32;
959  char username[128];
960  AVIOContext *pb = NULL;
961  AVHMAC *hmac = NULL;
962  WHIPContext *whip = s->priv_data;
963 
964  pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL);
965  if (!pb)
966  return AVERROR(ENOMEM);
967 
968  hmac = av_hmac_alloc(AV_HMAC_SHA1);
969  if (!hmac) {
970  ret = AVERROR(ENOMEM);
971  goto end;
972  }
973 
974  /* Write 20 bytes header */
975  avio_wb16(pb, 0x0001); /* STUN binding request */
976  avio_wb16(pb, 0); /* length */
977  avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */
978  avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */
979  avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */
980  avio_wb32(pb, av_lfg_get(&whip->rnd)); /* transaction ID */
981 
982  /* The username is the concatenation of the two ICE ufrag */
983  ret = snprintf(username, sizeof(username), "%s:%s", whip->ice_ufrag_remote, whip->ice_ufrag_local);
984  if (ret <= 0 || ret >= sizeof(username)) {
985  av_log(whip, AV_LOG_ERROR, "Failed to build username %s:%s, max=%zu, ret=%d\n",
986  whip->ice_ufrag_remote, whip->ice_ufrag_local, sizeof(username), ret);
987  ret = AVERROR(EIO);
988  goto end;
989  }
990 
991  /* Write the username attribute */
992  avio_wb16(pb, STUN_ATTR_USERNAME); /* attribute type username */
993  avio_wb16(pb, ret); /* size of username */
994  avio_write(pb, username, ret); /* bytes of username */
995  ffio_fill(pb, 0, (4 - (ret % 4)) % 4); /* padding */
996 
997  /* Write the use-candidate attribute */
998  avio_wb16(pb, STUN_ATTR_USE_CANDIDATE); /* attribute type use-candidate */
999  avio_wb16(pb, 0); /* size of use-candidate */
1000 
1001  /* Build and update message integrity */
1002  avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */
1003  avio_wb16(pb, 20); /* size of message integrity */
1004  ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */
1005  size = avio_tell(pb);
1006  buf[2] = (size - 20) >> 8;
1007  buf[3] = (size - 20) & 0xFF;
1008  av_hmac_init(hmac, whip->ice_pwd_remote, strlen(whip->ice_pwd_remote));
1009  av_hmac_update(hmac, buf, size - 24);
1010  av_hmac_final(hmac, buf + size - 20, 20);
1011 
1012  /* Write the fingerprint attribute */
1013  avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */
1014  avio_wb16(pb, 4); /* size of fingerprint */
1015  ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */
1016  size = avio_tell(pb);
1017  buf[2] = (size - 20) >> 8;
1018  buf[3] = (size - 20) & 0xFF;
1019  /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */
1020  crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF;
1021  avio_skip(pb, -4);
1022  avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */
1023 
1024  *request_size = size;
1025 
1026 end:
1027  avio_context_free(&pb);
1028  av_hmac_free(hmac);
1029  return ret;
1030 }
1031 
1032 /**
1033  * Create an ICE binding response.
1034  *
1035  * This function generates an ICE binding response and writes it to the provided
1036  * buffer. The response is signed using the local password for message integrity.
1037  *
1038  * @param s Pointer to the AVFormatContext structure.
1039  * @param tid Pointer to the transaction ID of the binding request. The tid_size should be 12.
1040  * @param tid_size The size of the transaction ID, should be 12.
1041  * @param buf Pointer to the buffer where the response will be written.
1042  * @param buf_size The size of the buffer provided for the response.
1043  * @param response_size Pointer to an integer that will store the size of the generated response.
1044  * @return Returns 0 if successful or AVERROR_xxx if an error occurs.
1045  */
1046 static int ice_create_response(AVFormatContext *s, char *tid, int tid_size, uint8_t *buf, int buf_size, int *response_size)
1047 {
1048  int ret = 0, size, crc32;
1049  AVIOContext *pb = NULL;
1050  AVHMAC *hmac = NULL;
1051  WHIPContext *whip = s->priv_data;
1052 
1053  if (tid_size != 12) {
1054  av_log(whip, AV_LOG_ERROR, "Invalid transaction ID size. Expected 12, got %d\n", tid_size);
1055  return AVERROR(EINVAL);
1056  }
1057 
1058  pb = avio_alloc_context(buf, buf_size, 1, NULL, NULL, NULL, NULL);
1059  if (!pb)
1060  return AVERROR(ENOMEM);
1061 
1062  hmac = av_hmac_alloc(AV_HMAC_SHA1);
1063  if (!hmac) {
1064  ret = AVERROR(ENOMEM);
1065  goto end;
1066  }
1067 
1068  /* Write 20 bytes header */
1069  avio_wb16(pb, 0x0101); /* STUN binding response */
1070  avio_wb16(pb, 0); /* length */
1071  avio_wb32(pb, STUN_MAGIC_COOKIE); /* magic cookie */
1072  avio_write(pb, tid, tid_size); /* transaction ID */
1073 
1074  /* Build and update message integrity */
1075  avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */
1076  avio_wb16(pb, 20); /* size of message integrity */
1077  ffio_fill(pb, 0, 20); /* fill with zero to directly write and skip it */
1078  size = avio_tell(pb);
1079  buf[2] = (size - 20) >> 8;
1080  buf[3] = (size - 20) & 0xFF;
1081  av_hmac_init(hmac, whip->ice_pwd_local, strlen(whip->ice_pwd_local));
1082  av_hmac_update(hmac, buf, size - 24);
1083  av_hmac_final(hmac, buf + size - 20, 20);
1084 
1085  /* Write the fingerprint attribute */
1086  avio_wb16(pb, STUN_ATTR_FINGERPRINT); /* attribute type fingerprint */
1087  avio_wb16(pb, 4); /* size of fingerprint */
1088  ffio_fill(pb, 0, 4); /* fill with zero to directly write and skip it */
1089  size = avio_tell(pb);
1090  buf[2] = (size - 20) >> 8;
1091  buf[3] = (size - 20) & 0xFF;
1092  /* Refer to the av_hash_alloc("CRC32"), av_hash_init and av_hash_final */
1093  crc32 = av_crc(av_crc_get_table(AV_CRC_32_IEEE_LE), 0xFFFFFFFF, buf, size - 8) ^ 0xFFFFFFFF;
1094  avio_skip(pb, -4);
1095  avio_wb32(pb, crc32 ^ 0x5354554E); /* xor with "STUN" */
1096 
1097  *response_size = size;
1098 
1099 end:
1100  avio_context_free(&pb);
1101  av_hmac_free(hmac);
1102  return ret;
1103 }
1104 
1105 /**
1106  * A Binding request has class=0b00 (request) and method=0b000000000001 (Binding)
1107  * and is encoded into the first 16 bits as 0x0001.
1108  * See https://datatracker.ietf.org/doc/html/rfc5389#section-6
1109  */
1110 static int ice_is_binding_request(uint8_t *b, int size)
1111 {
1112  return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0001;
1113 }
1114 
1115 /**
1116  * A Binding response has class=0b10 (success response) and method=0b000000000001,
1117  * and is encoded into the first 16 bits as 0x0101.
1118  */
1119 static int ice_is_binding_response(uint8_t *b, int size)
1120 {
1121  return size >= ICE_STUN_HEADER_SIZE && AV_RB16(&b[0]) == 0x0101;
1122 }
1123 
1124 /**
1125  * In RTP packets, the first byte is represented as 0b10xxxxxx, where the initial
1126  * two bits (0b10) indicate the RTP version,
1127  * see https://www.rfc-editor.org/rfc/rfc3550#section-5.1
1128  * The RTCP packet header is similar to RTP,
1129  * see https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1
1130  */
1131 static int media_is_rtp_rtcp(const uint8_t *b, int size)
1132 {
1133  return size >= WHIP_RTP_HEADER_SIZE && (b[0] & 0xC0) == 0x80;
1134 }
1135 
1136 /* Whether the packet is RTCP. */
1137 static int media_is_rtcp(const uint8_t *b, int size)
1138 {
1139  return size >= WHIP_RTP_HEADER_SIZE && b[1] >= WHIP_RTCP_PT_START && b[1] <= WHIP_RTCP_PT_END;
1140 }
1141 
1142 /**
1143  * This function handles incoming binding request messages by responding to them.
1144  * If the message is not a binding request, it will be ignored.
1145  */
1146 static int ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_size)
1147 {
1148  int ret = 0, size;
1149  char tid[12];
1150  WHIPContext *whip = s->priv_data;
1151 
1152  /* Ignore if not a binding request. */
1153  if (!ice_is_binding_request(buf, buf_size))
1154  return ret;
1155 
1156  if (buf_size < ICE_STUN_HEADER_SIZE) {
1157  av_log(whip, AV_LOG_ERROR, "Invalid STUN message, expected at least %d, got %d\n",
1158  ICE_STUN_HEADER_SIZE, buf_size);
1159  return AVERROR(EINVAL);
1160  }
1161 
1162  /* Parse transaction id from binding request in buf. */
1163  memcpy(tid, buf + 8, 12);
1164 
1165  /* Build the STUN binding response. */
1166  ret = ice_create_response(s, tid, sizeof(tid), whip->buf, sizeof(whip->buf), &size);
1167  if (ret < 0) {
1168  av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding response, size=%d\n", size);
1169  return ret;
1170  }
1171 
1172  ret = ffurl_write(whip->udp, whip->buf, size);
1173  if (ret < 0) {
1174  av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding response, size=%d\n", size);
1175  return ret;
1176  }
1177 
1178  return 0;
1179 }
1180 
1181 /**
1182  * To establish a connection with the UDP server, we utilize ICE-LITE in a Client-Server
1183  * mode. In this setup, FFmpeg acts as the UDP client, while the peer functions as the
1184  * UDP server.
1185  */
1187 {
1188  int ret = 0;
1189  char url[256];
1190  AVDictionary *opts = NULL;
1191  WHIPContext *whip = s->priv_data;
1192 
1193  /* Build UDP URL and create the UDP context as transport. */
1194  ff_url_join(url, sizeof(url), "udp", NULL, whip->ice_host, whip->ice_port, NULL);
1195 
1196  av_dict_set_int(&opts, "connect", 1, 0);
1197  av_dict_set_int(&opts, "fifo_size", 0, 0);
1198  /* Set the max packet size to the buffer size. */
1199  av_dict_set_int(&opts, "pkt_size", whip->pkt_size, 0);
1200 
1201  ret = ffurl_open_whitelist(&whip->udp, url, AVIO_FLAG_WRITE, &s->interrupt_callback,
1202  &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
1203  if (ret < 0) {
1204  av_log(whip, AV_LOG_ERROR, "Failed to connect udp://%s:%d\n", whip->ice_host, whip->ice_port);
1205  goto end;
1206  }
1207 
1208  /* Make the socket non-blocking, set to READ and WRITE mode after connected */
1211 
1212  if (whip->state < WHIP_STATE_UDP_CONNECTED)
1214  whip->whip_udp_time = av_gettime();
1215  av_log(whip, AV_LOG_VERBOSE, "UDP state=%d, elapsed=%dms, connected to udp://%s:%d\n",
1216  whip->state, ELAPSED(whip->whip_starttime, av_gettime()), whip->ice_host, whip->ice_port);
1217 
1218 end:
1219  av_dict_free(&opts);
1220  return ret;
1221 }
1222 
1224 {
1225  int ret = 0, size, i;
1226  int64_t starttime = av_gettime(), now;
1227  WHIPContext *whip = s->priv_data;
1228  AVDictionary *opts = NULL;
1229  char buf[256], *cert_buf = NULL, *key_buf = NULL;
1230 
1231  if (whip->state < WHIP_STATE_UDP_CONNECTED || !whip->udp) {
1232  av_log(whip, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp);
1233  return AVERROR(EINVAL);
1234  }
1235 
1236  while (1) {
1237  if (whip->state <= WHIP_STATE_ICE_CONNECTING) {
1238  /* Build the STUN binding request. */
1239  ret = ice_create_request(s, whip->buf, sizeof(whip->buf), &size);
1240  if (ret < 0) {
1241  av_log(whip, AV_LOG_ERROR, "Failed to create STUN binding request, size=%d\n", size);
1242  goto end;
1243  }
1244 
1245  ret = ffurl_write(whip->udp, whip->buf, size);
1246  if (ret < 0) {
1247  av_log(whip, AV_LOG_ERROR, "Failed to send STUN binding request, size=%d\n", size);
1248  goto end;
1249  }
1250 
1251  if (whip->state < WHIP_STATE_ICE_CONNECTING)
1253  }
1254 
1255 next_packet:
1256  if (whip->state >= WHIP_STATE_DTLS_FINISHED)
1257  /* DTLS handshake is done, exit the loop. */
1258  break;
1259 
1260  now = av_gettime();
1261  if (now - starttime >= whip->handshake_timeout * 1000) {
1262  av_log(whip, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n",
1263  whip->handshake_timeout, ELAPSED(starttime, now), ELAPSED(whip->whip_starttime, now), whip->state);
1264  ret = AVERROR(ETIMEDOUT);
1265  goto end;
1266  }
1267 
1268  /* Read the STUN or DTLS messages from peer. */
1269  for (i = 0; i < ICE_DTLS_READ_INTERVAL / 5 && whip->state < WHIP_STATE_DTLS_CONNECTING; i++) {
1270  ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf));
1271  if (ret > 0)
1272  break;
1273  if (ret == AVERROR(EAGAIN)) {
1274  av_usleep(5 * 1000);
1275  continue;
1276  }
1277  av_log(whip, AV_LOG_ERROR, "Failed to read message\n");
1278  goto end;
1279  }
1280 
1281  /* Got nothing, continue to process handshake. */
1282  if (ret <= 0 && whip->state < WHIP_STATE_DTLS_CONNECTING)
1283  continue;
1284 
1285  /* Handle the ICE binding response. */
1286  if (ice_is_binding_response(whip->buf, ret)) {
1287  if (whip->state < WHIP_STATE_ICE_CONNECTED) {
1289  whip->whip_ice_time = av_gettime();
1290  av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n",
1291  whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "",
1293 
1294  ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL);
1295  av_dict_set_int(&opts, "mtu", whip->pkt_size, 0);
1296  if (whip->cert_file) {
1297  av_dict_set(&opts, "cert_file", whip->cert_file, 0);
1298  } else
1299  av_dict_set(&opts, "cert_pem", whip->cert_buf, 0);
1300 
1301  if (whip->key_file) {
1302  av_dict_set(&opts, "key_file", whip->key_file, 0);
1303  } else
1304  av_dict_set(&opts, "key_pem", whip->key_buf, 0);
1305  av_dict_set_int(&opts, "external_sock", 1, 0);
1306  av_dict_set_int(&opts, "listen", 1, 0);
1307  /* If got the first binding response, start DTLS handshake. */
1308  ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
1309  &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
1310  av_dict_free(&opts);
1311  if (ret < 0)
1312  goto end;
1313  dtls_initialize(s);
1314  }
1315  goto next_packet;
1316  }
1317 
1318  /* When a binding request is received, it is necessary to respond immediately. */
1319  if (ice_is_binding_request(whip->buf, ret)) {
1320  if ((ret = ice_handle_binding_request(s, whip->buf, ret)) < 0)
1321  goto end;
1322  goto next_packet;
1323  }
1324 
1325  /* If got any DTLS messages, handle it. */
1328  if ((ret = ffurl_handshake(whip->dtls_uc)) < 0)
1329  goto end;
1331  goto next_packet;
1332  }
1333  }
1334 
1335 end:
1336  if (cert_buf)
1337  av_free(cert_buf);
1338  if (key_buf)
1339  av_free(key_buf);
1340  return ret;
1341 }
1342 
1343 /**
1344  * Establish the SRTP context using the keying material exported from DTLS.
1345  *
1346  * Create separate SRTP contexts for sending video and audio, as their sequences differ
1347  * and should not share a single context. Generate a single SRTP context for receiving
1348  * RTCP only.
1349  *
1350  * @return 0 if OK, AVERROR_xxx on error
1351  */
1353 {
1354  int ret;
1355  char recv_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN];
1356  char send_key[DTLS_SRTP_KEY_LEN + DTLS_SRTP_SALT_LEN];
1358  /**
1359  * The profile for OpenSSL's SRTP is SRTP_AES128_CM_SHA1_80, see ssl/d1_srtp.c.
1360  * The profile for FFmpeg's SRTP is SRTP_AES128_CM_HMAC_SHA1_80, see libavformat/srtp.c.
1361  */
1362  const char* suite = "SRTP_AES128_CM_HMAC_SHA1_80";
1363  WHIPContext *whip = s->priv_data;
1365  if (ret < 0)
1366  goto end;
1367  /**
1368  * This represents the material used to build the SRTP master key. It is
1369  * generated by DTLS and has the following layout:
1370  * 16B 16B 14B 14B
1371  * client_key | server_key | client_salt | server_salt
1372  */
1373  char *client_key = whip->dtls_srtp_materials;
1374  char *server_key = whip->dtls_srtp_materials + DTLS_SRTP_KEY_LEN;
1375  char *client_salt = server_key + DTLS_SRTP_KEY_LEN;
1376  char *server_salt = client_salt + DTLS_SRTP_SALT_LEN;
1377 
1378  /* As DTLS server, the recv key is client master key plus salt. */
1379  memcpy(recv_key, client_key, DTLS_SRTP_KEY_LEN);
1380  memcpy(recv_key + DTLS_SRTP_KEY_LEN, client_salt, DTLS_SRTP_SALT_LEN);
1381 
1382  /* As DTLS server, the send key is server master key plus salt. */
1383  memcpy(send_key, server_key, DTLS_SRTP_KEY_LEN);
1384  memcpy(send_key + DTLS_SRTP_KEY_LEN, server_salt, DTLS_SRTP_SALT_LEN);
1385 
1386  /* Setup SRTP context for outgoing packets */
1387  if (!av_base64_encode(buf, sizeof(buf), send_key, sizeof(send_key))) {
1388  av_log(whip, AV_LOG_ERROR, "Failed to encode send key\n");
1389  ret = AVERROR(EIO);
1390  goto end;
1391  }
1392 
1393  ret = ff_srtp_set_crypto(&whip->srtp_audio_send, suite, buf);
1394  if (ret < 0) {
1395  av_log(whip, AV_LOG_ERROR, "Failed to set crypto for audio send\n");
1396  goto end;
1397  }
1398 
1399  ret = ff_srtp_set_crypto(&whip->srtp_video_send, suite, buf);
1400  if (ret < 0) {
1401  av_log(whip, AV_LOG_ERROR, "Failed to set crypto for video send\n");
1402  goto end;
1403  }
1404 
1405  ret = ff_srtp_set_crypto(&whip->srtp_rtcp_send, suite, buf);
1406  if (ret < 0) {
1407  av_log(whip, AV_LOG_ERROR, "Failed to set crypto for rtcp send\n");
1408  goto end;
1409  }
1410 
1411  /* Setup SRTP context for incoming packets */
1412  if (!av_base64_encode(buf, sizeof(buf), recv_key, sizeof(recv_key))) {
1413  av_log(whip, AV_LOG_ERROR, "Failed to encode recv key\n");
1414  ret = AVERROR(EIO);
1415  goto end;
1416  }
1417 
1418  ret = ff_srtp_set_crypto(&whip->srtp_recv, suite, buf);
1419  if (ret < 0) {
1420  av_log(whip, AV_LOG_ERROR, "Failed to set crypto for recv\n");
1421  goto end;
1422  }
1423 
1424  if (whip->state < WHIP_STATE_SRTP_FINISHED)
1426  whip->whip_srtp_time = av_gettime();
1427  av_log(whip, AV_LOG_VERBOSE, "SRTP setup done, state=%d, suite=%s, key=%zuB, elapsed=%dms\n",
1428  whip->state, suite, sizeof(send_key), ELAPSED(whip->whip_starttime, av_gettime()));
1429 
1430 end:
1431  return ret;
1432 }
1433 
1434 /**
1435  * Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
1436  *
1437  * This function modifies the video STAP packet, removing the markers, and updating the
1438  * NRI of the first NALU. Additionally, it uses the corresponding SRTP context to encrypt
1439  * the RTP packet, where the video packet is handled by the video SRTP context.
1440  */
1441 static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
1442 {
1443  int ret, cipher_size, is_rtcp, is_video;
1444  uint8_t payload_type;
1445  AVFormatContext *s = opaque;
1446  WHIPContext *whip = s->priv_data;
1447  SRTPContext *srtp;
1448 
1449  /* Ignore if not RTP or RTCP packet. */
1450  if (!media_is_rtp_rtcp(buf, buf_size))
1451  return 0;
1452 
1453  /* Only support audio, video and rtcp. */
1454  is_rtcp = media_is_rtcp(buf, buf_size);
1455  payload_type = buf[1] & 0x7f;
1456  is_video = payload_type == whip->video_payload_type;
1457  if (!is_rtcp && payload_type != whip->video_payload_type && payload_type != whip->audio_payload_type)
1458  return 0;
1459 
1460  /* Get the corresponding SRTP context. */
1461  srtp = is_rtcp ? &whip->srtp_rtcp_send : (is_video? &whip->srtp_video_send : &whip->srtp_audio_send);
1462 
1463  /* Encrypt by SRTP and send out. */
1464  cipher_size = ff_srtp_encrypt(srtp, buf, buf_size, whip->buf, sizeof(whip->buf));
1465  if (cipher_size <= 0 || cipher_size < buf_size) {
1466  av_log(whip, AV_LOG_WARNING, "Failed to encrypt packet=%dB, cipher=%dB\n", buf_size, cipher_size);
1467  return 0;
1468  }
1469 
1470  ret = ffurl_write(whip->udp, whip->buf, cipher_size);
1471  if (ret < 0) {
1472  av_log(whip, AV_LOG_ERROR, "Failed to write packet=%dB, ret=%d\n", cipher_size, ret);
1473  return ret;
1474  }
1475 
1476  return ret;
1477 }
1478 
1479 /**
1480  * Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP
1481  * packets from the encoded frames.
1482  *
1483  * The corresponding SRTP context is utilized to encrypt each stream's RTP packets. For
1484  * example, a video SRTP context is used for the video stream. Additionally, the
1485  * "on_rtp_write_packet" callback function is set as the write function for each RTP
1486  * muxer to send out encrypted RTP packets.
1487  *
1488  * @return 0 if OK, AVERROR_xxx on error
1489  */
1491 {
1492  int ret, i, is_video, buffer_size, max_packet_size;
1493  AVFormatContext *rtp_ctx = NULL;
1494  AVDictionary *opts = NULL;
1495  uint8_t *buffer = NULL;
1496  char buf[64];
1497  WHIPContext *whip = s->priv_data;
1498  whip->udp->flags |= AVIO_FLAG_NONBLOCK;
1499 
1500  const AVOutputFormat *rtp_format = av_guess_format("rtp", NULL, NULL);
1501  if (!rtp_format) {
1502  av_log(whip, AV_LOG_ERROR, "Failed to guess rtp muxer\n");
1503  ret = AVERROR(ENOSYS);
1504  goto end;
1505  }
1506 
1507  /* The UDP buffer size, may greater than MTU. */
1508  buffer_size = MAX_UDP_BUFFER_SIZE;
1509  /* The RTP payload max size. Reserved some bytes for SRTP checksum and padding. */
1510  max_packet_size = whip->pkt_size - DTLS_SRTP_CHECKSUM_LEN;
1511 
1512  for (i = 0; i < s->nb_streams; i++) {
1513  rtp_ctx = avformat_alloc_context();
1514  if (!rtp_ctx) {
1515  ret = AVERROR(ENOMEM);
1516  goto end;
1517  }
1518 
1519  rtp_ctx->oformat = rtp_format;
1520  if (!avformat_new_stream(rtp_ctx, NULL)) {
1521  ret = AVERROR(ENOMEM);
1522  goto end;
1523  }
1524  /* Pass the interrupt callback on */
1525  rtp_ctx->interrupt_callback = s->interrupt_callback;
1526  /* Copy the max delay setting; the rtp muxer reads this. */
1527  rtp_ctx->max_delay = s->max_delay;
1528  /* Copy other stream parameters. */
1529  rtp_ctx->streams[0]->sample_aspect_ratio = s->streams[i]->sample_aspect_ratio;
1530  rtp_ctx->flags |= s->flags & AVFMT_FLAG_BITEXACT;
1531  rtp_ctx->strict_std_compliance = s->strict_std_compliance;
1532 
1533  /* Set the synchronized start time. */
1534  rtp_ctx->start_time_realtime = s->start_time_realtime;
1535 
1536  avcodec_parameters_copy(rtp_ctx->streams[0]->codecpar, s->streams[i]->codecpar);
1537  rtp_ctx->streams[0]->time_base = s->streams[i]->time_base;
1538 
1539  /**
1540  * For H.264, consistently utilize the annexb format through the Bitstream Filter (BSF);
1541  * therefore, we deactivate the extradata detection for the RTP muxer.
1542  */
1543  if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_H264) {
1544  av_freep(&rtp_ctx->streams[i]->codecpar->extradata);
1545  rtp_ctx->streams[i]->codecpar->extradata_size = 0;
1546  }
1547 
1548  buffer = av_malloc(buffer_size);
1549  if (!buffer) {
1550  ret = AVERROR(ENOMEM);
1551  goto end;
1552  }
1553 
1554  rtp_ctx->pb = avio_alloc_context(buffer, buffer_size, 1, s, NULL, on_rtp_write_packet, NULL);
1555  if (!rtp_ctx->pb) {
1556  ret = AVERROR(ENOMEM);
1557  goto end;
1558  }
1559  rtp_ctx->pb->max_packet_size = max_packet_size;
1560  rtp_ctx->pb->av_class = &ff_avio_class;
1561 
1562  is_video = s->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO;
1563  snprintf(buf, sizeof(buf), "%d", is_video? whip->video_payload_type : whip->audio_payload_type);
1564  av_dict_set(&opts, "payload_type", buf, 0);
1565  snprintf(buf, sizeof(buf), "%d", is_video? whip->video_ssrc : whip->audio_ssrc);
1566  av_dict_set(&opts, "ssrc", buf, 0);
1567 
1568  ret = avformat_write_header(rtp_ctx, &opts);
1569  if (ret < 0) {
1570  av_log(whip, AV_LOG_ERROR, "Failed to write rtp header\n");
1571  goto end;
1572  }
1573 
1574  ff_format_set_url(rtp_ctx, av_strdup(s->url));
1575  s->streams[i]->time_base = rtp_ctx->streams[0]->time_base;
1576  s->streams[i]->priv_data = rtp_ctx;
1577  rtp_ctx = NULL;
1578  }
1579 
1580  if (whip->state < WHIP_STATE_READY)
1581  whip->state = WHIP_STATE_READY;
1582  av_log(whip, AV_LOG_INFO, "Muxer state=%d, buffer_size=%d, max_packet_size=%d, "
1583  "elapsed=%dms(init:%d,offer:%d,answer:%d,udp:%d,ice:%d,dtls:%d,srtp:%d)\n",
1584  whip->state, buffer_size, max_packet_size, ELAPSED(whip->whip_starttime, av_gettime()),
1585  ELAPSED(whip->whip_starttime, whip->whip_init_time),
1586  ELAPSED(whip->whip_init_time, whip->whip_offer_time),
1588  ELAPSED(whip->whip_answer_time, whip->whip_udp_time),
1589  ELAPSED(whip->whip_udp_time, whip->whip_ice_time),
1590  ELAPSED(whip->whip_ice_time, whip->whip_dtls_time),
1591  ELAPSED(whip->whip_dtls_time, whip->whip_srtp_time));
1592 
1593 end:
1594  if (rtp_ctx)
1595  avio_context_free(&rtp_ctx->pb);
1596  avformat_free_context(rtp_ctx);
1597  av_dict_free(&opts);
1598  return ret;
1599 }
1600 
1601 /**
1602  * RTC is connectionless, for it's based on UDP, so it check whether sesison is
1603  * timeout. In such case, publishers can't republish the stream util the session
1604  * is timeout.
1605  * This function is called to notify the server that the stream is ended, server
1606  * should expire and close the session immediately, so that publishers can republish
1607  * the stream quickly.
1608  */
1610 {
1611  int ret;
1612  char buf[MAX_URL_SIZE];
1613  URLContext *whip_uc = NULL;
1614  AVDictionary *opts = NULL;
1615  WHIPContext *whip = s->priv_data;
1616 
1617  if (!whip->whip_resource_url)
1618  return 0;
1619 
1620  ret = snprintf(buf, sizeof(buf), "Cache-Control: no-cache\r\n");
1621  if (whip->authorization)
1622  ret += snprintf(buf + ret, sizeof(buf) - ret, "Authorization: Bearer %s\r\n", whip->authorization);
1623  if (ret <= 0 || ret >= sizeof(buf)) {
1624  av_log(whip, AV_LOG_ERROR, "Failed to generate headers, size=%d, %s\n", ret, buf);
1625  ret = AVERROR(EINVAL);
1626  goto end;
1627  }
1628 
1629  av_dict_set(&opts, "headers", buf, 0);
1630  av_dict_set_int(&opts, "chunked_post", 0, 0);
1631  av_dict_set(&opts, "method", "DELETE", 0);
1632  ret = ffurl_open_whitelist(&whip_uc, whip->whip_resource_url, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
1633  &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
1634  if (ret < 0) {
1635  av_log(whip, AV_LOG_ERROR, "Failed to DELETE url=%s\n", whip->whip_resource_url);
1636  goto end;
1637  }
1638 
1639  while (1) {
1640  ret = ffurl_read(whip_uc, buf, sizeof(buf));
1641  if (ret == AVERROR_EOF) {
1642  ret = 0;
1643  break;
1644  }
1645  if (ret < 0) {
1646  av_log(whip, AV_LOG_ERROR, "Failed to read response from DELETE url=%s\n", whip->whip_resource_url);
1647  goto end;
1648  }
1649  }
1650 
1651  av_log(whip, AV_LOG_INFO, "Dispose resource %s ok\n", whip->whip_resource_url);
1652 
1653 end:
1654  ffurl_closep(&whip_uc);
1655  av_dict_free(&opts);
1656  return ret;
1657 }
1658 
1659 /**
1660  * Since the h264_mp4toannexb filter only processes the MP4 ISOM format and bypasses
1661  * the annexb format, it is necessary to manually insert encoder metadata before each
1662  * IDR when dealing with annexb format packets. For instance, in the case of H.264,
1663  * we must insert SPS and PPS before the IDR frame.
1664  */
1666 {
1667  int ret = 0;
1668  AVPacket *in = NULL;
1669  AVCodecParameters *par = s->streams[pkt->stream_index]->codecpar;
1670  uint32_t nal_size = 0, out_size = par ? par->extradata_size : 0;
1671  uint8_t unit_type, sps_seen = 0, pps_seen = 0, idr_seen = 0, *out;
1672  const uint8_t *buf, *buf_end, *r1;
1673 
1674  if (!par || !par->extradata || par->extradata_size <= 0)
1675  return ret;
1676 
1677  /* Discover NALU type from packet. */
1678  buf_end = pkt->data + pkt->size;
1679  for (buf = ff_nal_find_startcode(pkt->data, buf_end); buf < buf_end; buf += nal_size) {
1680  while (!*(buf++));
1681  r1 = ff_nal_find_startcode(buf, buf_end);
1682  if ((nal_size = r1 - buf) > 0) {
1683  unit_type = *buf & 0x1f;
1684  if (unit_type == H264_NAL_SPS) {
1685  sps_seen = 1;
1686  } else if (unit_type == H264_NAL_PPS) {
1687  pps_seen = 1;
1688  } else if (unit_type == H264_NAL_IDR_SLICE) {
1689  idr_seen = 1;
1690  }
1691 
1692  out_size += 3 + nal_size;
1693  }
1694  }
1695 
1696  if (!idr_seen || (sps_seen && pps_seen))
1697  return ret;
1698 
1699  /* See av_bsf_send_packet */
1700  in = av_packet_alloc();
1701  if (!in)
1702  return AVERROR(ENOMEM);
1703 
1705  if (ret < 0)
1706  goto fail;
1707 
1708  av_packet_move_ref(in, pkt);
1709 
1710  /* Create a new packet with sps/pps inserted. */
1712  if (ret < 0)
1713  goto fail;
1714 
1715  ret = av_packet_copy_props(pkt, in);
1716  if (ret < 0)
1717  goto fail;
1718 
1719  memcpy(pkt->data, par->extradata, par->extradata_size);
1720  out = pkt->data + par->extradata_size;
1721  buf_end = in->data + in->size;
1722  for (buf = ff_nal_find_startcode(in->data, buf_end); buf < buf_end; buf += nal_size) {
1723  while (!*(buf++));
1724  r1 = ff_nal_find_startcode(buf, buf_end);
1725  if ((nal_size = r1 - buf) > 0) {
1726  AV_WB24(out, 0x00001);
1727  memcpy(out + 3, buf, nal_size);
1728  out += 3 + nal_size;
1729  }
1730  }
1731 
1732 fail:
1733  if (ret < 0)
1735  av_packet_free(&in);
1736 
1737  return ret;
1738 }
1739 
1741 {
1742  int ret;
1743  WHIPContext *whip = s->priv_data;
1744 
1745  if ((ret = initialize(s)) < 0)
1746  goto end;
1747 
1748  if ((ret = parse_codec(s)) < 0)
1749  goto end;
1750 
1751  if ((ret = generate_sdp_offer(s)) < 0)
1752  goto end;
1753 
1754  if ((ret = exchange_sdp(s)) < 0)
1755  goto end;
1756 
1757  if ((ret = parse_answer(s)) < 0)
1758  goto end;
1759 
1760  if ((ret = udp_connect(s)) < 0)
1761  goto end;
1762 
1763  if ((ret = ice_dtls_handshake(s)) < 0)
1764  goto end;
1765 
1766  if ((ret = setup_srtp(s)) < 0)
1767  goto end;
1768 
1769  if ((ret = create_rtp_muxer(s)) < 0)
1770  goto end;
1771 
1772 end:
1773  if (ret < 0 && whip->state < WHIP_STATE_FAILED)
1774  whip->state = WHIP_STATE_FAILED;
1775  if (ret >= 0 && whip->state >= WHIP_STATE_FAILED && whip->dtls_ret < 0)
1776  ret = whip->dtls_ret;
1777  return ret;
1778 }
1779 
1781 {
1782  int ret;
1783  WHIPContext *whip = s->priv_data;
1784  AVStream *st = s->streams[pkt->stream_index];
1785  AVFormatContext *rtp_ctx = st->priv_data;
1786 
1787  /* TODO: Send binding request every 1s as WebRTC heartbeat. */
1788 
1789  /**
1790  * Receive packets from the server such as ICE binding requests, DTLS messages,
1791  * and RTCP like PLI requests, then respond to them.
1792  */
1793  ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf));
1794  if (ret > 0) {
1795  if (is_dtls_packet(whip->buf, ret)) {
1796  if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) {
1797  av_log(whip, AV_LOG_ERROR, "Failed to handle DTLS message\n");
1798  goto end;
1799  }
1800  }
1801  } else if (ret != AVERROR(EAGAIN)) {
1802  av_log(whip, AV_LOG_ERROR, "Failed to read from UDP socket\n");
1803  goto end;
1804  }
1805 
1807  if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) {
1808  av_log(whip, AV_LOG_ERROR, "Failed to insert SPS/PPS before IDR\n");
1809  goto end;
1810  }
1811  }
1812 
1813  ret = ff_write_chained(rtp_ctx, 0, pkt, s, 0);
1814  if (ret < 0) {
1815  if (ret == AVERROR(EINVAL)) {
1816  av_log(whip, AV_LOG_WARNING, "Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret);
1817  ret = 0;
1818  } else
1819  av_log(whip, AV_LOG_ERROR, "Failed to write packet, size=%d\n", pkt->size);
1820  goto end;
1821  }
1822 
1823 end:
1824  if (ret < 0 && whip->state < WHIP_STATE_FAILED)
1825  whip->state = WHIP_STATE_FAILED;
1826  if (ret >= 0 && whip->state >= WHIP_STATE_FAILED && whip->dtls_ret < 0)
1827  ret = whip->dtls_ret;
1828  if (ret >= 0 && whip->dtls_closed)
1829  ret = AVERROR(EIO);
1830  return ret;
1831 }
1832 
1834 {
1835  int i, ret;
1836  WHIPContext *whip = s->priv_data;
1837 
1838  ret = dispose_session(s);
1839  if (ret < 0)
1840  av_log(whip, AV_LOG_WARNING, "Failed to dispose resource, ret=%d\n", ret);
1841 
1842  for (i = 0; i < s->nb_streams; i++) {
1843  AVFormatContext* rtp_ctx = s->streams[i]->priv_data;
1844  if (!rtp_ctx)
1845  continue;
1846 
1847  av_write_trailer(rtp_ctx);
1848  /**
1849  * Keep in mind that it is necessary to free the buffer of pb since we allocate
1850  * it and pass it to pb using avio_alloc_context, while avio_context_free does
1851  * not perform this action.
1852  */
1853  av_freep(&rtp_ctx->pb->buffer);
1854  avio_context_free(&rtp_ctx->pb);
1855  avformat_free_context(rtp_ctx);
1856  s->streams[i]->priv_data = NULL;
1857  }
1858 
1859  av_freep(&whip->sdp_offer);
1860  av_freep(&whip->sdp_answer);
1861  av_freep(&whip->whip_resource_url);
1862  av_freep(&whip->ice_ufrag_remote);
1863  av_freep(&whip->ice_pwd_remote);
1864  av_freep(&whip->ice_protocol);
1865  av_freep(&whip->ice_host);
1866  av_freep(&whip->authorization);
1867  av_freep(&whip->cert_file);
1868  av_freep(&whip->key_file);
1869  ff_srtp_free(&whip->srtp_audio_send);
1870  ff_srtp_free(&whip->srtp_video_send);
1871  ff_srtp_free(&whip->srtp_rtcp_send);
1872  ff_srtp_free(&whip->srtp_recv);
1873  ffurl_close(whip->dtls_uc);
1874  ffurl_closep(&whip->udp);
1875 }
1876 
1878 {
1879  int ret = 1, extradata_isom = 0;
1880  uint8_t *b = pkt->data;
1881  WHIPContext *whip = s->priv_data;
1882 
1883  if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
1884  extradata_isom = st->codecpar->extradata_size > 0 && st->codecpar->extradata[0] == 1;
1885  if (pkt->size >= 5 && AV_RB32(b) != 0x0000001 && (AV_RB24(b) != 0x000001 || extradata_isom)) {
1886  ret = ff_stream_add_bitstream_filter(st, "h264_mp4toannexb", NULL);
1887  av_log(whip, AV_LOG_VERBOSE, "Enable BSF h264_mp4toannexb, packet=[%x %x %x %x %x ...], extradata_isom=%d\n",
1888  b[0], b[1], b[2], b[3], b[4], extradata_isom);
1889  } else
1890  whip->h264_annexb_insert_sps_pps = 1;
1891  }
1892 
1893  return ret;
1894 }
1895 
1896 #define OFFSET(x) offsetof(WHIPContext, x)
1897 #define ENC AV_OPT_FLAG_ENCODING_PARAM
1898 static const AVOption options[] = {
1899  { "handshake_timeout", "Timeout in milliseconds for ICE and DTLS handshake.", OFFSET(handshake_timeout), AV_OPT_TYPE_INT, { .i64 = 5000 }, -1, INT_MAX, ENC },
1900  { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, ENC },
1901  { "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
1902  { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
1903  { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
1904  { NULL },
1905 };
1906 
1907 static const AVClass whip_muxer_class = {
1908  .class_name = "WHIP muxer",
1909  .item_name = av_default_item_name,
1910  .option = options,
1911  .version = LIBAVUTIL_VERSION_INT,
1912 };
1913 
1915  .p.name = "whip",
1916  .p.long_name = NULL_IF_CONFIG_SMALL("WHIP(WebRTC-HTTP ingestion protocol) muxer"),
1917  .p.audio_codec = AV_CODEC_ID_OPUS,
1918  .p.video_codec = AV_CODEC_ID_H264,
1920  .p.priv_class = &whip_muxer_class,
1921  .priv_data_size = sizeof(WHIPContext),
1922  .init = whip_init,
1924  .deinit = whip_deinit,
1926 };
error
static void error(const char *err)
Definition: target_bsf_fuzzer.c:32
H264SPS
Definition: avc.h:32
WHIPContext::whip_udp_time
int64_t whip_udp_time
Definition: whip.c:256
ICE_DTLS_READ_INTERVAL
#define ICE_DTLS_READ_INTERVAL
When sending ICE or DTLS messages, responses are received via UDP.
Definition: whip.c:78
on_rtp_write_packet
static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
Definition: whip.c:1441
ff_get_chomp_line
int ff_get_chomp_line(AVIOContext *s, char *buf, int maxlen)
Same as ff_get_line but strip the white-space characters in the text tail.
Definition: aviobuf.c:786
AVHMAC
Definition: hmac.c:40
av_packet_unref
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: packet.c:432
H264_NAL_IDR_SLICE
@ H264_NAL_IDR_SLICE
Definition: h264.h:39
H264_NAL_SPS
@ H264_NAL_SPS
Definition: h264.h:41
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:216
AVCodecParameters::extradata
uint8_t * extradata
Extra binary data needed for initializing the decoder, codec-dependent.
Definition: codec_par.h:69
level
uint8_t level
Definition: svq3.c:208
whip_deinit
static av_cold void whip_deinit(AVFormatContext *s)
Definition: whip.c:1833
AVOutputFormat::name
const char * name
Definition: avformat.h:506
av_bprint_is_complete
static int av_bprint_is_complete(const AVBPrint *buf)
Test if the print buffer is complete (not truncated).
Definition: bprint.h:218
r
const char * r
Definition: vf_curves.c:127
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
WHIPContext::sdp_offer
char * sdp_offer
This is the SDP offer generated by the muxer based on the codec parameters, DTLS, and ICE information...
Definition: whip.c:232
AVCodecParameters::codec_type
enum AVMediaType codec_type
General type of the encoded data.
Definition: codec_par.h:51
ff_dtls_state
int ff_dtls_state(URLContext *h)
Definition: tls_openssl.c:543
STUN_MAGIC_COOKIE
#define STUN_MAGIC_COOKIE
Definition: whip.c:81
WHIP_STATE_ANSWER
@ WHIP_STATE_ANSWER
Definition: whip.c:172
out
FILE * out
Definition: movenc.c:55
av_lfg_init
av_cold void av_lfg_init(AVLFG *c, unsigned int seed)
Definition: lfg.c:32
dtls_initialize
static av_cold int dtls_initialize(AVFormatContext *s)
Definition: whip.c:386
av_bprint_init
void av_bprint_init(AVBPrint *buf, unsigned size_init, unsigned size_max)
Definition: bprint.c:69
AVCodecParameters
This struct describes the properties of an encoded stream.
Definition: codec_par.h:47
av_stristr
char * av_stristr(const char *s1, const char *s2)
Locate the first case-independent occurrence in the string haystack of the string needle.
Definition: avstring.c:58
avformat_new_stream
AVStream * avformat_new_stream(AVFormatContext *s, const struct AVCodec *c)
Add a new stream to a media file.
AVStream::priv_data
void * priv_data
Definition: avformat.h:769
AVERROR_EOF
#define AVERROR_EOF
End of file.
Definition: error.h:57
AVIO_FLAG_READ_WRITE
#define AVIO_FLAG_READ_WRITE
read-write pseudo flag
Definition: avio.h:619
STUN_ATTR_FINGERPRINT
@ STUN_ATTR_FINGERPRINT
bind request/response
Definition: whip.c:161
WHIP_STATE_DTLS_FINISHED
@ WHIP_STATE_DTLS_FINISHED
Definition: whip.c:187
avio_context_free
void avio_context_free(AVIOContext **s)
Free the supplied IO context and everything associated with it.
Definition: aviobuf.c:126
int64_t
long long int64_t
Definition: coverity.c:34
WHIPContext::ice_pwd_remote
char * ice_pwd_remote
Definition: whip.c:236
WHIPContext::dtls_uc
URLContext * dtls_uc
Definition: whip.c:277
ffurl_write
static int ffurl_write(URLContext *h, const uint8_t *buf, int size)
Write size bytes from buf to the resource accessed by h.
Definition: url.h:202
av_strcasecmp
int av_strcasecmp(const char *a, const char *b)
Locale-independent case-insensitive compare.
Definition: avstring.c:207
initialize
static av_cold int initialize(AVFormatContext *s)
Initialize and check the options for the WebRTC muxer.
Definition: whip.c:402
out_size
int out_size
Definition: movenc.c:56
WHIPContext::video_ssrc
uint32_t video_ssrc
Definition: whip.c:224
AVFormatContext::streams
AVStream ** streams
A list of all streams in the file.
Definition: avformat.h:1332
deinit
static void deinit(AVFormatContext *s)
Definition: chromaprint.c:52
WHIPContext::dtls_closed
int dtls_closed
Definition: whip.c:203
AVFormatContext::strict_std_compliance
int strict_std_compliance
Allow non-standard and experimental extension.
Definition: avformat.h:1618
AVPacket::data
uint8_t * data
Definition: packet.h:552
avio_alloc_context
AVIOContext * avio_alloc_context(unsigned char *buffer, int buffer_size, int write_flag, void *opaque, int(*read_packet)(void *opaque, uint8_t *buf, int buf_size), int(*write_packet)(void *opaque, const uint8_t *buf, int buf_size), int64_t(*seek)(void *opaque, int64_t offset, int whence))
Allocate and initialize an AVIOContext for buffered I/O.
Definition: aviobuf.c:109
AVOption
AVOption.
Definition: opt.h:429
srtp.h
b
#define b
Definition: input.c:42
ICE_STUN_HEADER_SIZE
#define ICE_STUN_HEADER_SIZE
The STUN message header, which is 20 bytes long, comprises the STUNMessageType (1B),...
Definition: whip.c:124
WHIPContext::handshake_timeout
int handshake_timeout
Definition: whip.c:292
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:226
is_dtls_packet
static int is_dtls_packet(uint8_t *b, int size)
Whether the packet is a DTLS packet.
Definition: whip.c:311
ffurl_close
int ffurl_close(URLContext *h)
Definition: avio.c:612
AVIOContext::max_packet_size
int max_packet_size
Definition: avio.h:241
ice_create_request
static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, int *request_size)
Creates and marshals an ICE binding request packet.
Definition: whip.c:956
AVDictionary
Definition: dict.c:32
WHIPContext::srtp_video_send
SRTPContext srtp_video_send
Definition: whip.c:281
WHIPContext::udp
URLContext * udp
Definition: whip.c:287
SRTPContext
Definition: srtp.h:30
AVChannelLayout::nb_channels
int nb_channels
Number of channels in this layout.
Definition: channel_layout.h:329
dtls_context_on_state
static int dtls_context_on_state(AVFormatContext *s, const char *type, const char *desc)
When DTLS state change.
Definition: whip.c:354
WHIP_SDP_CREATOR_IP
#define WHIP_SDP_CREATOR_IP
Definition: whip.c:151
WHIPContext::h264_annexb_insert_sps_pps
int h264_annexb_insert_sps_pps
The h264_mp4toannexb Bitstream Filter (BSF) bypasses the AnnexB packet; therefore,...
Definition: whip.c:214
udp_connect
static int udp_connect(AVFormatContext *s)
To establish a connection with the UDP server, we utilize ICE-LITE in a Client-Server mode.
Definition: whip.c:1186
av_packet_free
void av_packet_free(AVPacket **pkt)
Free the packet, if the packet is reference counted, it will be unreferenced first.
Definition: packet.c:75
av_hmac_final
int av_hmac_final(AVHMAC *c, uint8_t *out, unsigned int outlen)
Finish hashing and output the HMAC digest.
Definition: hmac.c:167
DTLS_SRTP_CHECKSUM_LEN
#define DTLS_SRTP_CHECKSUM_LEN
The maximum size of the Secure Real-time Transport Protocol (SRTP) HMAC checksum and padding that is ...
Definition: whip.c:67
WHIPContext::ssl_error_message
char ssl_error_message[256]
Definition: whip.c:274
WHIP_STATE_ICE_CONNECTED
@ WHIP_STATE_ICE_CONNECTED
Definition: whip.c:183
FFOutputFormat::p
AVOutputFormat p
The public AVOutputFormat.
Definition: mux.h:65
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
av_get_random_seed
uint32_t av_get_random_seed(void)
Get a seed to use in conjunction with random functions.
Definition: random_seed.c:196
WHIPContext::ice_port
int ice_port
Definition: whip.c:244
WHIP_SDP_SESSION_ID
#define WHIP_SDP_SESSION_ID
In the case of ICE-LITE, these fields are not used; instead, they are defined as constant values.
Definition: whip.c:150
crc.h
WHIPContext::key_file
char * key_file
Definition: whip.c:305
AVFormatContext::interrupt_callback
AVIOInterruptCB interrupt_callback
Custom interrupt callbacks for the I/O layer.
Definition: avformat.h:1534
ff_whip_muxer
const FFOutputFormat ff_whip_muxer
Definition: whip.c:1914
WHIPContext::cert_buf
char cert_buf[MAX_CERTIFICATE_SIZE]
Definition: whip.c:262
fail
#define fail()
Definition: checkasm.h:199
ff_avc_decode_sps
int ff_avc_decode_sps(H264SPS *sps, const uint8_t *buf, int buf_size)
Definition: avc.c:208
WHIP_STATE_SRTP_FINISHED
@ WHIP_STATE_SRTP_FINISHED
Definition: whip.c:189
avio_tell
static av_always_inline int64_t avio_tell(AVIOContext *s)
ftell() equivalent for AVIOContext.
Definition: avio.h:494
WHIPContext::whip_ice_time
int64_t whip_ice_time
Definition: whip.c:257
WHIPContext
Definition: whip.c:196
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
parse_answer
static int parse_answer(AVFormatContext *s)
Parses the ICE ufrag, pwd, and candidates from the SDP answer.
Definition: whip.c:849
ff_data_to_hex
char * ff_data_to_hex(char *buf, const uint8_t *src, int size, int lowercase)
Write hexadecimal string corresponding to given binary data.
Definition: utils.c:451
ff_srtp_encrypt
int ff_srtp_encrypt(struct SRTPContext *s, const uint8_t *in, int len, uint8_t *out, int outlen)
Definition: srtp.c:239
WHIP_RTP_PAYLOAD_TYPE_H264
#define WHIP_RTP_PAYLOAD_TYPE_H264
Definition: whip.c:115
ice_handle_binding_request
static int ice_handle_binding_request(AVFormatContext *s, char *buf, int buf_size)
This function handles incoming binding request messages by responding to them.
Definition: whip.c:1146
h264_annexb_insert_sps_pps
static int h264_annexb_insert_sps_pps(AVFormatContext *s, AVPacket *pkt)
Since the h264_mp4toannexb filter only processes the MP4 ISOM format and bypasses the annexb format,...
Definition: whip.c:1665
pkt
AVPacket * pkt
Definition: movenc.c:60
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:210
av_cold
#define av_cold
Definition: attributes.h:90
AV_PROFILE_UNKNOWN
#define AV_PROFILE_UNKNOWN
Definition: defs.h:65
WHIPContext::sdp_answer
char * sdp_answer
Definition: whip.c:247
ice_dtls_handshake
static int ice_dtls_handshake(AVFormatContext *s)
Definition: whip.c:1223
ffurl_open_whitelist
int ffurl_open_whitelist(URLContext **puc, const char *filename, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options, const char *whitelist, const char *blacklist, URLContext *parent)
Create an URLContext for accessing to the resource indicated by url, and open it.
Definition: avio.c:363
WHIP_STATE_OFFER
@ WHIP_STATE_OFFER
Definition: whip.c:170
ice_is_binding_request
static int ice_is_binding_request(uint8_t *b, int size)
A Binding request has class=0b00 (request) and method=0b000000000001 (Binding) and is encoded into th...
Definition: whip.c:1110
AVCodecDescriptor
This struct describes the properties of a single codec described by an AVCodecID.
Definition: codec_desc.h:38
intreadwrite.h
s
#define s(width, name)
Definition: cbs_vp9.c:198
DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC
#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC
The DTLS content type.
Definition: whip.c:88
av_new_packet
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: packet.c:99
av_lfg_get
static unsigned int av_lfg_get(AVLFG *c)
Get the next random unsigned 32-bit number using an ALFG.
Definition: lfg.h:53
WHIPContext::srtp_audio_send
SRTPContext srtp_audio_send
Definition: whip.c:280
AVFormatContext::flags
int flags
Flags modifying the (de)muxer behaviour.
Definition: avformat.h:1415
WHIPContext::whip_dtls_time
int64_t whip_dtls_time
Definition: whip.c:258
AVMEDIA_TYPE_AUDIO
@ AVMEDIA_TYPE_AUDIO
Definition: avutil.h:201
H264_NAL_PPS
@ H264_NAL_PPS
Definition: h264.h:42
WHIPContext::ice_ufrag_remote
char * ice_ufrag_remote
Definition: whip.c:235
STUN_ATTR_USE_CANDIDATE
@ STUN_ATTR_USE_CANDIDATE
shared secret response/bind request
Definition: whip.c:159
lfg.h
URLContext::flags
int flags
Definition: url.h:40
ff_url_join
int ff_url_join(char *str, int size, const char *proto, const char *authorization, const char *hostname, int port, const char *fmt,...)
Definition: url.c:40
WHIPContext::ice_ufrag_local
char ice_ufrag_local[9]
Definition: whip.c:220
AVIO_FLAG_WRITE
#define AVIO_FLAG_WRITE
write-only
Definition: avio.h:618
av_usleep
int av_usleep(unsigned usec)
Sleep for a period of time.
Definition: time.c:84
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
DTLS_SRTP_SALT_LEN
#define DTLS_SRTP_SALT_LEN
Definition: whip.c:59
avformat_write_header
av_warn_unused_result int avformat_write_header(AVFormatContext *s, AVDictionary **options)
Allocate the stream private data and write the stream header to an output media file.
Definition: mux.c:467
WHIPContext::whip_srtp_time
int64_t whip_srtp_time
Definition: whip.c:259
STUNAttr
STUNAttr
Definition: whip.c:157
ice_create_response
static int ice_create_response(AVFormatContext *s, char *tid, int tid_size, uint8_t *buf, int buf_size, int *response_size)
Create an ICE binding response.
Definition: whip.c:1046
parse_codec
static int parse_codec(AVFormatContext *s)
Parses video SPS/PPS from the extradata of codecpar and checks the codec.
Definition: whip.c:512
WHIP_STATE_READY
@ WHIP_STATE_READY
Definition: whip.c:191
AVFormatContext
Format I/O context.
Definition: avformat.h:1264
dispose_session
static int dispose_session(AVFormatContext *s)
RTC is connectionless, for it's based on UDP, so it check whether sesison is timeout.
Definition: whip.c:1609
internal.h
crc32
static unsigned crc32(const uint8_t *data, unsigned size)
Definition: crypto_bench.c:575
opts
AVDictionary * opts
Definition: movenc.c:51
AVStream::codecpar
AVCodecParameters * codecpar
Codec parameters associated with this stream.
Definition: avformat.h:767
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
STUN_ATTR_USERNAME
@ STUN_ATTR_USERNAME
Definition: whip.c:158
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:76
AVStream::time_base
AVRational time_base
This is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented.
Definition: avformat.h:783
WHIPContext::whip_init_time
int64_t whip_init_time
Definition: whip.c:253
NULL
#define NULL
Definition: coverity.c:32
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
AV_LEVEL_UNKNOWN
#define AV_LEVEL_UNKNOWN
Definition: defs.h:206
WHIPContext::srtp_recv
SRTPContext srtp_recv
Definition: whip.c:284
DTLS_VERSION_12
#define DTLS_VERSION_12
Definition: whip.c:103
certificate_key_init
static av_cold int certificate_key_init(AVFormatContext *s)
Get or Generate a self-signed certificate and private key for DTLS, fingerprint for SDP.
Definition: whip.c:323
WHIPContext::video_payload_type
uint8_t video_payload_type
Definition: whip.c:227
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:240
AVFormatContext::pb
AVIOContext * pb
I/O context.
Definition: avformat.h:1306
DTLS_STATE_CLOSED
@ DTLS_STATE_CLOSED
Definition: tls.h:42
avc.h
DTLS_SRTP_KEY_LEN
#define DTLS_SRTP_KEY_LEN
The size of the Secure Real-time Transport Protocol (SRTP) master key material that is exported by Se...
Definition: whip.c:58
options
Definition: swscale.c:43
av_hmac_update
void av_hmac_update(AVHMAC *c, const uint8_t *data, unsigned int len)
Hash data with the HMAC.
Definition: hmac.c:162
WHIPContext::key_buf
char key_buf[MAX_CERTIFICATE_SIZE]
Definition: whip.c:263
avpriv_find_start_code
const uint8_t * avpriv_find_start_code(const uint8_t *p, const uint8_t *end, uint32_t *state)
FFOutputFormat
Definition: mux.h:61
WHIP_STATE_FAILED
@ WHIP_STATE_FAILED
Definition: whip.c:193
whip_init
static av_cold int whip_init(AVFormatContext *s)
Definition: whip.c:1740
time.h
ffio_fill
void ffio_fill(AVIOContext *s, int b, int64_t count)
Definition: aviobuf.c:187
AVCodecParameters::ch_layout
AVChannelLayout ch_layout
Audio only.
Definition: codec_par.h:180
av_packet_move_ref
void av_packet_move_ref(AVPacket *dst, AVPacket *src)
Move every field in src to dst and reset src.
Definition: packet.c:489
seed
static unsigned int seed
Definition: videogen.c:78
base64.h
media_is_rtp_rtcp
static int media_is_rtp_rtcp(const uint8_t *b, int size)
In RTP packets, the first byte is represented as 0b10xxxxxx, where the initial two bits (0b10) indica...
Definition: whip.c:1131
AVCodecParameters::level
int level
Definition: codec_par.h:129
WHIPContext::ice_host
char * ice_host
Definition: whip.c:243
AVCodecParameters::sample_rate
int sample_rate
Audio only.
Definition: codec_par.h:184
AV_HMAC_SHA1
@ AV_HMAC_SHA1
Definition: hmac.h:34
AVCodecParameters::extradata_size
int extradata_size
Size of the extradata content in bytes.
Definition: codec_par.h:73
whip_muxer_class
static const AVClass whip_muxer_class
Definition: whip.c:1907
DTLS_RECORD_LAYER_HEADER_LEN
#define DTLS_RECORD_LAYER_HEADER_LEN
The DTLS record layer header has a total size of 13 bytes, consisting of ContentType (1 byte),...
Definition: whip.c:96
suite
FFmpeg currently uses a custom build this text attempts to document some of its obscure features and options Makefile the full command issued by make and its output will be shown on the screen DBG Preprocess x86 external assembler files to a dbg asm file in the object which then gets compiled Helps in developing those assembler files DESTDIR Destination directory for the install useful to prepare packages or install FFmpeg in cross environments GEN Set to ‘1’ to generate the missing or mismatched references Makefile builds all the libraries and the executables fate Run the fate test suite
Definition: build_system.txt:28
startcode.h
MAX_UDP_BUFFER_SIZE
#define MAX_UDP_BUFFER_SIZE
Maximum size of the buffer for sending and receiving UDP packets.
Definition: whip.c:112
WHIP_RTCP_PT_START
#define WHIP_RTCP_PT_START
For RTCP, PT is [128, 223] (or without marker [0, 95]).
Definition: whip.c:143
AVLFG
Context structure for the Lagged Fibonacci PRNG.
Definition: lfg.h:33
AVIOContext
Bytestream IO Context.
Definition: avio.h:160
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
AVPacket::size
int size
Definition: packet.h:553
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
avformat_alloc_context
AVFormatContext * avformat_alloc_context(void)
Allocate an AVFormatContext.
Definition: options.c:162
av_bprint_finalize
int av_bprint_finalize(AVBPrint *buf, char **ret_str)
Finalize a print buffer.
Definition: bprint.c:240
size
int size
Definition: twinvq_data.h:10344
WHIPContext::cert_file
char * cert_file
Definition: whip.c:304
AV_RB32
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_RB32
Definition: bytestream.h:96
STUN_ATTR_MESSAGE_INTEGRITY
@ STUN_ATTR_MESSAGE_INTEGRITY
bind request
Definition: whip.c:160
AVCodecParameters::profile
int profile
Codec-specific bitstream restrictions that the stream conforms to.
Definition: codec_par.h:128
AV_CODEC_ID_OPUS
@ AV_CODEC_ID_OPUS
Definition: codec_id.h:510
AVFMT_NOFILE
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:468
AV_WB24
#define AV_WB24(p, d)
Definition: intreadwrite.h:446
WHIPContext::dtls_ret
int dtls_ret
Definition: whip.c:202
AVStream::sample_aspect_ratio
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown)
Definition: avformat.h:822
options
static const AVOption options[]
Definition: whip.c:1898
ff_socket_nonblock
int ff_socket_nonblock(int socket, int enable)
avio_write
void avio_write(AVIOContext *s, const unsigned char *buf, int size)
Definition: aviobuf.c:201
avio_wb32
void avio_wb32(AVIOContext *s, unsigned int val)
Definition: aviobuf.c:365
WHIPContext::audio_par
AVCodecParameters * audio_par
Definition: whip.c:206
parse_profile_level
static int parse_profile_level(AVFormatContext *s, AVCodecParameters *par)
When duplicating a stream, the demuxer has already set the extradata, profile, and level of the par.
Definition: whip.c:445
ff_srtp_free
void ff_srtp_free(struct SRTPContext *s)
Definition: srtp.c:32
av_crc_get_table
const AVCRC * av_crc_get_table(AVCRCId crc_id)
Get an initialized standard CRC table.
Definition: crc.c:374
line
Definition: graph2dot.c:48
WHIPContext::dtls_fingerprint
char * dtls_fingerprint
Definition: whip.c:265
av_packet_make_refcounted
int av_packet_make_refcounted(AVPacket *pkt)
Ensure the data described by a given packet is reference counted.
Definition: packet.c:495
av_packet_alloc
AVPacket * av_packet_alloc(void)
Allocate an AVPacket and set its fields to default values.
Definition: packet.c:64
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:233
av_strstart
int av_strstart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str.
Definition: avstring.c:36
WHIPContext::rnd
AVLFG rnd
Definition: whip.c:217
version
version
Definition: libkvazaar.c:315
WHIPContext::whip_resource_url
char * whip_resource_url
Definition: whip.c:249
WHIP_STATE_INIT
@ WHIP_STATE_INIT
Definition: whip.c:168
av_hmac_alloc
AVHMAC * av_hmac_alloc(enum AVHMACType type)
Allocate an AVHMAC context.
Definition: hmac.c:68
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:221
WHIP_STATE_NONE
@ WHIP_STATE_NONE
Definition: whip.c:165
WHIPState
WHIPState
Definition: whip.c:164
ENC
#define ENC
Definition: whip.c:1897
ELAPSED
#define ELAPSED(starttime, endtime)
Definition: whip.c:154
av_hmac_free
void av_hmac_free(AVHMAC *c)
Free an AVHMAC context.
Definition: hmac.c:133
av_write_trailer
int av_write_trailer(AVFormatContext *s)
Write the stream trailer to an output media file and free the file private data.
Definition: mux.c:1238
av_packet_copy_props
int av_packet_copy_props(AVPacket *dst, const AVPacket *src)
Copy only "properties" fields from src to dst.
Definition: packet.c:395
generate_sdp_offer
static int generate_sdp_offer(AVFormatContext *s)
Generate SDP offer according to the codec parameters, DTLS and ICE information.
Definition: whip.c:595
bprint.h
AV_BASE64_SIZE
#define AV_BASE64_SIZE(x)
Calculate the output size needed to base64-encode x bytes to a null-terminated string.
Definition: base64.h:66
URLContext
Definition: url.h:35
AVFMT_GLOBALHEADER
#define AVFMT_GLOBALHEADER
Format wants global header.
Definition: avformat.h:477
AVOutputFormat
Definition: avformat.h:505
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
avio_internal.h
check_bitstream
static int check_bitstream(AVFormatContext *s, FFStream *sti, AVPacket *pkt)
Definition: mux.c:1056
av_hmac_init
void av_hmac_init(AVHMAC *c, const uint8_t *key, unsigned int keylen)
Initialize an AVHMAC context with an authentication key.
Definition: hmac.c:141
DTLS_STATE_FAILED
@ DTLS_STATE_FAILED
Definition: tls.h:44
exchange_sdp
static int exchange_sdp(AVFormatContext *s)
Exchange SDP offer with WebRTC peer to get the answer.
Definition: whip.c:730
whip_check_bitstream
static int whip_check_bitstream(AVFormatContext *s, AVStream *st, const AVPacket *pkt)
Definition: whip.c:1877
WHIPContext::state
enum WHIPState state
Definition: whip.c:200
create_rtp_muxer
static int create_rtp_muxer(AVFormatContext *s)
Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP packets from the enc...
Definition: whip.c:1490
ff_avio_class
const AVClass ff_avio_class
Definition: avio.c:98
AVFormatContext::max_delay
int max_delay
Definition: avformat.h:1409
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
AVFMT_EXPERIMENTAL
#define AVFMT_EXPERIMENTAL
The muxer/demuxer is experimental and should be used with caution.
Definition: avformat.h:475
setup_srtp
static int setup_srtp(AVFormatContext *s)
Establish the SRTP context using the keying material exported from DTLS.
Definition: whip.c:1352
OFFSET
#define OFFSET(x)
Definition: whip.c:1896
WHIPContext::whip_offer_time
int64_t whip_offer_time
Definition: whip.c:254
profile
int profile
Definition: mxfenc.c:2250
ff_srtp_set_crypto
int ff_srtp_set_crypto(struct SRTPContext *s, const char *suite, const char *params)
Definition: srtp.c:66
nal.h
WHIP_STATE_DTLS_CONNECTING
@ WHIP_STATE_DTLS_CONNECTING
Definition: whip.c:185
write_packet
static int write_packet(Muxer *mux, OutputStream *ost, AVPacket *pkt)
Definition: ffmpeg_mux.c:209
WHIPContext::whip_starttime
int64_t whip_starttime
Definition: whip.c:252
avcodec.h
ffurl_closep
int ffurl_closep(URLContext **hh)
Close the resource accessed by the URLContext h, and free the memory used by it.
Definition: avio.c:589
AVFMT_FLAG_BITEXACT
#define AVFMT_FLAG_BITEXACT
When muxing, try to avoid writing any random/volatile data to the output.
Definition: avformat.h:1432
ret
ret
Definition: filter_design.txt:187
AVStream
Stream structure.
Definition: avformat.h:744
ff_http_get_new_location
const char * ff_http_get_new_location(URLContext *h)
Definition: http.c:565
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:81
AVFormatContext::oformat
const struct AVOutputFormat * oformat
The output container format.
Definition: avformat.h:1283
sps
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
Definition: cbs_h264_syntax_template.c:260
whip_write_packet
static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
Definition: whip.c:1780
WHIPContext::buf
char buf[MAX_UDP_BUFFER_SIZE]
Definition: whip.c:289
av_bprintf
void av_bprintf(AVBPrint *buf, const char *fmt,...)
Definition: bprint.c:99
WHIPContext::dtls_srtp_materials
uint8_t dtls_srtp_materials[(DTLS_SRTP_KEY_LEN+DTLS_SRTP_SALT_LEN) *2]
This represents the material used to build the SRTP master key.
Definition: whip.c:272
AV_PROFILE_H264_CONSTRAINED
#define AV_PROFILE_H264_CONSTRAINED
Definition: defs.h:107
network.h
tls.h
av_get_media_type_string
const char * av_get_media_type_string(enum AVMediaType media_type)
Return a string describing the media_type enum, NULL if media_type is unknown.
Definition: utils.c:28
ff_dtls_export_materials
int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz)
Definition: tls_openssl.c:528
random_seed.h
MAX_URL_SIZE
#define MAX_URL_SIZE
Definition: internal.h:30
WHIP_STATE_UDP_CONNECTED
@ WHIP_STATE_UDP_CONNECTED
Definition: whip.c:179
buffer
the frame and frame reference mechanism is intended to as much as expensive copies of that data while still allowing the filters to produce correct results The data is stored in buffers represented by AVFrame structures Several references can point to the same frame buffer
Definition: filter_design.txt:49
media_is_rtcp
static int media_is_rtcp(const uint8_t *b, int size)
Definition: whip.c:1137
av_crc
uint32_t av_crc(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer, size_t length)
Calculate the CRC of a block.
Definition: crc.c:392
WHIPContext::av_class
AVClass * av_class
Definition: whip.c:197
WHIP_STATE_ICE_CONNECTING
@ WHIP_STATE_ICE_CONNECTING
Definition: whip.c:181
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Underlying C type is int.
Definition: opt.h:259
avformat_free_context
void avformat_free_context(AVFormatContext *s)
Free an AVFormatContext and all its streams.
Definition: avformat.c:141
WHIP_RTP_PAYLOAD_TYPE_OPUS
#define WHIP_RTP_PAYLOAD_TYPE_OPUS
Definition: whip.c:116
av_base64_encode
char * av_base64_encode(char *out, int out_size, const uint8_t *in, int in_size)
Encode data to base64 and null-terminate.
Definition: base64.c:147
AVPacket::stream_index
int stream_index
Definition: packet.h:554
WHIPContext::whip_answer_time
int64_t whip_answer_time
Definition: whip.c:255
ff_tls_set_external_socket
int ff_tls_set_external_socket(URLContext *h, URLContext *sock)
Definition: tls_openssl.c:515
WHIPContext::ice_protocol
char * ice_protocol
This represents the ICE candidate protocol, priority, host and port.
Definition: whip.c:242
WHIP_RTP_HEADER_SIZE
#define WHIP_RTP_HEADER_SIZE
The RTP header is 12 bytes long, comprising the Version(1B), PT(1B), SequenceNumber(2B),...
Definition: whip.c:131
avio_skip
int64_t avio_skip(AVIOContext *s, int64_t offset)
Skip given number of bytes forward.
Definition: aviobuf.c:318
av_gettime
int64_t av_gettime(void)
Get the current time in microseconds.
Definition: time.c:39
AV_CRC_32_IEEE_LE
@ AV_CRC_32_IEEE_LE
Definition: crc.h:53
av_dict_set_int
int av_dict_set_int(AVDictionary **pm, const char *key, int64_t value, int flags)
Convenience wrapper for av_dict_set() that converts the value to a string and stores it.
Definition: dict.c:177
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:617
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:272
desc
const char * desc
Definition: libsvtav1.c:79
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:200
av_guess_format
const AVOutputFormat * av_guess_format(const char *short_name, const char *filename, const char *mime_type)
Return the output format in the list of registered output formats which best matches the provided par...
Definition: format.c:79
ff_ssl_read_key_cert
int ff_ssl_read_key_cert(char *key_url, char *cert_url, char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint)
Definition: tls_openssl.c:160
mem.h
AVCodecParameters::video_delay
int video_delay
Video only.
Definition: codec_par.h:175
MAX_CERTIFICATE_SIZE
#define MAX_CERTIFICATE_SIZE
Maximum size limit of a certificate and private key size.
Definition: tls.h:34
AVFormatContext::start_time_realtime
int64_t start_time_realtime
Start time of the stream in real world time, in microseconds since the Unix epoch (00:00 1st January ...
Definition: avformat.h:1509
AVIOContext::buffer
unsigned char * buffer
Start of the buffer.
Definition: avio.h:225
ff_ssl_gen_key_cert
int ff_ssl_gen_key_cert(char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint)
Definition: tls_openssl.c:399
WHIPContext::authorization
char * authorization
The optional Bearer token for WHIP Authorization.
Definition: whip.c:302
WHIPContext::srtp_rtcp_send
SRTPContext srtp_rtcp_send
Definition: whip.c:282
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
ffurl_handshake
int ffurl_handshake(URLContext *c)
Perform one step of the protocol handshake to accept a new client.
Definition: avio.c:284
AVCodecParameters::codec_id
enum AVCodecID codec_id
Specific type of the encoded data (the codec used).
Definition: codec_par.h:55
WHIP_RTCP_PT_END
#define WHIP_RTCP_PT_END
Definition: whip.c:144
AVPacket
This structure stores compressed data.
Definition: packet.h:529
WHIPContext::ice_pwd_local
char ice_pwd_local[33]
Definition: whip.c:221
AVIO_FLAG_NONBLOCK
#define AVIO_FLAG_NONBLOCK
Use non-blocking mode.
Definition: avio.h:636
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
av_dict_set
int av_dict_set(AVDictionary **pm, const char *key, const char *value, int flags)
Set the given entry in *pm, overwriting an existing entry.
Definition: dict.c:86
avio_find_protocol_name
const char * avio_find_protocol_name(const char *url)
Return the name of the protocol that will handle the passed URL.
Definition: avio.c:658
h264.h
state
static struct @511 state
avio_wb16
void avio_wb16(AVIOContext *s, unsigned int val)
Definition: aviobuf.c:443
DTLS_VERSION_10
#define DTLS_VERSION_10
The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2.
Definition: whip.c:102
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ice_is_binding_response
static int ice_is_binding_response(uint8_t *b, int size)
A Binding response has class=0b10 (success response) and method=0b000000000001, and is encoded into t...
Definition: whip.c:1119
avcodec_descriptor_get
const AVCodecDescriptor * avcodec_descriptor_get(enum AVCodecID id)
Definition: codec_desc.c:3801
WHIPContext::audio_ssrc
uint32_t audio_ssrc
Definition: whip.c:223
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Underlying C type is a uint8_t* that is either NULL or points to a C string allocated with the av_mal...
Definition: opt.h:276
AV_RB24
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_WB32 unsigned int_TMPL AV_RB24
Definition: bytestream.h:97
WHIPContext::audio_payload_type
uint8_t audio_payload_type
Definition: whip.c:226
http.h
codec_desc.h
ff_nal_find_startcode
const uint8_t * ff_nal_find_startcode(const uint8_t *p, const uint8_t *end)
Definition: nal.c:68
DTLS_STATE_FINISHED
@ DTLS_STATE_FINISHED
Definition: tls.h:40
snprintf
#define snprintf
Definition: snprintf.h:34
ff_stream_add_bitstream_filter
int ff_stream_add_bitstream_filter(AVStream *st, const char *name, const char *args)
Add a bitstream filter to a stream.
Definition: mux.c:1294
ff_format_set_url
void ff_format_set_url(AVFormatContext *s, char *url)
Set AVFormatContext url field to the provided pointer.
Definition: avformat.c:861
WHIPContext::video_par
AVCodecParameters * video_par
Definition: whip.c:207
hmac.h
WHIP_STATE_NEGOTIATED
@ WHIP_STATE_NEGOTIATED
After parsing the answer received from the peer, the muxer negotiates the abilities in the offer that...
Definition: whip.c:177
ffurl_get_file_handle
int ffurl_get_file_handle(URLContext *h)
Return the file descriptor associated with this URL.
Definition: avio.c:815
avcodec_parameters_copy
int avcodec_parameters_copy(AVCodecParameters *dst, const AVCodecParameters *src)
Copy the contents of src to dst.
Definition: codec_par.c:106
WHIPContext::pkt_size
int pkt_size
The size of RTP packet, should generally be set to MTU.
Definition: whip.c:297
AVIOContext::av_class
const AVClass * av_class
A class for private options.
Definition: avio.h:173
AV_RB16
uint64_t_TMPL AV_WL64 unsigned int_TMPL AV_WL32 unsigned int_TMPL AV_WL24 unsigned int_TMPL AV_WL16 uint64_t_TMPL AV_WB64 unsigned int_TMPL AV_WB32 unsigned int_TMPL AV_WB24 unsigned int_TMPL AV_RB16
Definition: bytestream.h:98
MAX_SDP_SIZE
#define MAX_SDP_SIZE
Maximum size limit of a Session Description Protocol (SDP), be it an offer or answer.
Definition: whip.c:50
avio_feof
int avio_feof(AVIOContext *s)
Similar to feof() but also returns nonzero on read errors.
Definition: aviobuf.c:346
ffurl_read
static int ffurl_read(URLContext *h, uint8_t *buf, int size)
Read up to size bytes from the resource accessed by h, and store the read bytes in buf.
Definition: url.h:181
mux.h
ff_write_chained
int ff_write_chained(AVFormatContext *dst, int dst_stream, AVPacket *pkt, AVFormatContext *src, int interleave)
Write a packet to another muxer than the one the user originally intended.
Definition: mux.c:1337