From e1c7704cfdef76cd5b177ba7fbb92c041bddd40e Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 14 Jul 2023 12:08:00 +0200 Subject: [PATCH 01/34] Added new videos to FAQ --- src/mainpage.dox | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mainpage.dox b/src/mainpage.dox index 52ba916920..2d503c9c33 100644 --- a/src/mainpage.dox +++ b/src/mainpage.dox @@ -4067,7 +4067,9 @@ ldd janus | grep asan * - Presentation on Janode, the official Janus nodejs SDK, at CommCon 2021; * - Presentation on WebRTC broadcasting with WHIP and how to scale at FOSDEM 2022; * - Presentation on SIP call transfers and multiple calls at OpenSIPS 2022; - * - Presentation on SFU/VideoRoom cascading at IIT RTC 2022. \n\n + * - Presentation on SFU/VideoRoom cascading at IIT RTC 2022; + * - Presentation on Social audio applications with Janus at FOSDEM 2023; + * - Presentation on Real-Time Text in SIP and WebRTC at Kamailio World 2023. \n\n * Apart from these presentations, make sure you also check the slides * and presentations from JanusCon, * the Janus conference we hosted here in Napoli. From 85925598d1a78dafb740e71684f6f117c14c4744 Mon Sep 17 00:00:00 2001 From: Adnan Elezovic Date: Fri, 4 Aug 2023 11:40:50 +0200 Subject: [PATCH 02/34] Fix simulated leave message for longer string ID rooms in TextRoom (#3243) --- src/plugins/janus_textroom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/janus_textroom.c b/src/plugins/janus_textroom.c index 4d9d694559..16eab29202 100644 --- a/src/plugins/janus_textroom.c +++ b/src/plugins/janus_textroom.c @@ -3027,7 +3027,7 @@ static void janus_textroom_hangup_media_internal(janus_plugin_session *handle) { } janus_mutex_unlock(&session->mutex); JANUS_LOG(LOG_VERB, "Leaving %d rooms\n", g_list_length(list)); - char request[100]; + char request[200]; GList *first = list; while(list) { char *room_id_str = (char *)list->data; From b8e510c7f10716ab65058a28a3c4ed14b176a998 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 4 Aug 2023 18:54:19 +0900 Subject: [PATCH 03/34] Use speexdsp's jitter buffer in the AudioBridge (see #3214) (#3233) --- .github/workflows/janus-ci.yml | 1 + conf/janus.plugin.audiobridge.jcfg.sample | 1 - configure.ac | 7 +- src/plugins/janus_audiobridge.c | 526 ++++++++-------------- 4 files changed, 200 insertions(+), 335 deletions(-) diff --git a/.github/workflows/janus-ci.yml b/.github/workflows/janus-ci.yml index 9768d1624a..2e5e65c174 100644 --- a/.github/workflows/janus-ci.yml +++ b/.github/workflows/janus-ci.yml @@ -58,6 +58,7 @@ jobs: libopus-dev librabbitmq-dev libsofia-sip-ua-dev + libspeexdsp-dev libssl-dev libtool libvorbis-dev diff --git a/conf/janus.plugin.audiobridge.jcfg.sample b/conf/janus.plugin.audiobridge.jcfg.sample index 39d41d5a55..cd98be559d 100644 --- a/conf/janus.plugin.audiobridge.jcfg.sample +++ b/conf/janus.plugin.audiobridge.jcfg.sample @@ -10,7 +10,6 @@ # audiolevel_event = true|false (whether to emit event to other users or not, default=false) # audio_active_packets = 100 (number of packets with audio level, default=100, 2 seconds) # audio_level_average = 25 (average value of audio level, 127=muted, 0='too loud', default=25) -# default_prebuffering = number of packets to buffer before decoding each particiant (default=6) # default_expectedloss = percent of packets we expect participants may miss, to help with FEC (default=0, max=20; automatically used for forwarders too) # default_bitrate = default bitrate in bps to use for the all participants (default=0, which means libopus decides; automatically used for forwarders too) # record = true|false (whether this room should be recorded, default=false) diff --git a/configure.ac b/configure.ac index 0715f79a86..b7b9588b16 100644 --- a/configure.ac +++ b/configure.ac @@ -788,14 +788,17 @@ AC_SUBST([SOFIA_CFLAGS]) AC_SUBST([SOFIA_LIBS]) PKG_CHECK_MODULES([OPUS], - [opus], + [ + opus + speexdsp + ], [ AS_IF([test "x$enable_plugin_audiobridge" = "xmaybe"], [enable_plugin_audiobridge=yes]) ], [ AS_IF([test "x$enable_plugin_audiobridge" = "xyes"], - [AC_MSG_ERROR([libopus not found. See README.md for installation instructions or use --disable-plugin-audiobridge])]) + [AC_MSG_ERROR([libopus or libspeexdsp not found. See README.md for installation instructions or use --disable-plugin-audiobridge])]) ]) AC_SUBST([OPUS_CFLAGS]) AC_SUBST([OPUS_LIBS]) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 9bd6b1bf3a..72434659dc 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -38,7 +38,6 @@ room-: { audiolevel_event = true|false (whether to emit event to other users or not, default=false) audio_active_packets = 100 (number of packets with audio level, default=100, 2 seconds) audio_level_average = 25 (average value of audio level, 127=muted, 0='too loud', default=25) - default_prebuffering = number of packets to buffer before decoding each participant (default=DEFAULT_PREBUFFERING) default_expectedloss = percent of packets we expect participants may miss, to help with FEC (default=0, max=20; automatically used for forwarders too) default_bitrate = default bitrate in bps to use for the all participants (default=0, which means libopus decides; automatically used for forwarders too) record = true|false (whether this room should be recorded, default=false) @@ -141,7 +140,6 @@ room-: { "audiolevel_event" : , "audio_active_packets" : , "audio_level_average" : , - "default_prebuffering" : , "default_expectedloss" : , "default_bitrate" : , "record" : , @@ -739,7 +737,6 @@ room-: { "token" : "", "muted" : , "codec" : "", - "prebuffer" : , "bitrate" : , "quality" : <0-10, Opus-related complexity to use, the higher the value, the better the quality (but more CPU); optional, default is 4>, "expected_loss" : <0-20, a percentage of the expected loss (capped at 20%), only needed in case FEC is used; optional, default is 0 (FEC disabled even when negotiated) or the room default>, @@ -827,7 +824,6 @@ room-: { "request" : "configure", "muted" : , "display" : "", - "prebuffer" : , "bitrate" : , "quality" : , "expected_loss" : @@ -1034,6 +1030,8 @@ room-: { #ifdef HAVE_LIBOGG #include #endif +#include + #include #include #include @@ -1174,7 +1172,6 @@ static struct janus_json_parameter create_parameters[] = { {"audiolevel_event", JANUS_JSON_BOOL, 0}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_level_average", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, - {"default_prebuffering", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"default_expectedloss", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"default_bitrate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"groups", JSON_ARRAY, 0} @@ -1206,7 +1203,6 @@ static struct janus_json_parameter join_parameters[] = { {"group", JSON_STRING, 0}, {"muted", JANUS_JSON_BOOL, 0}, {"codec", JSON_STRING, 0}, - {"prebuffer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"bitrate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"quality", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"expected_loss", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, @@ -1238,7 +1234,6 @@ static struct janus_json_parameter rtp_parameters[] = { }; static struct janus_json_parameter configure_parameters[] = { {"muted", JANUS_JSON_BOOL, 0}, - {"prebuffer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"bitrate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"quality", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"expected_loss", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, @@ -1329,7 +1324,6 @@ typedef struct janus_audiobridge_room { gboolean spatial_audio; /* Whether the mix will use spatial audio, using stereo */ gboolean audiolevel_ext; /* Whether the ssrc-audio-level extension must be negotiated or not for new joins */ gboolean audiolevel_event; /* Whether to emit event to other users about audiolevel */ - uint default_prebuffering; /* Number of packets to buffer before decoding each participant */ uint default_expectedloss; /* Percent of packets we expect participants may miss, to help with FEC: can be overridden per-participant */ int32_t default_bitrate; /* Default bitrate to use for all Opus streams when encoding */ int audio_active_packets; /* Amount of packets with audio level for checkup */ @@ -1559,8 +1553,6 @@ typedef struct janus_audiobridge_participant { gchar *user_id_str; /* Unique ID in the room (when using strings) */ gchar *display; /* Display name (opaque value, only meaningful to application) */ gboolean admin; /* If the participant is an admin (can't be globally muted) */ - gboolean prebuffering; /* Whether this participant needs pre-buffering of a few packets (just joined) */ - uint prebuffer_count; /* Number of packets to buffer before decoding this participant */ volatile gint active; /* Whether this participant can receive media at all */ volatile gint encoding; /* Whether this participant is currently encoding */ volatile gint decoding; /* Whether this participant is currently decoding */ @@ -1571,9 +1563,9 @@ typedef struct janus_audiobridge_participant { gboolean stereo; /* Whether stereo will be used for spatial audio */ int spatial_position; /* Panning of this participant in the mix */ /* RTP stuff */ - GList *inbuf; /* Incoming audio from this participant, as an ordered list of packets */ - GAsyncQueue *outbuf; /* Mixed audio for this participant */ - gint64 last_drop; /* When we last dropped a packet because the imcoming queue was full */ + JitterBuffer *jitter; /* Jitter buffer of incoming audio packets */ + GList *inbuf; /* Decoded audio from this participant, to feed to the mixer */ + GAsyncQueue *outbuf; /* Mixed audio to send to this participant */ janus_mutex qmutex; /* Incoming queue mutex */ int opus_pt; /* Opus payload type */ int extmap_id; /* Audio level RTP extension id, if any */ @@ -1620,6 +1612,32 @@ typedef struct janus_audiobridge_rtp_relay_packet { gboolean silence; } janus_audiobridge_rtp_relay_packet; +/* Buffered audio/video packet */ +typedef struct janus_audiobridge_buffer_packet { + /* Pointer to the packet data, if RTP */ + char *buffer; + /* Size of the packet */ + int len; + /* Whether the packet contains silence, according to the RTP extension */ + gboolean silence; + /* Monotonic insert time */ + int64_t inserted; +} janus_audiobridge_buffer_packet; +static janus_audiobridge_buffer_packet *janus_audiobridge_buffer_packet_create(char *buffer, int len, gboolean silence) { + janus_audiobridge_buffer_packet *pkt = g_malloc(sizeof(janus_audiobridge_buffer_packet)); + pkt->buffer = g_malloc(len); + pkt->len = len; + pkt->silence = silence; + memcpy(pkt->buffer, buffer, len); + pkt->inserted = janus_get_monotonic_time(); + return pkt; +} +static void janus_audiobridge_buffer_packet_destroy(janus_audiobridge_buffer_packet *pkt) { + if(!pkt) + return; + g_free(pkt->buffer); + g_free(pkt); +} static void janus_audiobridge_participant_destroy(janus_audiobridge_participant *participant) { if(!participant) @@ -1646,6 +1664,8 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant opus_encoder_destroy(participant->encoder); if(participant->decoder) opus_decoder_destroy(participant->decoder); + if(participant->jitter) + jitter_buffer_destroy(participant->jitter); while(participant->inbuf) { GList *first = g_list_first(participant->inbuf); janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; @@ -1783,26 +1803,6 @@ static guint32 janus_audiobridge_rtp_forwarder_add_helper(janus_audiobridge_room return rf->stream_id; } - -/* Helper to sort incoming RTP packets by sequence numbers */ -static gint janus_audiobridge_rtp_sort(gconstpointer a, gconstpointer b) { - janus_audiobridge_rtp_relay_packet *pkt1 = (janus_audiobridge_rtp_relay_packet *)a; - janus_audiobridge_rtp_relay_packet *pkt2 = (janus_audiobridge_rtp_relay_packet *)b; - if(pkt1->seq_number < 100 && pkt2->seq_number > 65000) { - /* Sequence number was probably reset, pkt2 is older */ - return 1; - } else if(pkt2->seq_number < 100 && pkt1->seq_number > 65000) { - /* Sequence number was probably reset, pkt1 is older */ - return -1; - } - /* Simply compare timestamps */ - if(pkt1->seq_number < pkt2->seq_number) - return -1; - else if(pkt1->seq_number > pkt2->seq_number) - return 1; - return 0; -} - /* Helper struct to generate and parse WAVE headers */ typedef struct wav_header { char riff[4]; @@ -1992,11 +1992,6 @@ static int janus_audiobridge_resample(int16_t *input, int input_num, int input_r } -/* Mixer settings */ -#define DEFAULT_PREBUFFERING 6 -#define MAX_PREBUFFERING 50 - - /* Opus settings */ #define OPUS_SAMPLES 960 #define G711_SAMPLES 160 @@ -2384,7 +2379,6 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { janus_config_item *audiolevel_event = janus_config_get(config, cat, janus_config_type_item, "audiolevel_event"); janus_config_item *audio_active_packets = janus_config_get(config, cat, janus_config_type_item, "audio_active_packets"); janus_config_item *audio_level_average = janus_config_get(config, cat, janus_config_type_item, "audio_level_average"); - janus_config_item *default_prebuffering = janus_config_get(config, cat, janus_config_type_item, "default_prebuffering"); janus_config_item *default_expectedloss = janus_config_get(config, cat, janus_config_type_item, "default_expectedloss"); janus_config_item *default_bitrate = janus_config_get(config, cat, janus_config_type_item, "default_bitrate"); janus_config_item *secret = janus_config_get(config, cat, janus_config_type_item, "secret"); @@ -2484,15 +2478,6 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { } } } - audiobridge->default_prebuffering = DEFAULT_PREBUFFERING; - if(default_prebuffering != NULL && default_prebuffering->value != NULL) { - int prebuffering = atoi(default_prebuffering->value); - if(prebuffering < 0 || prebuffering > MAX_PREBUFFERING) { - JANUS_LOG(LOG_WARN, "Invalid default_prebuffering value provided, using default: %d\n", audiobridge->default_prebuffering); - } else { - audiobridge->default_prebuffering = prebuffering; - } - } audiobridge->default_expectedloss = 0; if(default_expectedloss != NULL && default_expectedloss->value != NULL) { int expectedloss = atoi(default_expectedloss->value); @@ -2825,17 +2810,15 @@ json_t *janus_audiobridge_query_session(janus_plugin_session *handle) { json_object_set_new(info, "admin", json_true()); json_object_set_new(info, "muted", participant->muted ? json_true() : json_false()); json_object_set_new(info, "active", g_atomic_int_get(&participant->active) ? json_true() : json_false()); - json_object_set_new(info, "pre-buffering", participant->prebuffering ? json_true() : json_false()); - json_object_set_new(info, "prebuffer-count", json_integer(participant->prebuffer_count)); - if(participant->inbuf) { - janus_mutex_lock(&participant->qmutex); - json_object_set_new(info, "queue-in", json_integer(g_list_length(participant->inbuf))); - janus_mutex_unlock(&participant->qmutex); - } + janus_mutex_lock(&participant->qmutex); + spx_int32_t count = 0; + if(participant->jitter) + jitter_buffer_ctl(participant->jitter, JITTER_BUFFER_GET_AVALIABLE_COUNT, &count); + json_object_set_new(info, "buffer-in", json_integer(count)); + json_object_set_new(info, "queue-in", json_integer(g_list_length(participant->inbuf))); + janus_mutex_unlock(&participant->qmutex); if(participant->outbuf) json_object_set_new(info, "queue-out", json_integer(g_async_queue_length(participant->outbuf))); - if(participant->last_drop > 0) - json_object_set_new(info, "last-drop", json_integer(participant->last_drop)); if(participant->stereo) json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); if(participant->arc && participant->arc->filename) @@ -2972,7 +2955,6 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_t *audiolevel_event = json_object_get(root, "audiolevel_event"); json_t *audio_active_packets = json_object_get(root, "audio_active_packets"); json_t *audio_level_average = json_object_get(root, "audio_level_average"); - json_t *default_prebuffering = json_object_get(root, "default_prebuffering"); json_t *default_expectedloss = json_object_get(root, "default_expectedloss"); json_t *default_bitrate = json_object_get(root, "default_bitrate"); json_t *groups = json_object_get(root, "groups"); @@ -3125,13 +3107,6 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s audiobridge->audio_level_average); } } - audiobridge->default_prebuffering = default_prebuffering ? - json_integer_value(default_prebuffering) : DEFAULT_PREBUFFERING; - if(audiobridge->default_prebuffering > MAX_PREBUFFERING) { - audiobridge->default_prebuffering = DEFAULT_PREBUFFERING; - JANUS_LOG(LOG_WARN, "Invalid default_prebuffering value provided (too high), using default: %d\n", - audiobridge->default_prebuffering); - } audiobridge->default_expectedloss = 0; if(default_expectedloss != NULL) { int expectedloss = json_integer_value(default_expectedloss); @@ -3298,10 +3273,6 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s janus_config_add(config, c, janus_config_item_create("audio_level_average", value)); } } - if(audiobridge->default_prebuffering != DEFAULT_PREBUFFERING) { - g_snprintf(value, BUFSIZ, "%d", audiobridge->default_prebuffering); - janus_config_add(config, c, janus_config_item_create("default_prebuffering", value)); - } if(audiobridge->allow_plainrtp) janus_config_add(config, c, janus_config_item_create("allow_rtp_participants", "yes")); if(audiobridge->groups) { @@ -3483,10 +3454,6 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s janus_config_add(config, c, janus_config_item_create("audio_level_average", value)); } } - if(audiobridge->default_prebuffering != DEFAULT_PREBUFFERING) { - g_snprintf(value, BUFSIZ, "%d", audiobridge->default_prebuffering); - janus_config_add(config, c, janus_config_item_create("default_prebuffering", value)); - } if(audiobridge->allow_plainrtp) janus_config_add(config, c, janus_config_item_create("allow_rtp_participants", "yes")); if(audiobridge->groups) { @@ -4087,6 +4054,8 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s if(participant->muted) { /* Clear the queued packets waiting to be handled */ janus_mutex_lock(&participant->qmutex); + if(participant->jitter) + jitter_buffer_reset(participant->jitter); while(participant->inbuf) { GList *first = g_list_first(participant->inbuf); janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; @@ -5550,7 +5519,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } participant->reset = FALSE; } - /* Decode frame (Opus/G.711 -> slinear) */ + /* We'll need to decode the frame (Opus/G.711 -> slinear), so check the payload type */ janus_rtp_header *rtp = (janus_rtp_header *)buf; if((participant->codec == JANUS_AUDIOCODEC_PCMA && rtp->type != 8) || (participant->codec == JANUS_AUDIOCODEC_PCMU && rtp->type != 0)) { @@ -5558,43 +5527,14 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r rtp->type, participant->codec == JANUS_AUDIOCODEC_PCMA ? 8 : 0); return; } - janus_audiobridge_rtp_relay_packet *pkt = g_malloc(sizeof(janus_audiobridge_rtp_relay_packet)); - pkt->data = g_malloc0(BUFFER_SAMPLES*sizeof(opus_int16)); - pkt->ssrc = 0; - pkt->timestamp = ntohl(rtp->timestamp); - pkt->seq_number = ntohs(rtp->seq_number); - /* We might check the audio level extension to see if this is silence */ - pkt->silence = FALSE; - pkt->length = 0; - - /* First check if probation period */ - if(participant->probation == MIN_SEQUENTIAL) { - participant->probation--; - participant->expected_seq = pkt->seq_number + 1; - JANUS_LOG(LOG_VERB, "Probation started with ssrc = %"SCNu32", seq = %"SCNu16" \n", ntohl(rtp->ssrc), pkt->seq_number); - g_free(pkt->data); - g_free(pkt); - return; - } else if(participant->probation != 0) { - /* Decrease probation */ - participant->probation--; - /* TODO: Reset probation if sequence number is incorrect and DSSRC also; must have a correct sequence */ - if(!participant->probation){ - /* Probation is ended */ - JANUS_LOG(LOG_VERB, "Probation ended with ssrc = %"SCNu32", seq = %"SCNu16" \n", ntohl(rtp->ssrc), pkt->seq_number); - } - participant->expected_seq = pkt->seq_number + 1; - g_free(pkt->data); - g_free(pkt); - return; - } - + /* Check the audio levels, in case we need to notify participants about who's talking */ + gboolean silence = FALSE; if(participant->extmap_id > 0) { /* Check the audio levels, in case we need to notify participants about who's talking */ int level = packet->extensions.audio_level; if(level != -1) { /* Is this silence? */ - pkt->silence = (level == 127); + silence = (level == 127); if(participant->room && participant->room->audiolevel_event) { /* We also need to detect who's talking: update our monitoring stuff */ int audio_active_packets = participant->room ? participant->room->audio_active_packets : 100; @@ -5652,188 +5592,18 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } } } - if(!g_atomic_int_compare_and_exchange(&participant->decoding, 0, 1)) { - /* This means we're cleaning up, so don't try to decode */ - g_free(pkt->data); - g_free(pkt); - return; - } - int plen = 0; - const unsigned char *payload = (const unsigned char *)janus_rtp_payload(buf, len, &plen); - if(!payload) { - g_atomic_int_set(&participant->decoding, 0); - JANUS_LOG(LOG_ERR, "[%s] Ops! got an error accessing the RTP payload\n", - participant->codec == JANUS_AUDIOCODEC_OPUS ? "Opus" : "G.711"); - g_free(pkt->data); - g_free(pkt); - return; - } - /* Check sequence number received, verify if it's relevant to the expected one */ - if(pkt->seq_number == participant->expected_seq) { - /* Regular decode */ - if(participant->codec == JANUS_AUDIOCODEC_OPUS) { - /* Opus */ - pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)pkt->data, BUFFER_SAMPLES, 0); - } else if(participant->codec == JANUS_AUDIOCODEC_PCMA || participant->codec == JANUS_AUDIOCODEC_PCMU) { - /* G.711 */ - if(plen != 160) { - g_atomic_int_set(&participant->decoding, 0); - JANUS_LOG(LOG_WARN, "[G.711] Wrong packet size (expected 160, got %d), skipping audio packet\n", plen); - g_free(pkt->data); - g_free(pkt); - return; - } - int i = 0; - uint16_t *samples = (uint16_t *)pkt->data; - if(rtp->type == 0) { - /* mu-law */ - for(i=0; itype == 8) { - /* a-law */ - for(i=0; ilength = 320; - } - /* Update last_timestamp */ - participant->last_timestamp = pkt->timestamp; - /* Increment according to previous seq_number */ - participant->expected_seq = pkt->seq_number + 1; - } else if(pkt->seq_number > participant->expected_seq) { - /* Sequence(s) losts */ - uint16_t gap = pkt->seq_number - participant->expected_seq; - JANUS_LOG(LOG_HUGE, "%"SCNu16" sequence(s) lost, sequence = %"SCNu16", expected seq = %"SCNu16"\n", - gap, pkt->seq_number, participant->expected_seq); - - /* Use FEC if sequence lost < DEFAULT_PREBUFFERING (or any custom value) */ - uint16_t start_lost_seq = participant->expected_seq; - if(participant->codec == JANUS_AUDIOCODEC_OPUS && participant->fec && gap < participant->prebuffer_count) { - uint8_t i=0; - for(i=1; i<=gap ; i++) { - int32_t output_samples; - janus_audiobridge_rtp_relay_packet *lost_pkt = g_malloc(sizeof(janus_audiobridge_rtp_relay_packet)); - lost_pkt->data = g_malloc0(BUFFER_SAMPLES*sizeof(opus_int16)); - lost_pkt->ssrc = 0; - lost_pkt->timestamp = participant->last_timestamp + (i * OPUS_SAMPLES); - lost_pkt->seq_number = start_lost_seq++; - lost_pkt->silence = FALSE; - lost_pkt->length = 0; - if(i == gap) { - /* Attempt to decode with in-band FEC from next packet */ - opus_decoder_ctl(participant->decoder, OPUS_GET_LAST_PACKET_DURATION(&output_samples)); - lost_pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)lost_pkt->data, output_samples, 1); - } else { - opus_decoder_ctl(participant->decoder, OPUS_GET_LAST_PACKET_DURATION(&output_samples)); - lost_pkt->length = opus_decode(participant->decoder, NULL, plen, (opus_int16 *)lost_pkt->data, output_samples, 1); - } - if(lost_pkt->length < 0) { - g_atomic_int_set(&participant->decoding, 0); - JANUS_LOG(LOG_ERR, "[Opus] Ops! got an error decoding the Opus frame: %d (%s)\n", lost_pkt->length, opus_strerror(lost_pkt->length)); - g_free(lost_pkt->data); - g_free(lost_pkt); - g_free(pkt->data); - g_free(pkt); - return; - } - /* Enqueue the decoded frame */ - janus_mutex_lock(&participant->qmutex); - /* Insert packets sorting by sequence number */ - participant->inbuf = g_list_insert_sorted(participant->inbuf, lost_pkt, &janus_audiobridge_rtp_sort); - janus_mutex_unlock(&participant->qmutex); - } - } - /* Then go with the regular decode (no FEC) */ - if(participant->codec == JANUS_AUDIOCODEC_OPUS) { - /* Opus */ - pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)pkt->data, BUFFER_SAMPLES, 0); - } else if(participant->codec == JANUS_AUDIOCODEC_PCMA || participant->codec == JANUS_AUDIOCODEC_PCMU) { - /* G.711 */ - if(plen != 160) { - g_atomic_int_set(&participant->decoding, 0); - JANUS_LOG(LOG_WARN, "[G.711] Wrong packet size (expected 160, got %d), skipping audio packet\n", plen); - g_free(pkt->data); - g_free(pkt); - return; - } - int i = 0; - uint16_t *samples = (uint16_t *)pkt->data; - if(rtp->type == 0) { - /* mu-law */ - for(i=0; itype == 8) { - /* a-law */ - for(i=0; ilength = 320; - } - /* Increment according to previous seq_number */ - participant->expected_seq = pkt->seq_number + 1; - } else { - /* In late sequence or sequence wrapped */ - g_atomic_int_set(&participant->decoding, 0); - if((participant->expected_seq - pkt->seq_number) > MAX_MISORDER){ - JANUS_LOG(LOG_HUGE, "SN WRAPPED seq = %"SCNu16", expected_seq = %"SCNu16"\n", pkt->seq_number, participant->expected_seq); - participant->expected_seq = pkt->seq_number + 1; - } else { - JANUS_LOG(LOG_WARN, "IN LATE SN seq = %"SCNu16", expected_seq = %"SCNu16"\n", pkt->seq_number, participant->expected_seq); - } - g_free(pkt->data); - g_free(pkt); - return; - } - g_atomic_int_set(&participant->decoding, 0); - if(pkt->length < 0) { - if(participant->codec == JANUS_AUDIOCODEC_OPUS) { - JANUS_LOG(LOG_ERR, "[Opus] Ops! got an error decoding the Opus frame: %d (%s)\n", pkt->length, opus_strerror(pkt->length)); - } else { - JANUS_LOG(LOG_ERR, "[G.711] Ops! got an error decoding the audio frame\n"); - } - g_free(pkt->data); - g_free(pkt); - return; - } - /* Enqueue the decoded frame */ - janus_mutex_lock(&participant->qmutex); - /* Insert packets sorting by sequence number */ - participant->inbuf = g_list_insert_sorted(participant->inbuf, pkt, &janus_audiobridge_rtp_sort); - if(participant->prebuffering) { - /* Still pre-buffering: do we have enough packets now? */ - if(g_list_length(participant->inbuf) > participant->prebuffer_count) { - participant->prebuffering = FALSE; - JANUS_LOG(LOG_VERB, "Prebuffering done! Finally adding the user to the mix\n"); - } else { - JANUS_LOG(LOG_VERB, "Still prebuffering (got %d packets), not adding the user to the mix yet\n", g_list_length(participant->inbuf)); - } - } else { - /* Make sure we're not queueing too many packets: if so, get rid of the older ones */ - if(g_list_length(participant->inbuf) >= participant->prebuffer_count*2) { - gint64 now = janus_get_monotonic_time(); - if(now - participant->last_drop > 5*G_USEC_PER_SEC) { - JANUS_LOG(LOG_VERB, "Too many packets in queue (%d > %d), removing older ones\n", - g_list_length(participant->inbuf), participant->prebuffer_count*2); - participant->last_drop = now; - } - while(g_list_length(participant->inbuf) > participant->prebuffer_count) { - /* Remove this packet: it's too old */ - GList *first = g_list_first(participant->inbuf); - janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; - JANUS_LOG(LOG_VERB, "List length = %d, Remove sequence = %d\n", - g_list_length(participant->inbuf), pkt->seq_number); - participant->inbuf = g_list_delete_link(participant->inbuf, first); - first = NULL; - if(pkt == NULL) - continue; - g_free(pkt->data); - pkt->data = NULL; - g_free(pkt); - pkt = NULL; - } - } + /* Queue the audio packet in the jitter buffer (we won't decode now, there might be buffering involved) */ + if(participant->jitter) { + janus_audiobridge_buffer_packet *pkt = janus_audiobridge_buffer_packet_create(buf, len, silence); + janus_mutex_lock(&participant->qmutex); + JitterBufferPacket jbp = {0}; + jbp.data = (char *)pkt; + jbp.len = 0; + jbp.timestamp = ntohl(rtp->timestamp); + jbp.span = (participant->codec == JANUS_AUDIOCODEC_OPUS ? 960 : 160); + jitter_buffer_put(participant->jitter, &jbp); + janus_mutex_unlock(&participant->qmutex); } - janus_mutex_unlock(&participant->qmutex); } } @@ -5967,7 +5737,6 @@ static void janus_audiobridge_hangup_media_internal(janus_plugin_session *handle participant->muted = TRUE; g_free(participant->display); participant->display = NULL; - participant->prebuffering = TRUE; /* Make sure we're not using the encoder/decoder right now, we're going to destroy them */ while(!g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) g_usleep(5000); @@ -5988,6 +5757,8 @@ static void janus_audiobridge_hangup_media_internal(janus_plugin_session *handle g_free(participant->mjr_base); participant->mjr_base = NULL; /* Get rid of queued packets */ + if(participant->jitter) + jitter_buffer_reset(participant->jitter); while(participant->inbuf) { GList *first = g_list_first(participant->inbuf); janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; @@ -6000,7 +5771,6 @@ static void janus_audiobridge_hangup_media_internal(janus_plugin_session *handle g_free(pkt); pkt = NULL; } - participant->last_drop = 0; janus_mutex_unlock(&participant->qmutex); if(audiobridge != NULL) { janus_mutex_unlock(&audiobridge->mutex); @@ -6216,7 +5986,6 @@ static void *janus_audiobridge_handler(void *data) { json_t *display = json_object_get(root, "display"); const char *display_text = display ? json_string_value(display) : NULL; json_t *muted = json_object_get(root, "muted"); - json_t *prebuffer = json_object_get(root, "prebuffer"); json_t *gain = json_object_get(root, "volume"); json_t *spatial = json_object_get(root, "spatial_position"); json_t *bitrate = json_object_get(root, "bitrate"); @@ -6228,12 +5997,6 @@ static void *janus_audiobridge_handler(void *data) { json_t *record = json_object_get(root, "record"); json_t *recfile = json_object_get(root, "filename"); json_t *gen_offer = json_object_get(root, "generate_offer"); - uint prebuffer_count = prebuffer ? json_integer_value(prebuffer) : audiobridge->default_prebuffering; - if(prebuffer_count > MAX_PREBUFFERING) { - prebuffer_count = audiobridge->default_prebuffering; - JANUS_LOG(LOG_WARN, "Invalid prebuffering value provided (too high), using room default: %d\n", - audiobridge->default_prebuffering); - } int volume = gain ? json_integer_value(gain) : 100; int spatial_position = spatial ? json_integer_value(spatial) : 50; int32_t opus_bitrate = audiobridge->default_bitrate; @@ -6330,11 +6093,11 @@ static void *janus_audiobridge_handler(void *data) { janus_refcount_init(&participant->ref, janus_audiobridge_participant_free); g_atomic_int_set(&participant->active, 0); participant->codec = codec; - participant->prebuffering = TRUE; participant->display = NULL; + participant->jitter = jitter_buffer_init(participant->codec == JANUS_AUDIOCODEC_OPUS ? 960 : 160); + jitter_buffer_ctl(participant->jitter, JITTER_BUFFER_SET_DESTROY_CALLBACK, &janus_audiobridge_buffer_packet_destroy); participant->inbuf = NULL; participant->outbuf = NULL; - participant->last_drop = 0; participant->encoder = NULL; participant->decoder = NULL; participant->reset = FALSE; @@ -6357,7 +6120,6 @@ static void *janus_audiobridge_handler(void *data) { participant->admin = admin; participant->display = display_text ? g_strdup(display_text) : NULL; participant->muted = muted ? json_is_true(muted) : FALSE; /* By default, everyone's unmuted when joining */ - participant->prebuffer_count = prebuffer_count; participant->volume_gain = volume; participant->opus_complexity = complexity; participant->opus_bitrate = opus_bitrate; @@ -6686,7 +6448,6 @@ static void *janus_audiobridge_handler(void *data) { if(error_code != 0) goto error; json_t *muted = json_object_get(root, "muted"); - json_t *prebuffer = json_object_get(root, "prebuffer"); json_t *bitrate = json_object_get(root, "bitrate"); json_t *quality = json_object_get(root, "quality"); json_t *exploss = json_object_get(root, "expected_loss"); @@ -6698,32 +6459,6 @@ static void *janus_audiobridge_handler(void *data) { json_t *group = json_object_get(root, "group"); json_t *gen_offer = json_object_get(root, "generate_offer"); json_t *update = json_object_get(root, "update"); - if(prebuffer) { - uint prebuffer_count = json_integer_value(prebuffer); - if(prebuffer_count > MAX_PREBUFFERING) { - JANUS_LOG(LOG_WARN, "Invalid prebuffering value provided (too high), keeping previous value: %d\n", - participant->prebuffer_count); - } else if(prebuffer_count != participant->prebuffer_count) { - janus_mutex_lock(&participant->qmutex); - if(prebuffer_count < participant->prebuffer_count) { - /* We're switching to a shorter prebuffer, trim the incoming buffer */ - while(g_list_length(participant->inbuf) > prebuffer_count) { - GList *first = g_list_first(participant->inbuf); - janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; - participant->inbuf = g_list_delete_link(participant->inbuf, first); - if(pkt == NULL) - continue; - g_free(pkt->data); - g_free(pkt); - } - } else { - /* Reset the prebuffering state */ - participant->prebuffering = TRUE; - } - participant->prebuffer_count = prebuffer_count; - janus_mutex_unlock(&participant->qmutex); - } - } if(gain) participant->volume_gain = json_integer_value(gain); if(bitrate) { @@ -6779,6 +6514,8 @@ static void *janus_audiobridge_handler(void *data) { if(participant->muted) { /* Clear the queued packets waiting to be handled */ janus_mutex_lock(&participant->qmutex); + if(participant->jitter) + jitter_buffer_reset(participant->jitter); while(participant->inbuf) { GList *first = g_list_first(participant->inbuf); janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; @@ -7114,7 +6851,6 @@ static void *janus_audiobridge_handler(void *data) { } JANUS_LOG(LOG_VERB, " -- Participant ID in new room %s: %s\n", room_id_str, user_id_str); } - participant->prebuffering = TRUE; participant->audio_active_packets = 0; participant->audio_dBov_sum = 0; participant->talking = FALSE; @@ -7398,7 +7134,8 @@ static void *janus_audiobridge_handler(void *data) { /* Get rid of queued packets */ janus_mutex_lock(&participant->qmutex); g_atomic_int_set(&participant->active, 0); - participant->prebuffering = TRUE; + if(participant->jitter) + jitter_buffer_reset(participant->jitter); while(participant->inbuf) { GList *first = g_list_first(participant->inbuf); janus_audiobridge_rtp_relay_packet *pkt = (janus_audiobridge_rtp_relay_packet *)first->data; @@ -7962,7 +7699,7 @@ static void *janus_audiobridge_mixer_thread(void *data) { while(ps) { janus_audiobridge_participant *p = (janus_audiobridge_participant *)ps->data; janus_mutex_lock(&p->qmutex); - if(g_atomic_int_get(&p->destroyed) || !p->session || !g_atomic_int_get(&p->session->started) || !g_atomic_int_get(&p->active) || p->muted || p->prebuffering || !p->inbuf) { + if(g_atomic_int_get(&p->destroyed) || !p->session || !g_atomic_int_get(&p->session->started) || !g_atomic_int_get(&p->active) || p->muted || !p->inbuf) { janus_mutex_unlock(&p->qmutex); ps = ps->next; continue; @@ -8213,7 +7950,7 @@ static void *janus_audiobridge_mixer_thread(void *data) { } janus_audiobridge_rtp_relay_packet *pkt = NULL; janus_mutex_lock(&p->qmutex); - if(g_atomic_int_get(&p->active) && !p->muted && !p->prebuffering && p->inbuf) { + if(g_atomic_int_get(&p->active) && !p->muted && p->inbuf) { GList *first = g_list_first(p->inbuf); pkt = (janus_audiobridge_rtp_relay_packet *)(first ? first->data : NULL); p->inbuf = g_list_delete_link(p->inbuf, first); @@ -8450,11 +8187,135 @@ static void *janus_audiobridge_participant_thread(void *data) { outpkt->silence = FALSE; uint8_t *payload = (uint8_t *)outpkt->data; + JitterBufferPacket jbp = {0}; + janus_audiobridge_buffer_packet *bpkt = NULL; + janus_audiobridge_rtp_relay_packet *pkt = NULL; janus_audiobridge_rtp_relay_packet *mixedpkt = NULL; + janus_rtp_header *rtp = NULL; + gint64 now = janus_get_monotonic_time(), before = now; + gboolean first = TRUE, use_fec = FALSE; + int ret = 0; - /* Start working: check the outgoing queue for packets, then encode and send them */ + /* Start working: check both the incoming queue (to decode and queue) and the outgoing one (to encode and send) */ while(!g_atomic_int_get(&stopping) && g_atomic_int_get(&session->destroyed) == 0) { - mixedpkt = g_async_queue_timeout_pop(participant->outbuf, 100000); + /* Start with packets to decode and queue for the mixer */ + now = janus_get_monotonic_time(); + janus_mutex_lock(&participant->qmutex); + gboolean locked = TRUE; + /* Start by reading packets to decode from the jitter buffer on a clock */ + if(now - before >= 18000) { + before += 20000; + if(participant->jitter) { + ret = jitter_buffer_get(participant->jitter, &jbp, participant->codec == JANUS_AUDIOCODEC_OPUS ? 960 : 160, NULL); + jitter_buffer_tick(participant->jitter); + if(ret == JITTER_BUFFER_OK) { + bpkt = (janus_audiobridge_buffer_packet *)jbp.data; + janus_mutex_unlock(&participant->qmutex); + first = FALSE; + locked = FALSE; + rtp = (janus_rtp_header *)bpkt->buffer; + /* If this is Opus, check if there's a packet gap we should fix with FEC */ + use_fec = FALSE; + if(!first && participant->codec == JANUS_AUDIOCODEC_OPUS && participant->fec) { + if(ntohs(rtp->seq_number) != participant->expected_seq) { + /* Lost a packet here? Use FEC to recover */ + use_fec = TRUE; + } + } + if(!g_atomic_int_compare_and_exchange(&participant->decoding, 0, 1)) { + /* This means we're cleaning up, so don't try to decode */ + janus_audiobridge_buffer_packet_destroy(bpkt); + break; + } + /* Access the payload */ + int plen = 0; + const unsigned char *payload = (const unsigned char *)janus_rtp_payload(bpkt->buffer, bpkt->len, &plen); + if(!payload) { + g_atomic_int_set(&participant->decoding, 0); + JANUS_LOG(LOG_ERR, "[%s] Ops! got an error accessing the RTP payload\n", + participant->codec == JANUS_AUDIOCODEC_OPUS ? "Opus" : "G.711"); + janus_audiobridge_buffer_packet_destroy(bpkt); + break; + } + if(use_fec) { + /* There was a gap, try to get decode from redundant info first */ + pkt = g_malloc(sizeof(janus_audiobridge_rtp_relay_packet)); + pkt->data = g_malloc0(BUFFER_SAMPLES*sizeof(opus_int16)); + pkt->ssrc = 0; + pkt->timestamp = participant->last_timestamp + 960; /* FIXME */ + pkt->seq_number = participant->expected_seq; /* FIXME */ + /* This is a redundant packet, so we can't parse any extension info */ + pkt->silence = FALSE; + /* Decode the lost packet using fec=1 */ + int32_t output_samples; + opus_decoder_ctl(participant->decoder, OPUS_GET_LAST_PACKET_DURATION(&output_samples)); + pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)pkt->data, output_samples, 1); + /* Queue the decoded redundant packet for the mixer */ + janus_mutex_lock(&participant->qmutex); + participant->inbuf = g_list_append(participant->inbuf, pkt); + janus_mutex_unlock(&participant->qmutex); + /* Now we can process the next packet */ + } + /* Decode the packet */ + pkt = g_malloc(sizeof(janus_audiobridge_rtp_relay_packet)); + pkt->data = g_malloc0(BUFFER_SAMPLES*sizeof(opus_int16)); + pkt->ssrc = 0; + pkt->timestamp = ntohl(rtp->timestamp); + pkt->seq_number = ntohs(rtp->seq_number); + /* We might check the audio level extension to see if this is silence */ + pkt->silence = bpkt->silence; + pkt->length = 0; + if(participant->codec == JANUS_AUDIOCODEC_OPUS) { + /* Opus */ + pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)pkt->data, BUFFER_SAMPLES, 0); + } else if(participant->codec == JANUS_AUDIOCODEC_PCMA || participant->codec == JANUS_AUDIOCODEC_PCMU) { + /* G.711 */ + if(plen != 160) { + g_atomic_int_set(&participant->decoding, 0); + JANUS_LOG(LOG_WARN, "[G.711] Wrong packet size (expected 160, got %d), skipping audio packet\n", plen); + janus_audiobridge_buffer_packet_destroy(bpkt); + break; + } + int i = 0; + uint16_t *samples = (uint16_t *)pkt->data; + if(rtp->type == 0) { + /* mu-law */ + for(i=0; itype == 8) { + /* a-law */ + for(i=0; ilength = 320; + } + /* Get rid of the buffered packet */ + janus_audiobridge_buffer_packet_destroy(bpkt); + /* Update the details */ + participant->last_timestamp = pkt->timestamp; + participant->expected_seq = pkt->seq_number + 1; + g_atomic_int_set(&participant->decoding, 0); + if(pkt->length < 0) { + if(participant->codec == JANUS_AUDIOCODEC_OPUS) { + JANUS_LOG(LOG_ERR, "[Opus] Ops! got an error decoding the Opus frame: %d (%s)\n", pkt->length, opus_strerror(pkt->length)); + } else { + JANUS_LOG(LOG_ERR, "[G.711] Ops! got an error decoding the audio frame\n"); + } + g_free(pkt->data); + g_free(pkt); + break; + } + /* Queue the decoded packet for the mixer */ + janus_mutex_lock(&participant->qmutex); + locked = TRUE; + participant->inbuf = g_list_append(participant->inbuf, pkt); + } + } + } + if(locked) + janus_mutex_unlock(&participant->qmutex); + /* Now check if there's packets to encode */ + mixedpkt = g_async_queue_try_pop(participant->outbuf); if(mixedpkt != NULL && g_atomic_int_get(&session->destroyed) == 0 && g_atomic_int_get(&session->started)) { if(g_atomic_int_get(&participant->active) && (participant->codec == JANUS_AUDIOCODEC_PCMA || participant->codec == JANUS_AUDIOCODEC_PCMU) && g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) { @@ -8515,6 +8376,7 @@ static void *janus_audiobridge_participant_thread(void *data) { g_free(mixedpkt->data); g_free(mixedpkt); } + g_usleep(2500); } /* We're done, get rid of the resources */ g_free(outpkt->data); From bd6e1fe34fb2ee23555ab7651d06d595b6ea1d8e Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 4 Aug 2023 19:39:01 +0900 Subject: [PATCH 04/34] Extend sessions_mutex scope to avoid rogue inserts in mp viewers list (see #3246) (#3250) --- src/plugins/janus_streaming.c | 55 ++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index 72d73193d6..f44fc459ba 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -5733,11 +5733,11 @@ static void *janus_streaming_handler(void *data) { janus_streaming_message_free(msg); continue; } - janus_mutex_unlock(&sessions_mutex); /* Handle request */ error_code = 0; root = NULL; if(msg->message == NULL) { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "No message??\n"); error_code = JANUS_STREAMING_ERROR_NO_MESSAGE; g_snprintf(error_cause, 512, "%s", "No message??"); @@ -5748,8 +5748,10 @@ static void *janus_streaming_handler(void *data) { JANUS_VALIDATE_JSON_OBJECT(root, request_parameters, error_code, error_cause, TRUE, JANUS_STREAMING_ERROR_MISSING_ELEMENT, JANUS_STREAMING_ERROR_INVALID_ELEMENT); - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } json_t *request = json_object_get(root, "request"); const char *request_text = json_string_value(request); json_t *result = NULL; @@ -5763,8 +5765,10 @@ static void *janus_streaming_handler(void *data) { JANUS_VALIDATE_JSON_OBJECT(root, watch_parameters, error_code, error_cause, TRUE, JANUS_STREAMING_ERROR_MISSING_ELEMENT, JANUS_STREAMING_ERROR_INVALID_ELEMENT); - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } if(!string_ids) { JANUS_VALIDATE_JSON_OBJECT(root, id_parameters, error_code, error_cause, TRUE, @@ -5774,8 +5778,10 @@ static void *janus_streaming_handler(void *data) { error_code, error_cause, TRUE, JANUS_STREAMING_ERROR_MISSING_ELEMENT, JANUS_STREAMING_ERROR_INVALID_ELEMENT); } - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } json_t *id = json_object_get(root, "id"); guint64 id_value = 0; char id_num[30], *id_value_str = NULL; @@ -5794,6 +5800,7 @@ static void *janus_streaming_handler(void *data) { for(i=0; iref); janus_mutex_unlock(&mountpoints_mutex); + janus_mutex_unlock(&sessions_mutex); goto error; } janus_mutex_lock(&mp->mutex); @@ -5839,6 +5848,7 @@ static void *janus_streaming_handler(void *data) { /* Already triggered a renegotiation, and still waiting for an answer */ janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Already renegotiating mountpoint %s\n", session->mountpoint->id_str); error_code = JANUS_STREAMING_ERROR_INVALID_STATE; g_snprintf(error_cause, 512, "Already renegotiating mountpoint %s", session->mountpoint->id_str); @@ -5858,6 +5868,7 @@ static void *janus_streaming_handler(void *data) { g_snprintf(error_cause, 512, "Already watching mountpoint %s", session->mountpoint->id_str); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); goto error; } else { @@ -5870,6 +5881,7 @@ static void *janus_streaming_handler(void *data) { janus_refcount_decrease(&mp->ref); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); goto error; } if(!g_atomic_int_compare_and_exchange(&session->renegotiating, 0, 1)) { @@ -5880,6 +5892,7 @@ static void *janus_streaming_handler(void *data) { janus_refcount_decrease(&mp->ref); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); goto error; } /* Simple renegotiation, remove the extra uneeded reference */ @@ -5894,6 +5907,7 @@ static void *janus_streaming_handler(void *data) { if(g_list_find(mp->viewers, session) != NULL) { janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); JANUS_LOG(LOG_ERR, "Already watching a stream (found %p in %s's viewers)...\n", session, id_value_str); error_code = JANUS_STREAMING_ERROR_UNKNOWN_ERROR; @@ -5926,6 +5940,7 @@ static void *janus_streaming_handler(void *data) { session->mountpoint = NULL; janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); JANUS_LOG(LOG_ERR, "Can't offer an SDP with no audio, video or data for this mountpoint\n"); error_code = JANUS_STREAMING_ERROR_INVALID_REQUEST; @@ -5959,6 +5974,7 @@ static void *janus_streaming_handler(void *data) { janus_mutex_unlock(&session->mutex); janus_refcount_decrease(&session->ref); /* This is for the failed thread */ janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); /* This is for the failed thread */ janus_refcount_decrease(&mp->ref); JANUS_LOG(LOG_ERR, "Got error %d (%s) trying to launch the on-demand thread...\n", @@ -6018,7 +6034,8 @@ static void *janus_streaming_handler(void *data) { session->mountpoint = NULL; janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); - janus_refcount_decrease(&mp->ref); + janus_mutex_unlock(&sessions_mutex); + janus_refcount_decrease(&mp->ref); goto error; } /* In case this mountpoint is simulcasting, let's aim high by default */ @@ -6057,6 +6074,7 @@ static void *janus_streaming_handler(void *data) { session->mountpoint = NULL; janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); goto error; } @@ -6202,10 +6220,12 @@ static void *janus_streaming_handler(void *data) { } janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); } else if(!strcasecmp(request_text, "watch") && jsep_sdp != NULL) { /* New subscriber provided an offer, plugin will answer */ if(sdp_type == NULL || strcasecmp(sdp_type, "offer")) { /* This isn't an offer, respond with an error */ + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "User provided SDP for a watch request must be an offer\n"); error_code = JANUS_STREAMING_ERROR_INVALID_SDP; g_snprintf(error_cause, 512, "User provided SDP for a watch request must be an offer"); @@ -6214,6 +6234,7 @@ static void *janus_streaming_handler(void *data) { char error_str[512]; janus_sdp *parsed_sdp = janus_sdp_parse(jsep_sdp, error_str, sizeof(error_str)); if(parsed_sdp == NULL) { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Error parsing SDP: %s\n", error_str); error_code = JANUS_STREAMING_ERROR_INVALID_SDP; g_snprintf(error_cause, 512, "Error parsing SDP: %s", error_str); @@ -6226,6 +6247,7 @@ static void *janus_streaming_handler(void *data) { error_code, error_cause, TRUE, JANUS_STREAMING_ERROR_MISSING_ELEMENT, JANUS_STREAMING_ERROR_INVALID_ELEMENT); if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); janus_sdp_destroy(parsed_sdp); goto error; } @@ -6239,6 +6261,7 @@ static void *janus_streaming_handler(void *data) { JANUS_STREAMING_ERROR_MISSING_ELEMENT, JANUS_STREAMING_ERROR_INVALID_ELEMENT); } if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); janus_sdp_destroy(parsed_sdp); goto error; } @@ -6258,6 +6281,7 @@ static void *janus_streaming_handler(void *data) { string_ids ? (gpointer)id_value_str : (gpointer)&id_value); if(mp == NULL) { janus_mutex_unlock(&mountpoints_mutex); + janus_mutex_unlock(&sessions_mutex); janus_sdp_destroy(parsed_sdp); JANUS_LOG(LOG_VERB, "No such mountpoint/stream %s\n", id_value_str); error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; @@ -6271,6 +6295,7 @@ static void *janus_streaming_handler(void *data) { if(error_code != 0) { janus_refcount_decrease(&mp->ref); janus_mutex_unlock(&mountpoints_mutex); + janus_mutex_unlock(&sessions_mutex); janus_sdp_destroy(parsed_sdp); goto error; } @@ -6284,6 +6309,7 @@ static void *janus_streaming_handler(void *data) { g_snprintf(error_cause, 512, "Already watching mountpoint %s", session->mountpoint->id_str); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); janus_sdp_destroy(parsed_sdp); goto error; @@ -6291,6 +6317,7 @@ static void *janus_streaming_handler(void *data) { if(g_list_find(mp->viewers, session) != NULL) { janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&mp->ref); JANUS_LOG(LOG_ERR, "Already watching a stream (found %p in %s's viewers)...\n", session, id_value_str); error_code = JANUS_STREAMING_ERROR_UNKNOWN_ERROR; @@ -6495,7 +6522,9 @@ static void *janus_streaming_handler(void *data) { janus_refcount_increase(&session->ref); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); } else if(!strcasecmp(request_text, "start")) { + janus_mutex_unlock(&sessions_mutex); if(session->mountpoint == NULL) { JANUS_LOG(LOG_VERB, "Can't start: no mountpoint set\n"); error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; @@ -6529,6 +6558,7 @@ static void *janus_streaming_handler(void *data) { gateway->notify_event(&janus_streaming_plugin, session->handle, info); } } else if(!strcasecmp(request_text, "pause")) { + janus_mutex_unlock(&sessions_mutex); if(session->mountpoint == NULL) { JANUS_LOG(LOG_VERB, "Can't pause: no mountpoint set\n"); error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; @@ -6549,6 +6579,7 @@ static void *janus_streaming_handler(void *data) { gateway->notify_event(&janus_streaming_plugin, session->handle, info); } } else if(!strcasecmp(request_text, "configure")) { + janus_mutex_unlock(&sessions_mutex); janus_streaming_mountpoint *mp = session->mountpoint; if(mp == NULL) { JANUS_LOG(LOG_VERB, "Can't configure: not on a mountpoint\n"); @@ -6563,7 +6594,7 @@ static void *janus_streaming_handler(void *data) { json_t *audio = json_object_get(root, "audio"); json_t *video = json_object_get(root, "video"); json_t *data = json_object_get(root, "data"); - + /* We use an array of streams to state the changes we want to make, * were for each stream we specify the 'mid' to impact (e.g., send) */ json_t *streams = json_object_get(root, "streams"); @@ -6604,7 +6635,7 @@ static void *janus_streaming_handler(void *data) { json_array_append_new(streams, stream); json_object_set_new(root, "streams", streams); } - + size_t i = 0; size_t streams_size = json_array_size(streams); for(i=0; istreaming_source == janus_streaming_source_rtp) { /* Enforce the requested changes */ for(i=0; imountpoint; if(oldmp == NULL) { janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_VERB, "Can't switch: not on a mountpoint\n"); error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; g_snprintf(error_cause, 512, "Can't switch: not on a mountpoint"); @@ -6819,6 +6851,7 @@ static void *janus_streaming_handler(void *data) { if(oldmp->streaming_type != janus_streaming_type_live || oldmp->streaming_source != janus_streaming_source_rtp) { janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_VERB, "Can't switch: not on a live RTP mountpoint\n"); error_code = JANUS_STREAMING_ERROR_CANT_SWITCH; g_snprintf(error_cause, 512, "Can't switch: not on a live RTP mountpoint"); @@ -6836,6 +6869,7 @@ static void *janus_streaming_handler(void *data) { } if(error_code != 0) { janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&sessions_mutex); janus_refcount_decrease(&oldmp->ref); goto error; } @@ -6855,6 +6889,7 @@ static void *janus_streaming_handler(void *data) { if(mp == NULL || g_atomic_int_get(&mp->destroyed)) { janus_mutex_unlock(&mountpoints_mutex); janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_VERB, "No such mountpoint/stream %s\n", id_value_str); error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; g_snprintf(error_cause, 512, "No such mountpoint/stream %s", id_value_str); @@ -6867,6 +6902,7 @@ static void *janus_streaming_handler(void *data) { janus_refcount_decrease(&mp->ref); janus_mutex_unlock(&mountpoints_mutex); janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_VERB, "Can't switch: target is not a live RTP mountpoint\n"); error_code = JANUS_STREAMING_ERROR_CANT_SWITCH; g_snprintf(error_cause, 512, "Can't switch: target is not a live RTP mountpoint"); @@ -6937,6 +6973,7 @@ static void *janus_streaming_handler(void *data) { g_atomic_int_set(&session->paused, 0); janus_mutex_unlock(&session->mutex); janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); /* Done with the request, remove the references we took for that */ janus_refcount_decrease(&oldmp->ref); janus_refcount_decrease(&mp->ref); @@ -6951,6 +6988,7 @@ static void *janus_streaming_handler(void *data) { gateway->notify_event(&janus_streaming_plugin, session->handle, info); } } else if(!strcasecmp(request_text, "stop")) { + janus_mutex_unlock(&sessions_mutex); if(g_atomic_int_get(&session->stopping) || !g_atomic_int_get(&session->started)) { /* Been there, done that: ignore */ janus_streaming_message_free(msg); @@ -6971,6 +7009,7 @@ static void *janus_streaming_handler(void *data) { /* Tell the core to tear down the PeerConnection, hangup_media will do the rest */ gateway->close_pc(session->handle); } else { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_VERB, "Unknown request '%s'\n", request_text); error_code = JANUS_STREAMING_ERROR_INVALID_REQUEST; g_snprintf(error_cause, 512, "Unknown request '%s'", request_text); From bcd503188897bf4eb3a28f7d15afad073e9bb4bd Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 9 Aug 2023 16:40:46 +0900 Subject: [PATCH 05/34] Fix rare race condition in VideoRoom (see #3242) (#3247) --- src/plugins/janus_videoroom.c | 49 ++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index a7a7c2b550..c095fd3b15 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -2239,6 +2239,13 @@ typedef struct janus_videoroom_subscriber_stream { janus_refcount ref; } janus_videoroom_subscriber_stream; +typedef struct janus_videoroom_stream_mapping { + janus_videoroom_publisher_stream *ps; + janus_videoroom_subscriber *subscriber; + janus_videoroom_subscriber_stream *ss; + gboolean unref_ss; +} janus_videoroom_stream_mapping; + typedef struct janus_videoroom_rtp_relay_packet { janus_videoroom_publisher_stream *source; janus_rtp_header *data; @@ -8469,7 +8476,7 @@ static void janus_videoroom_hangup_media_internal(gpointer session_data) { participant->e2ee = FALSE; /* Get rid of streams */ janus_mutex_lock(&participant->streams_mutex); - GList *subscribers = NULL; + GList *subscribers = NULL, *mappings = NULL; GList *temp = participant->streams; while(temp) { janus_videoroom_publisher_stream *ps = (janus_videoroom_publisher_stream *)temp->data; @@ -8486,8 +8493,16 @@ static void janus_videoroom_hangup_media_internal(gpointer session_data) { janus_refcount_increase(&ss->subscriber->session->ref); subscribers = g_list_append(subscribers, ss->subscriber); } - /* Remove the subscription (turns the m-line to inactive) */ - janus_videoroom_subscriber_stream_remove(ss, ps, FALSE); + /* Take note of the subscription to remove */ + janus_videoroom_stream_mapping *m = g_malloc(sizeof(janus_videoroom_stream_mapping)); + janus_refcount_increase(&ps->ref); + janus_refcount_increase(&ss->ref); + janus_refcount_increase(&ss->subscriber->ref); + m->ps = ps; + m->ss = ss; + m->unref_ss = (g_slist_find(ps->subscribers, ss) != NULL); + m->subscriber = ss->subscriber; + mappings = g_list_append(mappings, m); } } g_slist_free(ps->subscribers); @@ -8498,6 +8513,28 @@ static void janus_videoroom_hangup_media_internal(gpointer session_data) { janus_mutex_unlock(&ps->subscribers_mutex); temp = temp->next; } + if(mappings) { + temp = mappings; + while(temp) { + janus_videoroom_stream_mapping *m = (janus_videoroom_stream_mapping *)temp->data; + /* Remove the subscription (turns the m-line to inactive) */ + janus_videoroom_publisher_stream *ps = m->ps; + janus_videoroom_subscriber *subscriber = m->subscriber; + janus_videoroom_subscriber_stream *ss = m->ss; + if(subscriber) { + janus_mutex_lock(&subscriber->streams_mutex); + janus_videoroom_subscriber_stream_remove(ss, ps, TRUE); + janus_mutex_unlock(&subscriber->streams_mutex); + if(m->unref_ss) + janus_refcount_decrease(&ss->ref); + janus_refcount_decrease(&subscriber->ref); + } + janus_refcount_decrease(&ss->ref); + janus_refcount_decrease(&ps->ref); + temp = temp->next; + } + g_list_free_full(mappings, (GDestroyNotify)g_free); + } /* Any subscriber session to update? */ janus_videoroom *room = participant->room; if(subscribers != NULL) { @@ -8595,8 +8632,8 @@ static void janus_videoroom_hangup_media_internal(gpointer session_data) { } list = list->next; } - janus_videoroom_subscriber_stream_remove(s, NULL, TRUE); temp = temp->next; + janus_videoroom_subscriber_stream_remove(s, NULL, TRUE); } /* Free streams */ g_list_free(subscriber->streams); @@ -9649,8 +9686,8 @@ static void *janus_videoroom_handler(void *data) { GList *temp = subscriber->streams; while(temp) { janus_videoroom_subscriber_stream *s = (janus_videoroom_subscriber_stream *)temp->data; - janus_videoroom_subscriber_stream_remove(s, NULL, TRUE); temp = temp->next; + janus_videoroom_subscriber_stream_remove(s, NULL, TRUE); } g_list_free(subscriber->streams); subscriber->streams = NULL; @@ -10432,10 +10469,10 @@ static void *janus_videoroom_handler(void *data) { list = list->next; continue; } - janus_videoroom_subscriber_stream_remove(stream, ps, TRUE); if(stream->type != JANUS_VIDEOROOM_MEDIA_DATA) changes++; list = list->next; + janus_videoroom_subscriber_stream_remove(stream, ps, TRUE); } temp = temp->next; } From 70c23832b2f344237444714ed613de4cacdccc6d Mon Sep 17 00:00:00 2001 From: Tristan Matthews Date: Wed, 9 Aug 2023 03:43:44 -0400 Subject: [PATCH 06/34] demos: update to webrtc-adapter 8.2.3 (#3256) --- bower.json | 2 +- html/audiobridgetest.html | 2 +- html/canvas.html | 2 +- html/devicetest.html | 2 +- html/e2etest.html | 2 +- html/echotest.html | 2 +- html/multiopus.html | 2 +- html/mvideoroomtest.html | 2 +- html/nosiptest.html | 2 +- html/recordplaytest.html | 2 +- html/screensharingtest.html | 2 +- html/siptest.html | 2 +- html/streamingtest.html | 2 +- html/textroomtest.html | 2 +- html/videocalltest.html | 2 +- html/videoroomtest.html | 2 +- html/virtualbg.html | 2 +- html/webaudio.html | 2 +- npm/package-lock.json | 14 +++++++------- npm/package.json | 2 +- 20 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bower.json b/bower.json index e2073945d2..4365ee417a 100644 --- a/bower.json +++ b/bower.json @@ -33,6 +33,6 @@ "tests" ], "dependencies": { - "webrtc-adapter": "8.2.2" + "webrtc-adapter": "8.2.3" } } diff --git a/html/audiobridgetest.html b/html/audiobridgetest.html index 0ed5b04e7a..90abc151e3 100644 --- a/html/audiobridgetest.html +++ b/html/audiobridgetest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Audio Bridge Demo - + diff --git a/html/canvas.html b/html/canvas.html index 6e68feae53..914a4bb731 100644 --- a/html/canvas.html +++ b/html/canvas.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Canvas Capture - + diff --git a/html/devicetest.html b/html/devicetest.html index 4924b365d6..07da6bad54 100644 --- a/html/devicetest.html +++ b/html/devicetest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Device Selection Test - + diff --git a/html/e2etest.html b/html/e2etest.html index 7db8b2a3a4..1c493a3681 100644 --- a/html/e2etest.html +++ b/html/e2etest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): End-to-end Encryption Test - + diff --git a/html/echotest.html b/html/echotest.html index 75f22335fc..6c2fc5618b 100644 --- a/html/echotest.html +++ b/html/echotest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Echo Test - + diff --git a/html/multiopus.html b/html/multiopus.html index 0f2906c254..569de9147e 100644 --- a/html/multiopus.html +++ b/html/multiopus.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Multichannel Opus (surround) - + diff --git a/html/mvideoroomtest.html b/html/mvideoroomtest.html index 414b8bcfe9..2f4bfd1ed2 100644 --- a/html/mvideoroomtest.html +++ b/html/mvideoroomtest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Video Room Demo (multistream) - + diff --git a/html/nosiptest.html b/html/nosiptest.html index 3af2bfa9b6..6881ad4f88 100644 --- a/html/nosiptest.html +++ b/html/nosiptest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): NoSIP (SDP/RTP) - + diff --git a/html/recordplaytest.html b/html/recordplaytest.html index 2fe643667d..d1eb6a7204 100644 --- a/html/recordplaytest.html +++ b/html/recordplaytest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Recorder/Playout Demo - + diff --git a/html/screensharingtest.html b/html/screensharingtest.html index 090afa0a7c..f3f25c75cd 100644 --- a/html/screensharingtest.html +++ b/html/screensharingtest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Screen Sharing Demo - + diff --git a/html/siptest.html b/html/siptest.html index 0eb2bf6296..acbb32e03e 100644 --- a/html/siptest.html +++ b/html/siptest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): SIP Gateway Demo - + diff --git a/html/streamingtest.html b/html/streamingtest.html index 33e55dd486..9c1460503d 100644 --- a/html/streamingtest.html +++ b/html/streamingtest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Streaming Demo - + diff --git a/html/textroomtest.html b/html/textroomtest.html index ceafd2ba73..d692a5aa3f 100644 --- a/html/textroomtest.html +++ b/html/textroomtest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Text Room - + diff --git a/html/videocalltest.html b/html/videocalltest.html index ee94cf0ff6..4c0c8190df 100644 --- a/html/videocalltest.html +++ b/html/videocalltest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Video Call Demo - + diff --git a/html/videoroomtest.html b/html/videoroomtest.html index fb480ea496..60cd27864a 100644 --- a/html/videoroomtest.html +++ b/html/videoroomtest.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Video Room Demo - + diff --git a/html/virtualbg.html b/html/virtualbg.html index c5da7add70..e470c352ee 100644 --- a/html/virtualbg.html +++ b/html/virtualbg.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Virtual Background - + diff --git a/html/webaudio.html b/html/webaudio.html index 961c462091..937ba7326c 100644 --- a/html/webaudio.html +++ b/html/webaudio.html @@ -5,7 +5,7 @@ Janus WebRTC Server (multistream): Web Audio Processing - + diff --git a/npm/package-lock.json b/npm/package-lock.json index ea0122faf5..5e416e9757 100644 --- a/npm/package-lock.json +++ b/npm/package-lock.json @@ -9,7 +9,7 @@ "version": "1.2.0", "license": "MIT", "dependencies": { - "webrtc-adapter": "8.2.2" + "webrtc-adapter": "8.2.3" }, "devDependencies": { "@rollup/plugin-replace": "^5.0.2", @@ -1324,9 +1324,9 @@ } }, "node_modules/webrtc-adapter": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-8.2.2.tgz", - "integrity": "sha512-jQWwqiAEAFZamWliJo0Q+dIC6ZMJ8BgCFvW/oXWVFby1Nw14dOUfPwZ3lVe4nafDXdTyCUT7xfLt5xXiioXUCQ==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-8.2.3.tgz", + "integrity": "sha512-gnmRz++suzmvxtp3ehQts6s2JtAGPuDPjA1F3a9ckNpG1kYdYuHWYpazoAnL9FS5/B21tKlhkorbdCXat0+4xQ==", "dependencies": { "sdp": "^3.2.0" }, @@ -2297,9 +2297,9 @@ } }, "webrtc-adapter": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-8.2.2.tgz", - "integrity": "sha512-jQWwqiAEAFZamWliJo0Q+dIC6ZMJ8BgCFvW/oXWVFby1Nw14dOUfPwZ3lVe4nafDXdTyCUT7xfLt5xXiioXUCQ==", + "version": "8.2.3", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-8.2.3.tgz", + "integrity": "sha512-gnmRz++suzmvxtp3ehQts6s2JtAGPuDPjA1F3a9ckNpG1kYdYuHWYpazoAnL9FS5/B21tKlhkorbdCXat0+4xQ==", "requires": { "sdp": "^3.2.0" } diff --git a/npm/package.json b/npm/package.json index 3528f936bd..d375838ec4 100644 --- a/npm/package.json +++ b/npm/package.json @@ -26,7 +26,7 @@ }, "homepage": "https://janus.conf.meetecho.com/docs/JS", "dependencies": { - "webrtc-adapter": "8.2.2" + "webrtc-adapter": "8.2.3" }, "devDependencies": { "@rollup/plugin-replace": "^5.0.2", From fd153e33b41444100b080fbc5a165cae54c87b10 Mon Sep 17 00:00:00 2001 From: veeting <95159055+veeting@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:46:54 +0200 Subject: [PATCH 07/34] Update janus.d.ts (#3255) janus.d.ts: add missing RemoteTrackMetadata parameter and definition Co-authored-by: Fabian Bernhard --- npm/janus.d.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/npm/janus.d.ts b/npm/janus.d.ts index 6fefbd3424..b7f77419d7 100644 --- a/npm/janus.d.ts +++ b/npm/janus.d.ts @@ -121,6 +121,10 @@ declare namespace JanusJS { error?: (error: string) => void; } + interface RemoteTrackMetadata { + reason: "created" | "ended" | "mute" | "unmute"; + } + enum MessageType { Recording = 'recording', Starting = 'starting', @@ -152,7 +156,7 @@ declare namespace JanusJS { slowLink?: (uplink: boolean, lost: number, mid: string) => void; onmessage?: (message: Message, jsep?: JSEP) => void; onlocaltrack?: (track: MediaStreamTrack, on: boolean) => void; - onremotetrack?: (track: MediaStreamTrack, mid: string, on: boolean) => void; + onremotetrack?: (track: MediaStreamTrack, mid: string, on: boolean, metadata?: RemoteTrackMetadata) => void; ondataopen?: Function; ondata?: Function; oncleanup?: Function; From 71d2d18acfec83b35e589c786c7239c092dd5e08 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 9 Aug 2023 10:02:12 +0200 Subject: [PATCH 08/34] Updated Changelog (1.2.0) --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a948e19615..438c9d5e02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to this project will be documented in this file. +## [v1.2.0] - 2023-08-09 + +- Added support for VP9/AV1 simulcast, and fixed broken AV1/SVC support [[PR-3218](https://github.com/meetecho/janus-gateway/pull/3218)] +- Fixed RTCP out quality stats [[PR-3228](https://github.com/meetecho/janus-gateway/pull/3228)] +- Default link quality stats to 100 +- Added support for ICE consent freshness [[PR-3234](https://github.com/meetecho/janus-gateway/pull/3234)] +- Fixed rare race condition in VideoRoom [[PR-3219](https://github.com/meetecho/janus-gateway/pull/3219)] [[PR-3247](https://github.com/meetecho/janus-gateway/pull/3247)] +- Use speexdsp's jitter buffer in the AudioBridge [[PR-3233](https://github.com/meetecho/janus-gateway/pull/3233)] +- Fixed crash in Streaming plugin on mountpoints with too many streams [[Issue-3225](https://github.com/meetecho/janus-gateway/pull/3225)] +- Support for batched configure requests in Streaming plugin (thanks @petarminchev!) [[PR-3239](https://github.com/meetecho/janus-gateway/pull/3239)] +- Fixed rare deadlock in Streamin plugin [[PR-3250](https://github.com/meetecho/janus-gateway/pull/3250)] +- Fix simulated leave message for longer string ID rooms in TextRoom (thanks @adnanel!) [PR-3243](https://github.com/meetecho/janus-gateway/pull/3243)] +- Notify about count of sessions, handles and PeerConnections on a regular basis, when event handlers are enabled [[PR-3221](https://github.com/meetecho/janus-gateway/pull/3221)] +- Fixed broken Insertable Streams for recvonly streams when answering in janus.js +- Added background selector and blur support to Virtual Background demo +- Other smaller fixes and improvements (thanks to all who contributed pull requests and reported issues!) + ## [v1.1.4] - 2023-05-19 - Moved discussions from Google Group to Discourse From 45dcf7278b1f0c6fc8be079d55705f9f3a458326 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 9 Aug 2023 10:25:36 +0200 Subject: [PATCH 09/34] Bumped to version 1.2.1 --- bower.json | 2 +- configure.ac | 8 ++++---- docs/janus-doxygen.cfg | 2 +- npm/package-lock.json | 4 ++-- npm/package.json | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bower.json b/bower.json index 4365ee417a..d81357c6d1 100644 --- a/bower.json +++ b/bower.json @@ -1,6 +1,6 @@ { "name": "janus-gateway", - "version": "1.2.0", + "version": "1.2.1", "homepage": "https://github.com/meetecho/janus-gateway", "authors": [ "Lorenzo Miniero ", diff --git a/configure.ac b/configure.ac index b7b9588b16..e1e9b5428e 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([Janus WebRTC Server],[1.2.0],[https://github.com/meetecho/janus-gateway],[janus-gateway],[https://janus.conf.meetecho.com]) +AC_INIT([Janus WebRTC Server],[1.2.1],[https://github.com/meetecho/janus-gateway],[janus-gateway],[https://janus.conf.meetecho.com]) AC_LANG(C) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) @@ -70,11 +70,11 @@ cc*) -Wunused-but-set-variable" esac -JANUS_VERSION=1200 +JANUS_VERSION=1201 AC_SUBST(JANUS_VERSION) -JANUS_VERSION_STRING="1.2.0" +JANUS_VERSION_STRING="1.2.1" AC_SUBST(JANUS_VERSION_STRING) -JANUS_VERSION_SO="2:0:0" +JANUS_VERSION_SO="2:1:0" AC_SUBST(JANUS_VERSION_SO) case "$host_os" in diff --git a/docs/janus-doxygen.cfg b/docs/janus-doxygen.cfg index 7b3154c54d..2b957f85b0 100644 --- a/docs/janus-doxygen.cfg +++ b/docs/janus-doxygen.cfg @@ -38,7 +38,7 @@ PROJECT_NAME = "Janus (multistream)" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.0 +PROJECT_NUMBER = 1.2.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/npm/package-lock.json b/npm/package-lock.json index 5e416e9757..d343a1cfa4 100644 --- a/npm/package-lock.json +++ b/npm/package-lock.json @@ -1,12 +1,12 @@ { "name": "janus-gateway", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "janus-gateway", - "version": "1.2.0", + "version": "1.2.1", "license": "MIT", "dependencies": { "webrtc-adapter": "8.2.3" diff --git a/npm/package.json b/npm/package.json index d375838ec4..f75ad13d3a 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "janus-gateway", - "version": "1.2.0", + "version": "1.2.1", "description": "A javascript library for interacting with the C based Janus WebRTC Server", "main": "dist/janus.es.js", "types": "janus.d.ts", From 7c7093e1885834ac4154244bd99f8dd93ae71170 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Mon, 21 Aug 2023 11:24:05 +0200 Subject: [PATCH 10/34] Set correct type (string) for h264_profile and vp9_profile in remote_publisher_stream_parameters (fixes #3251) --- src/plugins/janus_videoroom.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index c095fd3b15..8b7f8a8bc6 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -1918,8 +1918,8 @@ static struct janus_json_parameter remote_publisher_stream_parameters[] = { {"stereo", JANUS_JSON_BOOL, 0}, {"fec", JANUS_JSON_BOOL, 0}, {"dtx", JANUS_JSON_BOOL, 0}, - {"h264_profile", JANUS_JSON_BOOL, 0}, - {"vp9_profile", JANUS_JSON_BOOL, 0}, + {"h264_profile", JSON_STRING, 0}, + {"vp9_profile", JSON_STRING, 0}, {"simulcast", JANUS_JSON_BOOL, 0}, {"svc", JANUS_JSON_BOOL, 0}, {"audiolevel_ext_id", JANUS_JSON_INTEGER, 0}, From 2db954b8fac28858f10ce5b7ac46f569bd64f58a Mon Sep 17 00:00:00 2001 From: Tristan Matthews Date: Tue, 29 Aug 2023 03:18:55 -0400 Subject: [PATCH 11/34] videoroom: check participant before dereference (#3259) Refs #3249 --- src/plugins/janus_videoroom.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 8b7f8a8bc6..e9bc2f51f3 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -11606,6 +11606,12 @@ static void *janus_videoroom_handler(void *data) { } else { /* This is a new publisher, or an updated one */ participant = janus_videoroom_session_get_publisher(session); + if(participant == NULL) { + JANUS_LOG(LOG_ERR, "Invalid participant instance\n"); + error_code = JANUS_VIDEOROOM_ERROR_UNKNOWN_ERROR; + g_snprintf(error_cause, 512, "Invalid participant instance"); + goto error; + } janus_videoroom *videoroom = participant->room; int count = 0; GHashTableIter iter; From b3710af4f157b02bbb2b90351c5ba0d5b79f5489 Mon Sep 17 00:00:00 2001 From: Veeting Rooms <95159055+veeting@users.noreply.github.com> Date: Tue, 29 Aug 2023 09:40:31 +0200 Subject: [PATCH 12/34] When label is larger than 64, label is truncated and new context will never match (#3265) --- src/sctp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sctp.h b/src/sctp.h index 8a0f08b5aa..1eaf756625 100644 --- a/src/sctp.h +++ b/src/sctp.h @@ -81,7 +81,7 @@ typedef struct janus_sctp_channel { /*! \brief SCTP channel ID */ uint32_t id; /*! \brief SCTP channel label */ - char label[64]; + char label[128]; /*! \brief SCTP protocol */ char protocol[64]; /*! \brief Value of the PR-SCTP policy (http://tools.ietf.org/html/rfc6458) */ From ca2f1706ef663664d6e39f71e59c9ecbb750a10e Mon Sep 17 00:00:00 2001 From: Veeting Rooms <95159055+veeting@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:45:05 +0200 Subject: [PATCH 13/34] =?UTF-8?q?Update=20PluginDataParam=20type=20definit?= =?UTF-8?q?ions=20to=20include=20text=20label=20and=20pro=E2=80=A6=20(#326?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- npm/janus.d.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/npm/janus.d.ts b/npm/janus.d.ts index b7f77419d7..4fbbe478e3 100644 --- a/npm/janus.d.ts +++ b/npm/janus.d.ts @@ -317,6 +317,11 @@ declare namespace JanusJS { } type PluginDataParam = { + /** @deprecated use data instead */ + text?: string; + data?: any; + label?: string; + protocol?: string; success?: (data: unknown) => void; error?: (error: string) => void; } From c0094205caf2949288e61c16a19cb93279bcf20f Mon Sep 17 00:00:00 2001 From: Veeting Rooms <95159055+veeting@users.noreply.github.com> Date: Tue, 29 Aug 2023 17:07:12 +0200 Subject: [PATCH 14/34] Update janus.js to allow a flexible usage of insertable streams and e2ee (#3267) --- html/janus.js | 9 +++++++-- npm/janus.d.ts | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/html/janus.js b/html/janus.js index 326c274955..55c105ac0a 100644 --- a/html/janus.js +++ b/html/janus.js @@ -1889,6 +1889,10 @@ function Janus(gatewayCallbacks) { } } } + if(callbacks.externalEncryption) { + insertableStreams = true; + config.externalEncryption = true; + } if(RTCRtpSender && (RTCRtpSender.prototype.createEncodedStreams || (RTCRtpSender.prototype.createEncodedAudioStreams && RTCRtpSender.prototype.createEncodedVideoStreams)) && insertableStreams) { @@ -2186,7 +2190,7 @@ function Janus(gatewayCallbacks) { return null; } // If transforms are present, notify Janus that the media is end-to-end encrypted - if(config.insertableStreams) + if(config.insertableStreams || config.externalEncryption) offer.e2ee = true; return offer; } @@ -2223,7 +2227,7 @@ function Janus(gatewayCallbacks) { return null; } // If transforms are present, notify Janus that the media is end-to-end encrypted - if(config.insertableStreams) + if(config.insertableStreams || config.externalEncryption) answer.e2ee = true; return answer; } @@ -3171,6 +3175,7 @@ function Janus(gatewayCallbacks) { config.dataChannel = {}; config.dtmfSender = null; config.insertableStreams = false; + config.externalEncryption = false; } pluginHandle.oncleanup(); } diff --git a/npm/janus.d.ts b/npm/janus.d.ts index 4fbbe478e3..086c7648dc 100644 --- a/npm/janus.d.ts +++ b/npm/janus.d.ts @@ -174,6 +174,7 @@ declare namespace JanusJS { tracks?: TrackOption[]; trickle?: boolean; iceRestart?: boolean; + externalEncryption?: boolean; success?: (jsep: JSEP) => void; error?: (error: Error) => void; customizeSdp?: (jsep: JSEP) => void; @@ -250,14 +251,16 @@ declare namespace JanusJS { timer: number; }; - sdpSent: boolean, - insertableStreams?: any, - candidates: RTCIceCandidateInit[], + sdpSent: boolean; + insertableStreams?: boolean; + externalEncryption?: boolean; + candidates: RTCIceCandidateInit[]; } type PluginCreateAnswerParam = { jsep: JSEP; tracks?: TrackOption[]; + externalEncryption?: boolean; /** @deprecated use tracks instead */ media?: { audioSend: any, videoSend: any }; From 527afb1e45a483486ee923d2618c6d891f82402e Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 30 Aug 2023 08:39:46 +0200 Subject: [PATCH 15/34] Fixed small leak when parsing static rooms --- src/plugins/janus_videoroom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index e9bc2f51f3..6876a333f2 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -3578,6 +3578,7 @@ int janus_videoroom_init(janus_callbacks *callback, const char *config_path) { } cl = cl->next; } + g_list_free(clist); /* Done: we keep the configuration file open in case we get a "create" or "destroy" with permanent=true */ } From aeee493c473ddb9ff17243635be514dece2139f6 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 5 Sep 2023 11:18:54 +0200 Subject: [PATCH 16/34] Fixed libcurl deprecation warning --- src/plugins/janus_streaming.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index f44fc459ba..2cadc73d44 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -7981,7 +7981,11 @@ static int janus_streaming_rtsp_connect_to_server(janus_streaming_mountpoint *mp curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 0L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); #if CURL_AT_LEAST_VERSION(7, 66, 0) +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "rtsp"); +#else curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_RTSP); +#endif curl_easy_setopt(curl, CURLOPT_HTTP09_ALLOWED, 1L); #endif char *curl_errbuf = g_malloc(CURL_ERROR_SIZE); From 7a76cfe1a27a50b522cd70ff10a71c769dd6c93a Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 5 Sep 2023 17:22:47 +0200 Subject: [PATCH 17/34] Added missing unref (fixes #3269) --- src/plugins/janus_videoroom.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/janus_videoroom.c b/src/plugins/janus_videoroom.c index 6876a333f2..a9840e7494 100644 --- a/src/plugins/janus_videoroom.c +++ b/src/plugins/janus_videoroom.c @@ -8203,8 +8203,10 @@ void janus_videoroom_incoming_data(janus_plugin_session *handle, janus_plugin_da janus_videoroom_incoming_data_internal(session, participant, packet); } static void janus_videoroom_incoming_data_internal(janus_videoroom_session *session, janus_videoroom_publisher *participant, janus_plugin_data *packet) { - if(packet->buffer == NULL || packet->length == 0) + if(packet->buffer == NULL || packet->length == 0) { + janus_videoroom_publisher_dereference_nodebug(participant); return; + } if(g_atomic_int_get(&participant->destroyed) || participant->kicked || !participant->streams || participant->room == NULL) { janus_videoroom_publisher_dereference_nodebug(participant); return; From f936e7701169e640ac6d81764c8e3659adac44c7 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 7 Sep 2023 14:39:55 +0200 Subject: [PATCH 18/34] Added new video to documentation --- src/mainpage.dox | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mainpage.dox b/src/mainpage.dox index 2d503c9c33..90ac423c9b 100644 --- a/src/mainpage.dox +++ b/src/mainpage.dox @@ -4069,7 +4069,8 @@ ldd janus | grep asan * - Presentation on SIP call transfers and multiple calls at OpenSIPS 2022; * - Presentation on SFU/VideoRoom cascading at IIT RTC 2022; * - Presentation on Social audio applications with Janus at FOSDEM 2023; - * - Presentation on Real-Time Text in SIP and WebRTC at Kamailio World 2023. \n\n + * - Presentation on Real-Time Text in SIP and WebRTC at Kamailio World 2023; + * - Presentation on Hybrid meetings using Janus at CommCon 2023. \n\n * Apart from these presentations, make sure you also check the slides * and presentations from JanusCon, * the Janus conference we hosted here in Napoli. From 359706e54f3e9588a469e194ef3472099753efb2 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 7 Sep 2023 14:57:48 +0200 Subject: [PATCH 19/34] Extend sessions_mutex range for audiobridge message handler --- src/plugins/janus_audiobridge.c | 43 ++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 72434659dc..ee933de4ad 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -5811,11 +5811,11 @@ static void *janus_audiobridge_handler(void *data) { janus_audiobridge_message_free(msg); continue; } - janus_mutex_unlock(&sessions_mutex); /* Handle request */ error_code = 0; root = NULL; if(msg->message == NULL) { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "No message??\n"); error_code = JANUS_AUDIOBRIDGE_ERROR_NO_MESSAGE; g_snprintf(error_cause, 512, "%s", "No message??"); @@ -5826,8 +5826,10 @@ static void *janus_audiobridge_handler(void *data) { JANUS_VALIDATE_JSON_OBJECT(root, request_parameters, error_code, error_cause, TRUE, JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } json_t *request = json_object_get(root, "request"); const char *request_text = json_string_value(request); json_t *event = NULL; @@ -5841,6 +5843,7 @@ static void *janus_audiobridge_handler(void *data) { got_offer = !strcasecmp(msg_sdp_type, "offer"); got_answer = !strcasecmp(msg_sdp_type, "answer"); if(!got_offer && !got_answer) { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Unsupported SDP type '%s'\n", msg_sdp_type); error_code = JANUS_AUDIOBRIDGE_ERROR_INVALID_SDP; g_snprintf(error_cause, 512, "Unsupported SDP type '%s'\n", msg_sdp_type); @@ -5851,6 +5854,7 @@ static void *janus_audiobridge_handler(void *data) { JANUS_LOG(LOG_VERB, "Configuring new participant\n"); janus_audiobridge_participant *participant = session->participant; if(participant != NULL && participant->room != NULL) { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Already in a room (use changeroom to join another one)\n"); error_code = JANUS_AUDIOBRIDGE_ERROR_ALREADY_JOINED; g_snprintf(error_cause, 512, "Already in a room (use changeroom to join another one)"); @@ -5859,15 +5863,19 @@ static void *janus_audiobridge_handler(void *data) { JANUS_VALIDATE_JSON_OBJECT(root, join_parameters, error_code, error_cause, TRUE, JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } json_t *rtp = json_object_get(root, "rtp"); if(rtp != NULL) { JANUS_VALIDATE_JSON_OBJECT(root, rtp_parameters, error_code, error_cause, TRUE, JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } if(msg_sdp != NULL) { JANUS_LOG(LOG_WARN, "Added plain RTP details but negotiating a WebRTC PeerConnection: plain RTP will be ignored\n"); rtp = NULL; @@ -5883,8 +5891,10 @@ static void *janus_audiobridge_handler(void *data) { error_code, error_cause, TRUE, JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); } - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } if(!string_ids) { JANUS_VALIDATE_JSON_OBJECT(root, idopt_parameters, error_code, error_cause, TRUE, @@ -5894,8 +5904,10 @@ static void *janus_audiobridge_handler(void *data) { error_code, error_cause, TRUE, JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); } - if(error_code != 0) + if(error_code != 0) { + janus_mutex_unlock(&sessions_mutex); goto error; + } json_t *room = json_object_get(root, "room"); guint64 room_id = 0; char room_id_num[30], *room_id_str = NULL; @@ -5911,6 +5923,7 @@ static void *janus_audiobridge_handler(void *data) { string_ids ? (gpointer)room_id_str : (gpointer)&room_id); if(audiobridge == NULL) { janus_mutex_unlock(&rooms_mutex); + janus_mutex_unlock(&sessions_mutex); error_code = JANUS_AUDIOBRIDGE_ERROR_NO_SUCH_ROOM; JANUS_LOG(LOG_ERR, "No such room (%s)\n", room_id_str); g_snprintf(error_cause, 512, "No such room (%s)", room_id_str); @@ -5923,6 +5936,7 @@ static void *janus_audiobridge_handler(void *data) { /* Plain RTP participants are not allowed in this room */ janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); error_code = JANUS_AUDIOBRIDGE_ERROR_UNAUTHORIZED; JANUS_LOG(LOG_ERR, "Plain RTP participants not allowed in this room\n"); g_snprintf(error_cause, 512, "Plain RTP participants not allowed in this room"); @@ -5934,6 +5948,7 @@ static void *janus_audiobridge_handler(void *data) { if(error_code != 0) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); goto error; } /* A token might be required too */ @@ -5946,6 +5961,7 @@ static void *janus_audiobridge_handler(void *data) { g_snprintf(error_cause, 512, "Unauthorized (not in the allowed list)"); janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); goto error; } } @@ -5957,6 +5973,7 @@ static void *janus_audiobridge_handler(void *data) { if(error_code != 0) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); goto error; } admin = TRUE; @@ -5970,6 +5987,7 @@ static void *janus_audiobridge_handler(void *data) { if(error_code != 0) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); goto error; } const char *group_name = json_string_value(json_object_get(root, "group")); @@ -5977,6 +5995,7 @@ static void *janus_audiobridge_handler(void *data) { if(group == 0) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "No such group (%s)\n", group_name); error_code = JANUS_AUDIOBRIDGE_ERROR_NO_SUCH_GROUP; g_snprintf(error_cause, 512, "No such group (%s)", group_name); @@ -6011,6 +6030,7 @@ static void *janus_audiobridge_handler(void *data) { if(complexity < 1 || complexity > 10) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Invalid element (quality should be a positive integer between 1 and 10)\n"); error_code = JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT; g_snprintf(error_cause, 512, "Invalid element (quality should be a positive integer between 1 and 10)"); @@ -6020,6 +6040,7 @@ static void *janus_audiobridge_handler(void *data) { if(expected_loss > 20) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Invalid element (expected_loss should be a positive integer between 0 and 20)\n"); error_code = JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT; g_snprintf(error_cause, 512, "Invalid element (expected_loss should be a positive integer between 0 and 20)"); @@ -6031,6 +6052,7 @@ static void *janus_audiobridge_handler(void *data) { if(codec != JANUS_AUDIOCODEC_OPUS && codec != JANUS_AUDIOCODEC_PCMA && codec != JANUS_AUDIOCODEC_PCMU) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Invalid element (codec must opus, pcmu or pcma)\n"); error_code = JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT; g_snprintf(error_cause, 512, "Invalid element (codec must opus, pcmu or pcma)"); @@ -6054,6 +6076,7 @@ static void *janus_audiobridge_handler(void *data) { /* User ID already taken */ janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); error_code = JANUS_AUDIOBRIDGE_ERROR_ID_EXISTS; JANUS_LOG(LOG_ERR, "User ID %s already exists\n", user_id_str); g_snprintf(error_cause, 512, "User ID %s already exists", user_id_str); @@ -6156,6 +6179,7 @@ static void *janus_audiobridge_handler(void *data) { } janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); g_free(participant->display); g_free(participant); JANUS_LOG(LOG_ERR, "Error creating Opus encoder\n"); @@ -6196,6 +6220,7 @@ static void *janus_audiobridge_handler(void *data) { } janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); + janus_mutex_unlock(&sessions_mutex); g_free(participant->display); if(participant->encoder) opus_encoder_destroy(participant->encoder); @@ -6346,6 +6371,7 @@ static void *janus_audiobridge_handler(void *data) { g_hash_table_insert(audiobridge->participants, string_ids ? (gpointer)g_strdup(participant->user_id_str) : (gpointer)janus_uint64_dup(participant->user_id), participant); + janus_mutex_unlock(&sessions_mutex); /* Notify the other participants */ json_t *newuser = json_object(); json_object_set_new(newuser, "audiobridge", json_string("joined")); @@ -6433,6 +6459,7 @@ static void *janus_audiobridge_handler(void *data) { if(generate_offer) session->plugin_offer = generate_offer; } else if(!strcasecmp(request_text, "configure")) { + janus_mutex_unlock(&sessions_mutex); /* Handle this participant */ janus_audiobridge_participant *participant = (janus_audiobridge_participant *)session->participant; if(participant == NULL || participant->room == NULL) { @@ -6648,6 +6675,7 @@ static void *janus_audiobridge_handler(void *data) { session->plugin_offer = generate_offer; } } else if(!strcasecmp(request_text, "changeroom")) { + janus_mutex_unlock(&sessions_mutex); /* The participant wants to leave the current room and join another one without reconnecting (e.g., a sidebar) */ JANUS_VALIDATE_JSON_OBJECT(root, join_parameters, error_code, error_cause, TRUE, @@ -7087,11 +7115,13 @@ static void *janus_audiobridge_handler(void *data) { janus_mutex_unlock(&audiobridge->mutex); janus_mutex_unlock(&rooms_mutex); } else if(!strcasecmp(request_text, "hangup")) { + janus_mutex_unlock(&sessions_mutex); /* Get rid of an ongoing session */ gateway->close_pc(session->handle); event = json_object(); json_object_set_new(event, "audiobridge", json_string("hangingup")); } else if(!strcasecmp(request_text, "leave")) { + janus_mutex_unlock(&sessions_mutex); /* This participant is leaving */ janus_audiobridge_participant *participant = (janus_audiobridge_participant *)session->participant; if(participant == NULL || participant->room == NULL) { @@ -7182,6 +7212,7 @@ static void *janus_audiobridge_handler(void *data) { janus_refcount_decrease(&audiobridge->ref); } } else { + janus_mutex_unlock(&sessions_mutex); JANUS_LOG(LOG_ERR, "Unknown request '%s'\n", request_text); error_code = JANUS_AUDIOBRIDGE_ERROR_INVALID_REQUEST; g_snprintf(error_cause, 512, "Unknown request '%s'", request_text); From 9f036386bfb571af34a7dd6e74a492af895c80f2 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 13 Sep 2023 10:30:09 +0200 Subject: [PATCH 20/34] Don't advertise support for UPDATE in the SIP plugin --- src/plugins/janus_sip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/janus_sip.c b/src/plugins/janus_sip.c index e8b7366538..539de979b8 100644 --- a/src/plugins/janus_sip.c +++ b/src/plugins/janus_sip.c @@ -7344,7 +7344,7 @@ gpointer janus_sip_sofia_thread(gpointer user_data) { session->stack->s_nua = nua_create(session->stack->s_root, janus_sip_sofia_callback, session, - SIPTAG_ALLOW_STR("INVITE, ACK, BYE, CANCEL, OPTIONS, UPDATE, REFER, MESSAGE, INFO, NOTIFY"), + SIPTAG_ALLOW_STR("INVITE, ACK, BYE, CANCEL, OPTIONS, REFER, MESSAGE, INFO, NOTIFY"), NUTAG_M_USERNAME(session->account.username), NUTAG_URL(sip_url), TAG_IF(session->account.sips, NUTAG_SIPS_URL(sips_url)), From 49e522c5ee72f208e01f8ef84534af92b2983abe Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Fri, 22 Sep 2023 04:53:59 -0300 Subject: [PATCH 21/34] Update janus.d.ts sendEncodings should be an array (#3271) --- npm/janus.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm/janus.d.ts b/npm/janus.d.ts index 086c7648dc..1654607070 100644 --- a/npm/janus.d.ts +++ b/npm/janus.d.ts @@ -297,7 +297,7 @@ declare namespace JanusJS { medium: number; high: number; }; - sendEncodings?: RTCRtpEncodingParameters; + sendEncodings?: RTCRtpEncodingParameters[]; framerate?: number; bitrate?: number; dontStop?: boolean; From ae23fb0f47ca3207cdd74b78d54212c08a402171 Mon Sep 17 00:00:00 2001 From: Benson Muite Date: Mon, 2 Oct 2023 17:20:33 +0300 Subject: [PATCH 22/34] Use lua term (#3273) Replace ansicolors by lua-term --- src/plugins/lua/echotest.lua | 8 ++++---- src/plugins/lua/videoroom.lua | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins/lua/echotest.lua b/src/plugins/lua/echotest.lua index 4d7ffe21f4..e3f6860038 100644 --- a/src/plugins/lua/echotest.lua +++ b/src/plugins/lua/echotest.lua @@ -7,14 +7,14 @@ json = require('json') -- We also import our own SDP helper utilities: you may have better ones sdp = require('janus-sdp') -- Let's also use our ugly stdout logger just for the fun of it: to add --- some color to the text we use the ansicolors library --- (https://github.com/kikito/ansicolors.lua) -colors = require "ansicolors" +-- some color to the text we use the lua-term library +-- (https://github.com/hoelzro/lua-term) +colors = require('term.colors') logger = require('janus-logger') -- Example details name = "echotest.lua" -logger.prefix(colors("[%{blue}" .. name .. "%{reset}]")) +logger.prefix(colors.blue .. name .. colors.reset) logger.print("Loading...") -- State and properties diff --git a/src/plugins/lua/videoroom.lua b/src/plugins/lua/videoroom.lua index ffa0dd3c81..a263180268 100644 --- a/src/plugins/lua/videoroom.lua +++ b/src/plugins/lua/videoroom.lua @@ -7,14 +7,14 @@ json = require('json') -- We also import our own SDP helper utilities: you may have better ones sdp = require('janus-sdp') -- Let's also use our ugly stdout logger just for the fun of it: to add --- some color to the text we use the ansicolors library --- (https://github.com/kikito/ansicolors.lua) -colors = require "ansicolors" +-- some color to the text we use the lua-term library +-- (https://github.com/hoelzro/lua-term) +colors = require('term.colors') logger = require('janus-logger') -- Example details name = "videoroom.lua" -logger.prefix(colors("[%{blue}" .. name .. "%{reset}]")) +logger.prefix(colors.blue .. name .. colors.reset) logger.print("Loading...") -- State and properties From 6b1e64c59b1f67d0e45aba8800ca42cab03ebf10 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 26 Oct 2023 12:39:20 +0200 Subject: [PATCH 23/34] Log a warning in janus-pp-rec when an audio clock mismatch > 0.5s has been detected --- src/postprocessing/janus-pp-rec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/postprocessing/janus-pp-rec.c b/src/postprocessing/janus-pp-rec.c index 802ef686eb..087d0140fb 100644 --- a/src/postprocessing/janus-pp-rec.c +++ b/src/postprocessing/janus-pp-rec.c @@ -1172,6 +1172,12 @@ int main(int argc, char *argv[]) { tmp = tmp->next; } JANUS_LOG(LOG_INFO, "Counted %"SCNu32" frame packets\n", count); + if(!data && !video) { + double diff = ts - pts; + if(abs(diff) > 0.5) { + JANUS_LOG(LOG_WARN, "Detected audio clock mismatch, consider using skew compensation or restamping (rtp_time=%.2fs, real_time=%.2fs, diff=%.2fs)\n", ts, pts, diff); + } + } if(rotated != -1) { if(rotated == 0 && last_rotation != 0) { JANUS_LOG(LOG_INFO, "The video is rotated\n"); From 9de4c137ea4c797b5af4d9d3634bca9032dcd069 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Fri, 27 Oct 2023 12:21:56 +0200 Subject: [PATCH 24/34] Fix postprocessing restamping (broken since commit f51524e for DTX support) --- src/postprocessing/janus-pp-rec.c | 8 ++++++-- src/postprocessing/pp-opus.c | 5 +++-- src/postprocessing/pp-opus.h | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/postprocessing/janus-pp-rec.c b/src/postprocessing/janus-pp-rec.c index 087d0140fb..1dbca56c43 100644 --- a/src/postprocessing/janus-pp-rec.c +++ b/src/postprocessing/janus-pp-rec.c @@ -1277,7 +1277,11 @@ int main(int argc, char *argv[]) { } /* Run restamping */ - if(!video && !data && options.restamp_multiplier > 0) { + gboolean restamping = FALSE; + if(options.restamp_multiplier > 0) { + restamping = TRUE; + } + if(!video && !data && restamping) { tmp = list; uint64_t restamping_offset = 0; double restamp_threshold = (double) options.restamp_min_th/1000; @@ -1439,7 +1443,7 @@ int main(int argc, char *argv[]) { /* Loop */ if(!video && !data) { if(opus) { - if(janus_pp_opus_process(file, list, &working) < 0) { + if(janus_pp_opus_process(file, list, restamping, &working) < 0) { JANUS_LOG(LOG_ERR, "Error processing Opus RTP frames...\n"); } } else if(g711) { diff --git a/src/postprocessing/pp-opus.c b/src/postprocessing/pp-opus.c index df0fb85fed..1521e4a7d4 100644 --- a/src/postprocessing/pp-opus.c +++ b/src/postprocessing/pp-opus.c @@ -101,7 +101,7 @@ int janus_pp_opus_create(char *destination, char *metadata, gboolean multiopus, // It assumes ALL the packets are of the 20ms kind #define OPUS_PACKET_DURATION 48 * 20; -int janus_pp_opus_process(FILE *file, janus_pp_frame_packet *list, int *working) { +int janus_pp_opus_process(FILE *file, janus_pp_frame_packet *list, gboolean restamping, int *working) { if(!file || !list || !working) return -1; janus_pp_frame_packet *tmp = list; @@ -251,7 +251,8 @@ int janus_pp_opus_process(FILE *file, janus_pp_frame_packet *list, int *working) AVRational timebase = {1, 48000}; while(*working && tmp != NULL) { - if(tmp->prev != NULL && ((tmp->ts - tmp->prev->ts)/48/20 > 1) && (tmp->seq != tmp->prev->seq+1)) { + /* if restamping is being used, do not evaluate the sequence number jump */ + if(tmp->prev != NULL && ((tmp->ts - tmp->prev->ts)/48/20 > 1) && (restamping || (tmp->seq != tmp->prev->seq+1))) { JANUS_LOG(LOG_WARN, "Lost a packet here? (got seq %"SCNu16" after %"SCNu16", time ~%"SCNu64"s)\n", tmp->seq, tmp->prev->seq, (tmp->ts-list->ts)/48000); /* use ts differ to insert silence packet */ diff --git a/src/postprocessing/pp-opus.h b/src/postprocessing/pp-opus.h index 3d45317f59..bdf37cc4e0 100644 --- a/src/postprocessing/pp-opus.h +++ b/src/postprocessing/pp-opus.h @@ -19,7 +19,7 @@ /* Opus stuff */ const char **janus_pp_opus_get_extensions(void); int janus_pp_opus_create(char *destination, char *metadata, gboolean multiopus, const char *extension, int opusred_pt); -int janus_pp_opus_process(FILE *file, janus_pp_frame_packet *list, int *working); +int janus_pp_opus_process(FILE *file, janus_pp_frame_packet *list, gboolean restamping, int *working); void janus_pp_opus_close(void); #endif From 25dc7f8b7d3653e0de2cb802aaee0871eba872e8 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Fri, 27 Oct 2023 18:23:37 +0200 Subject: [PATCH 25/34] Avoid using abs for floating point check in postprocessing --- src/postprocessing/janus-pp-rec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/postprocessing/janus-pp-rec.c b/src/postprocessing/janus-pp-rec.c index 1dbca56c43..be564267d4 100644 --- a/src/postprocessing/janus-pp-rec.c +++ b/src/postprocessing/janus-pp-rec.c @@ -1174,7 +1174,7 @@ int main(int argc, char *argv[]) { JANUS_LOG(LOG_INFO, "Counted %"SCNu32" frame packets\n", count); if(!data && !video) { double diff = ts - pts; - if(abs(diff) > 0.5) { + if(diff < -0.5 || diff > 0.5 ) { { JANUS_LOG(LOG_WARN, "Detected audio clock mismatch, consider using skew compensation or restamping (rtp_time=%.2fs, real_time=%.2fs, diff=%.2fs)\n", ts, pts, diff); } } From 0e37c1326703aac52145748715fb514cf233b886 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Fri, 27 Oct 2023 18:31:12 +0200 Subject: [PATCH 26/34] Fix typo in previous commit --- src/postprocessing/janus-pp-rec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/postprocessing/janus-pp-rec.c b/src/postprocessing/janus-pp-rec.c index be564267d4..45422f1478 100644 --- a/src/postprocessing/janus-pp-rec.c +++ b/src/postprocessing/janus-pp-rec.c @@ -1174,7 +1174,7 @@ int main(int argc, char *argv[]) { JANUS_LOG(LOG_INFO, "Counted %"SCNu32" frame packets\n", count); if(!data && !video) { double diff = ts - pts; - if(diff < -0.5 || diff > 0.5 ) { { + if(diff < -0.5 || diff > 0.5) { JANUS_LOG(LOG_WARN, "Detected audio clock mismatch, consider using skew compensation or restamping (rtp_time=%.2fs, real_time=%.2fs, diff=%.2fs)\n", ts, pts, diff); } } From 0ae1c6d286bd59efe41d85efd5186dbad880a6eb Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Mon, 30 Oct 2023 11:24:19 +0100 Subject: [PATCH 27/34] Reset ice failure detection time when state changes to connected or ready --- src/ice.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ice.c b/src/ice.c index 5ed7310c19..0e5b25f4cb 100644 --- a/src/ice.c +++ b/src/ice.c @@ -2045,6 +2045,7 @@ static gboolean janus_ice_check_failed(gpointer data) { if(pc->state == NICE_COMPONENT_STATE_CONNECTED || pc->state == NICE_COMPONENT_STATE_READY) { /* ICE succeeded in the meanwhile, get rid of this timer */ JANUS_LOG(LOG_VERB, "[%"SCNu64"] ICE succeeded, disabling ICE state check timer!\n", handle->handle_id); + pc->icefailed_detected = 0; goto stoptimer; } /* Still in the failed state, how much time passed since we first detected it? */ From 7b91e80861e94904d0303a1a8e966ec962237021 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Mon, 13 Nov 2023 11:56:56 +0100 Subject: [PATCH 28/34] Check mountpoint in use before performing an ICE restart on a streaming session (fixes #3288). --- src/plugins/janus_streaming.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/plugins/janus_streaming.c b/src/plugins/janus_streaming.c index 2cadc73d44..4cca8d2903 100644 --- a/src/plugins/janus_streaming.c +++ b/src/plugins/janus_streaming.c @@ -5843,6 +5843,27 @@ static void *janus_streaming_handler(void *data) { /* Check if this is a new viewer, or if an update is taking place (i.e., ICE restart) */ gboolean audio = TRUE, video = TRUE, data = TRUE; if(do_restart) { + if(session->mountpoint == NULL) { + JANUS_LOG(LOG_ERR, "Can't perform ICE restart: no mountpoint set\n"); + error_code = JANUS_STREAMING_ERROR_NO_SUCH_MOUNTPOINT; + g_snprintf(error_cause, 512, "Can't perform ICE restart: no mountpoint set"); + janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); + janus_refcount_decrease(&mp->ref); + goto error; + } + if(session->mountpoint != mp) { + /* Already watching something else */ + JANUS_LOG(LOG_ERR, "Already watching mountpoint %s\n", session->mountpoint->id_str); + error_code = JANUS_STREAMING_ERROR_INVALID_STATE; + g_snprintf(error_cause, 512, "Already watching mountpoint %s", session->mountpoint->id_str); + janus_mutex_unlock(&session->mutex); + janus_mutex_unlock(&mp->mutex); + janus_mutex_unlock(&sessions_mutex); + janus_refcount_decrease(&mp->ref); + goto error; + } /* User asked for an ICE restart: provide a new offer */ if(!g_atomic_int_compare_and_exchange(&session->renegotiating, 0, 1)) { /* Already triggered a renegotiation, and still waiting for an answer */ From 655495602dfd105e8d16ca6958aad41e50e2c23a Mon Sep 17 00:00:00 2001 From: heyfluke Date: Tue, 14 Nov 2023 19:44:56 +0800 Subject: [PATCH 29/34] Fix pp-rec post-processing for h265 by handling nalu delimiter for missing types (#3274) --- src/postprocessing/pp-h265.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/postprocessing/pp-h265.c b/src/postprocessing/pp-h265.c index f988f4152b..376637fbc8 100644 --- a/src/postprocessing/pp-h265.c +++ b/src/postprocessing/pp-h265.c @@ -513,7 +513,7 @@ int janus_pp_h265_process(FILE *file, janus_pp_frame_packet *list, int *working) memcpy(&unit, buffer, sizeof(uint16_t)); unit = ntohs(unit); uint8_t type = (unit & 0x7E00) >> 9; - if(type == 32 || type == 33 || type == 34) { + if(type == 32 || type == 33 || type == 34 || type == 1) { if(type == 32 || type == 33) { keyFrame = 1; if(!keyframe_found) { @@ -582,6 +582,8 @@ int janus_pp_h265_process(FILE *file, janus_pp_frame_packet *list, int *working) /* Skip the FU header */ buffer += 3; len -= 3; + } else { + JANUS_LOG(LOG_HUGE, "unhandled nalu type: %d\n", type); } /* Frame manipulation */ memcpy(received_frame + frameLen, buffer, len); From 77e4ce31816c66e41e74f97228ed6f432e0af2ef Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 14 Nov 2023 12:46:07 +0100 Subject: [PATCH 30/34] Remove unneeded log line --- src/postprocessing/pp-h265.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/postprocessing/pp-h265.c b/src/postprocessing/pp-h265.c index 376637fbc8..2225ce5ce7 100644 --- a/src/postprocessing/pp-h265.c +++ b/src/postprocessing/pp-h265.c @@ -582,8 +582,6 @@ int janus_pp_h265_process(FILE *file, janus_pp_frame_packet *list, int *working) /* Skip the FU header */ buffer += 3; len -= 3; - } else { - JANUS_LOG(LOG_HUGE, "unhandled nalu type: %d\n", type); } /* Frame manipulation */ memcpy(received_frame + frameLen, buffer, len); From aed755a7c3d7683bf80fc0109714c742e006de1a Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 16 Nov 2023 14:25:55 +0100 Subject: [PATCH 31/34] pcap2mjr: avoid misaligned access --- src/postprocessing/pcap2mjr.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/postprocessing/pcap2mjr.c b/src/postprocessing/pcap2mjr.c index 73ad1a9c5e..d468d28ef5 100644 --- a/src/postprocessing/pcap2mjr.c +++ b/src/postprocessing/pcap2mjr.c @@ -208,6 +208,11 @@ int main(int argc, char *argv[]) { sizeof(struct udphdr) + 12, pkt_size = 0; gboolean header_written = FALSE; gint64 start_ts = 0, pkt_ts = 0; + pcap2mjr_ethernet_header eth; + struct sll_header lcc; + struct ip v4; + struct ip6_hdr v6; + janus_pp_rtp_header rtp; while(working && (ret = pcap_next_ex(pcap, &header, &buffer)) >= 0) { count++; if(header->len != header->caplen) { @@ -232,27 +237,27 @@ int main(int argc, char *argv[]) { int protocol = 0; if(link == DLT_EN10MB) { /* Ethernet */ - pcap2mjr_ethernet_header *eth = (pcap2mjr_ethernet_header *)temp; - protocol = ntohs(eth->type); + memcpy(ð, temp, sizeof(pcap2mjr_ethernet_header)); + protocol = ntohs(eth.type); temp += sizeof(pcap2mjr_ethernet_header); pkt_size -= sizeof(pcap2mjr_ethernet_header); } else { /* Linux Cooked Capture */ - struct sll_header *lcc = (struct sll_header *)temp; - protocol = ntohs(lcc->sll_protocol); + memcpy(&lcc, temp, sizeof(struct sll_header)); + protocol = ntohs(lcc.sll_protocol); temp += sizeof(struct sll_header); pkt_size -= sizeof(struct sll_header); } if(protocol == 0x0800) { /* IPv4 */ - struct ip *v4 = (struct ip *)temp; - protocol = v4->ip_p; + memcpy(&v4, temp, sizeof(struct ip)); + protocol = v4.ip_p; temp += sizeof(struct ip); pkt_size -= sizeof(struct ip); } else if(protocol == 0x86DD) { /* IPv6 */ - struct ip6_hdr *v6 = (struct ip6_hdr *)temp; - protocol = v6->ip6_ctlun.ip6_un1.ip6_un1_nxt; + memcpy(&v6, temp, sizeof(struct ip6_hdr)); + protocol = v6.ip6_ctlun.ip6_un1.ip6_un1_nxt; temp += sizeof(struct ip6_hdr); pkt_size -= sizeof(struct ip6_hdr); } else { @@ -271,14 +276,14 @@ int main(int argc, char *argv[]) { temp += sizeof(struct udphdr); pkt_size -= sizeof(struct udphdr); /* Make sure this is an RTP packet */ - janus_pp_rtp_header *rtp = (janus_pp_rtp_header *)temp; - if(rtp->version != 2 || (rtp->type >= 64 && rtp->type < 96)) { + memcpy(&rtp, temp, sizeof(janus_pp_rtp_header)); + if(rtp.version != 2 || (rtp.type >= 64 && rtp.type < 96)) { if(show_warnings) { JANUS_LOG(LOG_WARN, "Not an RTP packet, skipping packet #%"SCNu32"\n", count); } continue; } - pssrc = htonl(rtp->ssrc); + pssrc = htonl(rtp.ssrc); if(ssrc == 0) { ssrc = pssrc; JANUS_LOG(LOG_INFO, "Autodetected SSRC %"SCNu32"\n", ssrc); From d9a5853ce42d3d19bd9c3d56afe3b0862f58cbd2 Mon Sep 17 00:00:00 2001 From: Irek <34670509+pawnnail@users.noreply.github.com> Date: Fri, 17 Nov 2023 04:12:25 +1300 Subject: [PATCH 32/34] Allow SIP plugin to bind media to a specific address (see #3248) (#3263) --- src/plugins/janus_sip.c | 142 ++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 50 deletions(-) diff --git a/src/plugins/janus_sip.c b/src/plugins/janus_sip.c index 539de979b8..31adfc7d62 100644 --- a/src/plugins/janus_sip.c +++ b/src/plugins/janus_sip.c @@ -854,6 +854,7 @@ static gboolean ipv6_disabled = FALSE; static janus_callbacks *gateway = NULL; static char *local_ip = NULL, *sdp_ip = NULL, *local_media_ip = NULL; +static janus_network_address janus_network_local_media_ip = { 0 }; static int keepalive_interval = 120; static gboolean behind_nat = FALSE; static char *user_agent; @@ -1882,11 +1883,11 @@ int janus_sip_init(janus_callbacks *callback, const char *config_path) { } item = janus_config_get(config, config_general, janus_config_type_item, "local_media_ip"); - if(item && item->value) + if(item && item->value && strlen(item->value) > 0) local_media_ip = g_strdup(item->value); item = janus_config_get(config, config_general, janus_config_type_item, "sdp_ip"); - if(item && item->value) { + if(item && item->value && strlen(item->value) > 0) { sdp_ip = g_strdup(item->value); JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); } @@ -2003,6 +2004,25 @@ int janus_sip_init(janus_callbacks *callback, const char *config_path) { } JANUS_LOG(LOG_VERB, "Local IP set to %s\n", local_ip); + /* Since we might have to derive SDP connection address from local_media_ip make sure it has a meaningful value + * for the purpose of using it in the SDP c= header */ + janus_network_address_nullify(&janus_network_local_media_ip); + if(local_media_ip) { + if(janus_network_string_to_address(janus_network_query_options_any_ip, local_media_ip, &janus_network_local_media_ip) != 0) { + JANUS_LOG(LOG_ERR, "Invalid local media IP address [%s]...\n", local_media_ip); + return -1; + } + if((janus_network_local_media_ip.family == AF_INET && janus_network_local_media_ip.ipv4.s_addr == INADDR_ANY) || + (janus_network_local_media_ip.family == AF_INET6 && IN6_IS_ADDR_UNSPECIFIED(&janus_network_local_media_ip.ipv6))) { + janus_network_address_nullify(&janus_network_local_media_ip); + } + } + JANUS_LOG(LOG_VERB, "Binding media address set to [%s]...\n", janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip); + if(!sdp_ip) { + sdp_ip = janus_network_address_is_null(&janus_network_local_media_ip) ? local_ip : local_media_ip; + JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); + } + /* Setup sofia */ su_init(); if(notify_events && callback->events_is_enabled()) { @@ -2022,19 +2042,29 @@ int janus_sip_init(janus_callbacks *callback, const char *config_path) { /* This is the callback we'll need to invoke to contact the Janus core */ gateway = callback; - /* Finally, let's check if IPv6 is disabled, as we may need to know for RTP/RTCP sockets */ - int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); - if(fd <= 0) { - ipv6_disabled = TRUE; - } else { - int v6only = 0; - if(setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) + if(janus_network_address_is_null(&janus_network_local_media_ip) || + janus_network_local_media_ip.family == AF_INET6) { + /* Finally, let's check if IPv6 is disabled, as we may need to know for RTP/RTCP sockets */ + int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); + if(fd <= 0) { ipv6_disabled = TRUE; - } - if(fd > 0) - close(fd); - if(ipv6_disabled) { - JANUS_LOG(LOG_WARN, "IPv6 disabled, will only use IPv4 for RTP/RTCP sockets (SIP)\n"); + } else { + int v6only = 0; + if(setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) + ipv6_disabled = TRUE; + } + if(fd > 0) + close(fd); + if(ipv6_disabled) { + if(!janus_network_address_is_null(&janus_network_local_media_ip)) { + JANUS_LOG(LOG_ERR, "IPv6 disabled and local media address is IPv6...\n"); + return -1; + } + JANUS_LOG(LOG_WARN, "IPv6 disabled, will only use IPv4 for RTP/RTCP sockets (SIP)\n"); + } + } else if(janus_network_local_media_ip.family == AF_INET) { + /* Disable if we have a specified IPv4 address for RTP/RTCP sockets */ + ipv6_disabled = TRUE; } g_atomic_int_set(&initialized, 1); @@ -6489,7 +6519,7 @@ char *janus_sip_sdp_manipulate(janus_sip_session *session, janus_sdp *sdp, gbool JANUS_LOG(LOG_VERB, "Setting protocol to %s\n", session->media.require_srtp ? "RTP/SAVP" : "RTP/AVP"); if(sdp->c_addr) { g_free(sdp->c_addr); - sdp->c_addr = g_strdup(sdp_ip ? sdp_ip : (local_media_ip ? local_media_ip : local_ip)); + sdp->c_addr = g_strdup(sdp_ip); } int opusred_pt = answer ? janus_sdp_get_opusred_pt(sdp, -1) : -1; GList *temp = sdp->m_lines; @@ -6523,7 +6553,7 @@ char *janus_sip_sdp_manipulate(janus_sip_session *session, janus_sdp *sdp, gbool } } g_free(m->c_addr); - m->c_addr = g_strdup(sdp_ip ? sdp_ip : (local_media_ip ? local_media_ip : local_ip)); + m->c_addr = g_strdup(sdp_ip); if(answer && (m->type == JANUS_SDP_AUDIO || m->type == JANUS_SDP_VIDEO)) { /* Check which codec was negotiated eventually */ int pt = -1; @@ -6625,18 +6655,24 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u session->media.pipefd[1] = -1; } } + gboolean use_ipv6_address_family = !ipv6_disabled && + (janus_network_address_is_null(&janus_network_local_media_ip) || janus_network_local_media_ip.family == AF_INET6); + socklen_t addrlen = use_ipv6_address_family? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in); /* Start */ int attempts = 100; /* FIXME Don't retry forever */ if(session->media.has_audio) { - JANUS_LOG(LOG_VERB, "Allocating audio ports:\n"); + JANUS_LOG(LOG_VERB, "Allocating audio ports using address [%s]\n", + janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip); struct sockaddr_storage audio_rtp_address, audio_rtcp_address; while(session->media.local_audio_rtp_port == 0 || session->media.local_audio_rtcp_port == 0) { if(attempts == 0) /* Too many failures */ return -1; + memset(&audio_rtp_address, 0, sizeof(audio_rtp_address)); + memset(&audio_rtcp_address, 0, sizeof(audio_rtcp_address)); if(session->media.audio_rtp_fd == -1) { - session->media.audio_rtp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + session->media.audio_rtp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); int v6only = 0; - if(!ipv6_disabled && session->media.audio_rtp_fd != -1 && + if(use_ipv6_address_family && session->media.audio_rtp_fd != -1 && setsockopt(session->media.audio_rtp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on audio RTP socket (error=%s)\n", g_strerror(errno)); @@ -6652,9 +6688,9 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u } } if(session->media.audio_rtcp_fd == -1) { - session->media.audio_rtcp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + session->media.audio_rtcp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); int v6only = 0; - if(!ipv6_disabled && session->media.audio_rtcp_fd != -1 && + if(use_ipv6_address_family && session->media.audio_rtcp_fd != -1 && setsockopt(session->media.audio_rtcp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on audio RTCP socket (error=%s)\n", g_strerror(errno)); @@ -6667,39 +6703,40 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u int rtp_port = g_random_int_range(rtp_range_min, rtp_range_max); if(rtp_port % 2) rtp_port++; /* Pick an even port for RTP */ - if(!ipv6_disabled) { + if(use_ipv6_address_family) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&audio_rtp_address; addr->sin6_family = AF_INET6; addr->sin6_port = htons(rtp_port); - addr->sin6_addr = in6addr_any; + addr->sin6_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? in6addr_any : janus_network_local_media_ip.ipv6; } else { struct sockaddr_in *addr = (struct sockaddr_in *)&audio_rtp_address; addr->sin_family = AF_INET; addr->sin_port = htons(rtp_port); - addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_addr.s_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? INADDR_ANY : janus_network_local_media_ip.ipv4.s_addr; } - if(bind(session->media.audio_rtp_fd, (struct sockaddr *)(&audio_rtp_address), sizeof(audio_rtp_address)) < 0) { - JANUS_LOG(LOG_ERR, "Bind failed for audio RTP (port %d), trying a different one...\n", rtp_port); + if(bind(session->media.audio_rtp_fd, (struct sockaddr *)(&audio_rtp_address), addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Bind failed for audio RTP (port %d), error (%s), trying a different one...\n", rtp_port, g_strerror(errno)); close(session->media.audio_rtp_fd); session->media.audio_rtp_fd = -1; attempts--; continue; } - JANUS_LOG(LOG_VERB, "Audio RTP listener bound to %s:%d(%d)\n", (local_media_ip ? local_media_ip : local_ip), rtp_port, session->media.audio_rtp_fd); + JANUS_LOG(LOG_VERB, "Audio RTP listener bound to [%s]:%d(%d)\n", + janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip, rtp_port, session->media.audio_rtp_fd); int rtcp_port = rtp_port+1; - if(!ipv6_disabled) { + if(use_ipv6_address_family) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&audio_rtcp_address; addr->sin6_family = AF_INET6; addr->sin6_port = htons(rtcp_port); - addr->sin6_addr = in6addr_any; + addr->sin6_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? in6addr_any : janus_network_local_media_ip.ipv6; } else { struct sockaddr_in *addr = (struct sockaddr_in *)&audio_rtcp_address; addr->sin_family = AF_INET; addr->sin_port = htons(rtcp_port); - addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_addr.s_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? INADDR_ANY : janus_network_local_media_ip.ipv4.s_addr; } - if(bind(session->media.audio_rtcp_fd, (struct sockaddr *)(&audio_rtcp_address), sizeof(audio_rtcp_address)) < 0) { - JANUS_LOG(LOG_ERR, "Bind failed for audio RTCP (port %d), trying a different one...\n", rtcp_port); + if(bind(session->media.audio_rtcp_fd, (struct sockaddr *)(&audio_rtcp_address), addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Bind failed for audio RTCP (port %d), error (%s), trying a different one...\n", rtcp_port, g_strerror(errno)); /* RTP socket is not valid anymore, reset it */ close(session->media.audio_rtp_fd); session->media.audio_rtp_fd = -1; @@ -6708,21 +6745,24 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u attempts--; continue; } - JANUS_LOG(LOG_VERB, "Audio RTCP listener bound to %s:%d(%d)\n", (local_media_ip ? local_media_ip : local_ip), rtcp_port, session->media.audio_rtcp_fd); + JANUS_LOG(LOG_VERB, "Audio RTCP listener bound to [%s]:%d(%d)\n", janus_network_address_is_null(&janus_network_local_media_ip) ?"any":local_media_ip, rtcp_port, session->media.audio_rtcp_fd); session->media.local_audio_rtp_port = rtp_port; session->media.local_audio_rtcp_port = rtcp_port; } } if(session->media.has_video) { - JANUS_LOG(LOG_VERB, "Allocating video ports:\n"); + JANUS_LOG(LOG_VERB, "Allocating video ports using address [%s]\n", + janus_network_address_is_null(&janus_network_local_media_ip ) ?"any" : local_media_ip); struct sockaddr_storage video_rtp_address, video_rtcp_address; while(session->media.local_video_rtp_port == 0 || session->media.local_video_rtcp_port == 0) { if(attempts == 0) /* Too many failures */ return -1; + memset(&video_rtp_address, 0, sizeof(video_rtp_address)); + memset(&video_rtcp_address, 0, sizeof(video_rtcp_address)); if(session->media.video_rtp_fd == -1) { - session->media.video_rtp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + session->media.video_rtp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); int v6only = 0; - if(!ipv6_disabled && session->media.video_rtp_fd != -1 && + if(use_ipv6_address_family && session->media.video_rtp_fd != -1 && setsockopt(session->media.video_rtp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on video RTP socket (error=%s)\n", g_strerror(errno)); @@ -6738,9 +6778,9 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u } } if(session->media.video_rtcp_fd == -1) { - session->media.video_rtcp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + session->media.video_rtcp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); int v6only = 0; - if(!ipv6_disabled && session->media.video_rtcp_fd != -1 && + if(use_ipv6_address_family && session->media.video_rtcp_fd != -1 && setsockopt(session->media.video_rtcp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on video RTCP socket (error=%s)\n", g_strerror(errno)); @@ -6753,39 +6793,40 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u int rtp_port = g_random_int_range(rtp_range_min, rtp_range_max); if(rtp_port % 2) rtp_port++; /* Pick an even port for RTP */ - if(!ipv6_disabled) { + if(use_ipv6_address_family) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&video_rtp_address; addr->sin6_family = AF_INET6; addr->sin6_port = htons(rtp_port); - addr->sin6_addr = in6addr_any; + addr->sin6_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? in6addr_any : janus_network_local_media_ip.ipv6; } else { struct sockaddr_in *addr = (struct sockaddr_in *)&video_rtp_address; addr->sin_family = AF_INET; addr->sin_port = htons(rtp_port); - addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_addr.s_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? INADDR_ANY : janus_network_local_media_ip.ipv4.s_addr; } - if(bind(session->media.video_rtp_fd, (struct sockaddr *)(&video_rtp_address), sizeof(video_rtp_address)) < 0) { - JANUS_LOG(LOG_ERR, "Bind failed for video RTP (port %d), trying a different one...\n", rtp_port); + if(bind(session->media.video_rtp_fd, (struct sockaddr *)(&video_rtp_address), addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Bind failed for video RTP (port %d), error (%s), trying a different one...\n", rtp_port, g_strerror(errno)); close(session->media.video_rtp_fd); session->media.video_rtp_fd = -1; attempts--; continue; } - JANUS_LOG(LOG_VERB, "Video RTP listener bound to %s:%d(%d)\n", (local_media_ip ? local_media_ip : local_ip), rtp_port, session->media.video_rtp_fd); + JANUS_LOG(LOG_VERB, "Video RTP listener bound to [%s]:%d(%d)\n", + janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip, rtp_port, session->media.video_rtp_fd); int rtcp_port = rtp_port+1; - if(!ipv6_disabled) { + if(use_ipv6_address_family) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&video_rtcp_address; addr->sin6_family = AF_INET6; addr->sin6_port = htons(rtcp_port); - addr->sin6_addr = in6addr_any; + addr->sin6_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? in6addr_any : janus_network_local_media_ip.ipv6; } else { struct sockaddr_in *addr = (struct sockaddr_in *)&video_rtcp_address; addr->sin_family = AF_INET; addr->sin_port = htons(rtcp_port); - addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_addr.s_addr = janus_network_address_is_null(&janus_network_local_media_ip) ? INADDR_ANY : janus_network_local_media_ip.ipv4.s_addr; } - if(bind(session->media.video_rtcp_fd, (struct sockaddr *)(&video_rtcp_address), sizeof(video_rtcp_address)) < 0) { - JANUS_LOG(LOG_ERR, "Bind failed for video RTCP (port %d), trying a different one...\n", rtcp_port); + if(bind(session->media.video_rtcp_fd, (struct sockaddr *)(&video_rtcp_address), addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Bind failed for video RTCP (port %d), error (%s), trying a different one...\n", rtcp_port, g_strerror(errno)); /* RTP socket is not valid anymore, reset it */ close(session->media.video_rtp_fd); session->media.video_rtp_fd = -1; @@ -6794,7 +6835,8 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u attempts--; continue; } - JANUS_LOG(LOG_VERB, "Video RTCP listener bound to %s:%d(%d)\n", (local_media_ip ? local_media_ip : local_ip), rtcp_port, session->media.video_rtcp_fd); + JANUS_LOG(LOG_VERB, "Video RTCP listener bound to [%s]:%d(%d)\n", + janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip, rtcp_port, session->media.video_rtcp_fd); session->media.local_video_rtp_port = rtp_port; session->media.local_video_rtcp_port = rtcp_port; } From f74b313c36208649f5c9c041a00e77de5a3c3361 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 16 Nov 2023 16:14:26 +0100 Subject: [PATCH 33/34] Small tweaks to #3263, and updated NoSIP plugin with the same changes --- src/plugins/janus_nosip.c | 84 ++++++++++++++++++++++++++++----------- src/plugins/janus_sip.c | 10 +++-- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/src/plugins/janus_nosip.c b/src/plugins/janus_nosip.c index 2499aaab12..610a622e16 100644 --- a/src/plugins/janus_nosip.c +++ b/src/plugins/janus_nosip.c @@ -264,6 +264,7 @@ static gboolean ipv6_disabled = FALSE; static janus_callbacks *gateway = NULL; static char *local_ip = NULL, *sdp_ip = NULL; +static janus_network_address janus_network_local_ip = { 0 }; #define DEFAULT_RTP_RANGE_MIN 10000 #define DEFAULT_RTP_RANGE_MAX 60000 static uint16_t rtp_range_min = DEFAULT_RTP_RANGE_MIN; @@ -688,7 +689,7 @@ int janus_nosip_init(janus_callbacks *callback, const char *config_path) { janus_config_category *config_general = janus_config_get_create(config, NULL, janus_config_type_category, "general"); janus_config_item *item = janus_config_get(config, config_general, janus_config_type_item, "local_ip"); - if(item && item->value) { + if(item && item->value && strlen(item->value) > 0) { /* Verify that the address is valid */ struct ifaddrs *ifas = NULL; janus_network_address iface; @@ -711,11 +712,32 @@ int janus_nosip_init(janus_callbacks *callback, const char *config_path) { } item = janus_config_get(config, config_general, janus_config_type_item, "sdp_ip"); - if(item && item->value) { + if(item && item->value && strlen(item->value) > 0) { sdp_ip = g_strdup(item->value); JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); } + /* Make sure both IPs are valid, if provided */ + janus_network_address_nullify(&janus_network_local_ip); + if(local_ip) { + if(janus_network_string_to_address(janus_network_query_options_any_ip, local_ip, &janus_network_local_ip) != 0) { + JANUS_LOG(LOG_ERR, "Invalid local media IP address [%s]...\n", local_ip); + return -1; + } + if((janus_network_local_ip.family == AF_INET && janus_network_local_ip.ipv4.s_addr == INADDR_ANY) || + (janus_network_local_ip.family == AF_INET6 && IN6_IS_ADDR_UNSPECIFIED(&janus_network_local_ip.ipv6))) { + janus_network_address_nullify(&janus_network_local_ip); + } + } + JANUS_LOG(LOG_VERB, "Binding media address set to [%s]...\n", janus_network_address_is_null(&janus_network_local_ip) ? "any" : local_ip); + if(!sdp_ip) { + char *ip = janus_network_address_is_null(&janus_network_local_ip) ? local_ip : local_ip; + if(ip) { + sdp_ip = g_strdup(ip); + JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); + } + } + item = janus_config_get(config, config_general, janus_config_type_item, "rtp_port_range"); if(item && item->value) { /* Split in min and max port */ @@ -794,19 +816,29 @@ int janus_nosip_init(janus_callbacks *callback, const char *config_path) { /* This is the callback we'll need to invoke to contact the Janus core */ gateway = callback; - /* Finally, let's check if IPv6 is disabled, as we may need to know for RTP/RTCP sockets */ - int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); - if(fd <= 0) { - ipv6_disabled = TRUE; - } else { - int v6only = 0; - if(setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) + if(janus_network_address_is_null(&janus_network_local_ip) || + janus_network_local_ip.family == AF_INET6) { + /* Finally, let's check if IPv6 is disabled, as we may need to know for RTP/RTCP sockets */ + int fd = socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP); + if(fd <= 0) { ipv6_disabled = TRUE; - } - if(fd > 0) - close(fd); - if(ipv6_disabled) { - JANUS_LOG(LOG_WARN, "IPv6 disabled, will only use IPv4 for RTP/RTCP sockets (NoSIP)\n"); + } else { + int v6only = 0; + if(setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) + ipv6_disabled = TRUE; + } + if(fd > 0) + close(fd); + if(ipv6_disabled) { + if(!janus_network_address_is_null(&janus_network_local_ip)) { + JANUS_LOG(LOG_ERR, "IPv6 disabled and local media address is IPv6...\n"); + return -1; + } + JANUS_LOG(LOG_WARN, "IPv6 disabled, will only use IPv4 for RTP/RTCP sockets (SIP)\n"); + } + } else if(janus_network_local_ip.family == AF_INET) { + /* Disable if we have a specified IPv4 address for RTP/RTCP sockets */ + ipv6_disabled = TRUE; } g_atomic_int_set(&initialized, 1); @@ -1936,7 +1968,7 @@ char *janus_nosip_sdp_manipulate(janus_nosip_session *session, janus_sdp *sdp, g JANUS_LOG(LOG_VERB, "Setting protocol to %s\n", session->media.require_srtp ? "RTP/SAVP" : "RTP/AVP"); if(sdp->c_addr) { g_free(sdp->c_addr); - sdp->c_addr = g_strdup(sdp_ip ? sdp_ip : local_ip); + sdp->c_addr = g_strdup(sdp_ip); } int opusred_pt = answer ? janus_sdp_get_opusred_pt(sdp, -1) : -1; GList *temp = sdp->m_lines; @@ -1998,20 +2030,23 @@ char *janus_nosip_sdp_manipulate(janus_nosip_session *session, janus_sdp *sdp, g } static int janus_nosip_bind_socket(int fd, int port) { + gboolean use_ipv6_address_family = !ipv6_disabled && + (janus_network_address_is_null(&janus_network_local_ip) || janus_network_local_ip.family == AF_INET6); + socklen_t addrlen = use_ipv6_address_family? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in); struct sockaddr_storage rtp_address = { 0 }; if(!ipv6_disabled) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *)&rtp_address; addr->sin6_family = AF_INET6; addr->sin6_port = htons(port); - addr->sin6_addr = in6addr_any; + addr->sin6_addr = janus_network_address_is_null(&janus_network_local_ip) ? in6addr_any : janus_network_local_ip.ipv6; } else { struct sockaddr_in *addr = (struct sockaddr_in *)&rtp_address; addr->sin_family = AF_INET; addr->sin_port = htons(port); - addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_addr.s_addr = janus_network_address_is_null(&janus_network_local_ip) ? INADDR_ANY : janus_network_local_ip.ipv4.s_addr; } - if(bind(fd, (struct sockaddr *)(&rtp_address), sizeof(rtp_address)) < 0) { - JANUS_LOG(LOG_ERR, "Bind failed (port %d)\n", port); + if(bind(fd, (struct sockaddr *)(&rtp_address), addrlen) < 0) { + JANUS_LOG(LOG_ERR, "Bind failed (port %d), error (%s)\n", port, g_strerror(errno)); return -1; } return 0; @@ -2023,6 +2058,9 @@ static int janus_nosip_allocate_port_pair(gboolean video, int fds[2], int ports[ uint16_t rtp_port_start = rtp_port_next; gboolean rtp_port_wrap = FALSE; + gboolean use_ipv6_address_family = !ipv6_disabled && + (janus_network_address_is_null(&janus_network_local_ip) || janus_network_local_ip.family == AF_INET6); + int rtp_fd = -1, rtcp_fd = -1; while(1) { if(rtp_port_wrap && rtp_port_next >= rtp_port_start) { /* Full range scanned */ @@ -2031,9 +2069,9 @@ static int janus_nosip_allocate_port_pair(gboolean video, int fds[2], int ports[ break; } if(rtp_fd == -1) { - rtp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + rtp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); int v6only = 0; - if(!ipv6_disabled && rtp_fd != -1 && setsockopt(rtp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { + if(use_ipv6_address_family && rtp_fd != -1 && setsockopt(rtp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on RTP socket (error=%s)\n", g_strerror(errno)); } @@ -2056,8 +2094,8 @@ static int janus_nosip_allocate_port_pair(gboolean video, int fds[2], int ports[ } if(rtcp_fd == -1) { int v6only = 0; - rtcp_fd = socket(!ipv6_disabled ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); - if(!ipv6_disabled && rtcp_fd != -1 && setsockopt(rtcp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { + rtcp_fd = socket(use_ipv6_address_family ? AF_INET6 : AF_INET, SOCK_DGRAM, 0); + if(use_ipv6_address_family && rtcp_fd != -1 && setsockopt(rtcp_fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)) != 0) { JANUS_LOG(LOG_WARN, "Error setting v6only to false on RTP socket (error=%s)\n", g_strerror(errno)); } diff --git a/src/plugins/janus_sip.c b/src/plugins/janus_sip.c index 31adfc7d62..74e75d2e73 100644 --- a/src/plugins/janus_sip.c +++ b/src/plugins/janus_sip.c @@ -2019,8 +2019,11 @@ int janus_sip_init(janus_callbacks *callback, const char *config_path) { } JANUS_LOG(LOG_VERB, "Binding media address set to [%s]...\n", janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip); if(!sdp_ip) { - sdp_ip = janus_network_address_is_null(&janus_network_local_media_ip) ? local_ip : local_media_ip; - JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); + char *ip = janus_network_address_is_null(&janus_network_local_media_ip) ? local_ip : local_media_ip; + if(ip) { + sdp_ip = g_strdup(ip); + JANUS_LOG(LOG_VERB, "IP to advertise in SDP: %s\n", sdp_ip); + } } /* Setup sofia */ @@ -6745,7 +6748,8 @@ static int janus_sip_allocate_local_ports(janus_sip_session *session, gboolean u attempts--; continue; } - JANUS_LOG(LOG_VERB, "Audio RTCP listener bound to [%s]:%d(%d)\n", janus_network_address_is_null(&janus_network_local_media_ip) ?"any":local_media_ip, rtcp_port, session->media.audio_rtcp_fd); + JANUS_LOG(LOG_VERB, "Audio RTCP listener bound to [%s]:%d(%d)\n", + janus_network_address_is_null(&janus_network_local_media_ip) ? "any" : local_media_ip, rtcp_port, session->media.audio_rtcp_fd); session->media.local_audio_rtp_port = rtp_port; session->media.local_audio_rtcp_port = rtcp_port; } From 271d3193869ce2e6021fceea7b9f8d02a7a0f7ea Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 16 Nov 2023 16:31:18 +0100 Subject: [PATCH 34/34] Add support for abs-capture-time RTP extension (#3161) --- fuzzers/rtp_fuzzer.c | 3 ++- src/ice.c | 32 ++++++++++++++++++++++++++-- src/ice.h | 2 ++ src/janus.c | 32 +++++++++++++++++++++++----- src/plugins/janus_echotest.c | 2 ++ src/plugins/janus_recordplay.c | 4 +--- src/plugins/plugin.h | 4 +++- src/postprocessing/janus-pp-rec.c | 5 +---- src/record.c | 5 +---- src/rtp.c | 35 ++++++++++++++++++++++++++++++- src/rtp.h | 20 +++++++++++++++++- src/utils.h | 8 +++++++ 12 files changed, 130 insertions(+), 22 deletions(-) diff --git a/fuzzers/rtp_fuzzer.c b/fuzzers/rtp_fuzzer.c index 0fdb005a8a..34ffb07f22 100644 --- a/fuzzers/rtp_fuzzer.c +++ b/fuzzers/rtp_fuzzer.c @@ -76,7 +76,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { janus_rtp_header_extension_parse_audio_level((char *)data, size, 1, NULL, NULL); janus_rtp_header_extension_parse_playout_delay((char *)data, size, 1, NULL, NULL); janus_rtp_header_extension_parse_transport_wide_cc((char *)data, size, 1, &transport_seq_num); - janus_rtp_header_extension_parse_abs_sent_time((char *)data, size, 1, NULL); + janus_rtp_header_extension_parse_abs_send_time((char *)data, size, 1, NULL); + janus_rtp_header_extension_parse_abs_capture_time((char *)data, size, 1, NULL); janus_rtp_header_extension_parse_video_orientation((char * )data, size, 1, &c, &f, &r1, &r0); janus_rtp_header_extension_parse_dependency_desc((char *)data, size, 1, (uint8_t *)&dd, &sizedd); diff --git a/src/ice.c b/src/ice.c index 0e5b25f4cb..7d334efa26 100644 --- a/src/ice.c +++ b/src/ice.c @@ -2876,6 +2876,13 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp memcpy(rtp.extensions.dd_content, dd, len); } } + if(pc->abs_capture_time_ext_id != -1) { + uint64_t abs_ts = 0; + if(janus_rtp_header_extension_parse_abs_capture_time(buf, buflen, + pc->abs_capture_time_ext_id, &abs_ts) == 0) { + rtp.extensions.abs_capture_ts = abs_ts; + } + } /* Pass the packet to the plugin */ janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->incoming_rtp && handle->app_handle && @@ -3877,7 +3884,7 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p totlen += plen; /* We need to strip extensions, here, and add those that need to be there manually */ uint16_t extlen = 0; - char extensions[300]; + char extensions[320]; uint16_t extbufsize = sizeof(extensions); janus_rtp_header *header = (janus_rtp_header *)packet->data; header->extension = 0; @@ -3888,7 +3895,8 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p (!video && packet->extensions.audio_level > -1 && handle->pc->audiolevel_ext_id > 0) || (video && packet->extensions.video_rotation > -1 && handle->pc->videoorientation_ext_id > 0) || (video && packet->extensions.min_delay > -1 && packet->extensions.max_delay > -1 && handle->pc->playoutdelay_ext_id > 0) || - (video && packet->extensions.dd_len > 0 && handle->pc->dependencydesc_ext_id > 0)) { + (video && packet->extensions.dd_len > 0 && handle->pc->dependencydesc_ext_id > 0) || + (packet->extensions.abs_capture_ts > 0 && handle->pc->abs_capture_time_ext_id > 0)) { /* Do we need 2-byte extemsions, or are 1-byte extensions fine? */ gboolean use_2byte = (video && packet->extensions.dd_len > 16 && handle->pc->dependencydesc_ext_id > 0); /* Write the extension(s) */ @@ -4071,6 +4079,26 @@ static void janus_ice_rtp_extension_update(janus_ice_handle *handle, janus_ice_p } } } + /* Check if we need to add the abs-capture-time extension */ + if(packet->extensions.abs_capture_ts > 0 && handle->pc->abs_capture_time_ext_id > 0) { + uint64_t abs64 = htonll(packet->extensions.abs_capture_ts); + if(!use_2byte) { + *index = (handle->pc->abs_capture_time_ext_id << 4) + 15; + memcpy(index+1, &abs64, 8); + memset(index+9, 0, 8); + index += 17; + extlen += 17; + extbufsize -= 17; + } else { + *index = handle->pc->abs_capture_time_ext_id; + *(index+1) = 16; + memcpy(index+2, &abs64, 8); + memset(index+8, 0, 8); + index += 18; + extlen += 18; + extbufsize -= 18; + } + } /* Calculate the whole length */ uint16_t words = extlen/4; if(extlen%4 != 0) diff --git a/src/ice.h b/src/ice.h index 2c28d1b5c3..e9d121f2eb 100644 --- a/src/ice.h +++ b/src/ice.h @@ -472,6 +472,8 @@ struct janus_ice_peerconnection { gint dependencydesc_ext_id; /*! \brief Absolute Send Time ext ID */ gint abs_send_time_ext_id; + /*! \brief Absolute Capture Time ext ID */ + gint abs_capture_time_ext_id; /*! \brief Whether we do transport wide cc */ gboolean do_transport_wide_cc; /*! \brief Transport wide cc rtp ext ID */ diff --git a/src/janus.c b/src/janus.c index 80be2646c0..11378ddd36 100644 --- a/src/janus.c +++ b/src/janus.c @@ -1591,11 +1591,12 @@ int janus_process_incoming_request(janus_request *request) { janus_request_ice_handle_answer(handle, jsep_sdp); /* Check if the answer does contain the mid/abs-send-time/twcc extmaps */ int mindex = 0; - gboolean do_mid = FALSE, do_twcc = FALSE, do_dd = FALSE, do_abs_send_time = FALSE; + gboolean do_mid = FALSE, do_twcc = FALSE, do_dd = FALSE, do_abs_send_time = FALSE, do_abs_capture_time = FALSE; GList *temp = parsed_sdp->m_lines; while(temp) { janus_sdp_mline *m = (janus_sdp_mline *)temp->data; - gboolean have_mid = FALSE, have_twcc = FALSE, have_dd = FALSE, have_abs_send_time = FALSE; + gboolean have_mid = FALSE, have_twcc = FALSE, have_dd = FALSE, + have_abs_send_time = FALSE, have_abs_capture_time = FALSE; GList *tempA = m->attributes; while(tempA) { janus_sdp_attribute *a = (janus_sdp_attribute *)tempA->data; @@ -1608,6 +1609,8 @@ int janus_process_incoming_request(janus_request *request) { have_dd = TRUE; else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_SEND_TIME)) have_abs_send_time = TRUE; + else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME)) + have_abs_capture_time = TRUE; } tempA = tempA->next; } @@ -1615,6 +1618,7 @@ int janus_process_incoming_request(janus_request *request) { do_twcc = do_twcc || have_twcc; do_dd = do_dd || have_dd; do_abs_send_time = do_abs_send_time || have_abs_send_time; + do_abs_capture_time = do_abs_capture_time || have_abs_capture_time; mindex++; temp = temp->next; } @@ -1628,6 +1632,8 @@ int janus_process_incoming_request(janus_request *request) { handle->pc->dependencydesc_ext_id = 0; if(!do_abs_send_time && handle->pc) handle->pc->abs_send_time_ext_id = 0; + if(!do_abs_capture_time && handle->pc) + handle->pc->abs_capture_time_ext_id = 0; } else { /* Check if the mid RTP extension is being negotiated */ handle->pc->mid_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_MID); @@ -1642,6 +1648,8 @@ int janus_process_incoming_request(janus_request *request) { handle->pc->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); /* Check if the abs-send-time ID extension is being negotiated */ handle->pc->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME); + /* Check if the abs-capture-time ID extension is being negotiated */ + handle->pc->abs_capture_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME); /* Check if transport wide CC is supported */ int transport_wide_cc_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC); handle->pc->do_transport_wide_cc = transport_wide_cc_ext_id > 0 ? TRUE : FALSE; @@ -1708,6 +1716,8 @@ int janus_process_incoming_request(janus_request *request) { handle->pc->playoutdelay_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); /* Check if the abs-send-time ID extension is being negotiated */ handle->pc->abs_send_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_SEND_TIME); + /* Check if the abs-capture-time ID extension is being negotiated */ + handle->pc->abs_capture_time_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME); /* Check if transport wide CC is supported */ int transport_wide_cc_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC); handle->pc->do_transport_wide_cc = transport_wide_cc_ext_id > 0 ? TRUE : FALSE; @@ -3256,6 +3266,8 @@ json_t *janus_admin_peerconnection_summary(janus_ice_peerconnection *pc) { json_object_set_new(se, JANUS_RTP_EXTMAP_REPAIRED_RID, json_integer(pc->ridrtx_ext_id)); if(pc->abs_send_time_ext_id > 0) json_object_set_new(se, JANUS_RTP_EXTMAP_ABS_SEND_TIME, json_integer(pc->abs_send_time_ext_id)); + if(pc->abs_capture_time_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_ABS_SEND_TIME, json_integer(pc->abs_capture_time_ext_id)); if(pc->transport_wide_cc_ext_id > 0) json_object_set_new(se, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, json_integer(pc->transport_wide_cc_ext_id)); if(pc->audiolevel_ext_id > 0) @@ -3824,7 +3836,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } /* Make sure we don't send the rid/repaired-rid attributes when offering ourselves */ int mindex = 0; - int mid_ext_id = 0, transport_wide_cc_ext_id = 0, abs_send_time_ext_id = 0, + int mid_ext_id = 0, transport_wide_cc_ext_id = 0, abs_send_time_ext_id = 0, abs_capture_time_ext_id = 0, audiolevel_ext_id = 0, videoorientation_ext_id = 0, playoutdelay_ext_id = 0, dependencydesc_ext_id = 0; GList *temp = parsed_sdp->m_lines; while(temp) { @@ -3842,6 +3854,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug transport_wide_cc_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_SEND_TIME)) abs_send_time_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME)) + abs_capture_time_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_AUDIO_LEVEL)) audiolevel_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION)) @@ -3878,6 +3892,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } if(ice_handle->pc && ice_handle->pc->abs_send_time_ext_id != abs_send_time_ext_id) ice_handle->pc->abs_send_time_ext_id = abs_send_time_ext_id; + if(ice_handle->pc && ice_handle->pc->abs_capture_time_ext_id != abs_capture_time_ext_id) + ice_handle->pc->abs_capture_time_ext_id = abs_capture_time_ext_id; if(ice_handle->pc && ice_handle->pc->audiolevel_ext_id != audiolevel_ext_id) ice_handle->pc->audiolevel_ext_id = audiolevel_ext_id; if(ice_handle->pc && ice_handle->pc->videoorientation_ext_id != videoorientation_ext_id) @@ -3891,7 +3907,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug /* Check if the answer does contain the mid/rid/repaired-rid/abs-send-time/twcc extmaps */ int mindex = 0; gboolean do_mid = FALSE, do_rid = FALSE, do_repaired_rid = FALSE, - do_dd = FALSE, do_twcc = FALSE, do_abs_send_time = FALSE; + do_dd = FALSE, do_twcc = FALSE, do_abs_send_time = FALSE, do_abs_capture_time = FALSE; GList *temp = parsed_sdp->m_lines; janus_mutex_lock(&ice_handle->mutex); while(temp) { @@ -3899,7 +3915,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug janus_ice_peerconnection_medium *medium = ice_handle->pc ? g_hash_table_lookup(ice_handle->pc->media, GUINT_TO_POINTER(mindex)) : NULL; gboolean have_mid = FALSE, have_rid = FALSE, have_repaired_rid = FALSE, - have_twcc = FALSE, have_dd = FALSE, have_abs_send_time = FALSE, have_msid = FALSE; + have_twcc = FALSE, have_dd = FALSE, have_abs_send_time = FALSE, + have_abs_capture_time = FALSE, have_msid = FALSE; int opusred_pt = -1; GList *tempA = m->attributes; while(tempA) { @@ -3945,6 +3962,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug do_dd = TRUE; else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_SEND_TIME)) have_abs_send_time = TRUE; + else if(strstr(a->value, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME)) + have_abs_capture_time = TRUE; } else if(m->type == JANUS_SDP_AUDIO && medium != NULL && medium->opusred_pt > 0 && a->name && a->value && !strcasecmp(a->name, "rtpmap") && strstr(a->value, "red/48000/2")) { opusred_pt = atoi(a->value); @@ -3979,6 +3998,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug do_twcc = do_twcc || have_twcc; do_dd = do_dd || have_dd; do_abs_send_time = do_abs_send_time || have_abs_send_time; + do_abs_capture_time = do_abs_capture_time || have_abs_capture_time; mindex++; temp = temp->next; } @@ -3998,6 +4018,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug ice_handle->pc->dependencydesc_ext_id = 0; if(!do_abs_send_time && ice_handle->pc) ice_handle->pc->abs_send_time_ext_id = 0; + if(!do_abs_capture_time && ice_handle->pc) + ice_handle->pc->abs_capture_time_ext_id = 0; janus_mutex_unlock(&ice_handle->mutex); } if(!updating && !janus_ice_is_full_trickle_enabled()) { diff --git a/src/plugins/janus_echotest.c b/src/plugins/janus_echotest.c index 4ef6907014..5e54c15125 100644 --- a/src/plugins/janus_echotest.c +++ b/src/plugins/janus_echotest.c @@ -1261,6 +1261,8 @@ static void *janus_echotest_handler(void *data) { JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_DEPENDENCY_DESC, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_ABS_SEND_TIME, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME, JANUS_SDP_OA_DONE); temp = temp->next; } diff --git a/src/plugins/janus_recordplay.c b/src/plugins/janus_recordplay.c index 0812a0bdd9..6fd1086987 100644 --- a/src/plugins/janus_recordplay.c +++ b/src/plugins/janus_recordplay.c @@ -778,8 +778,6 @@ static void janus_recordplay_message_free(janus_recordplay_message *msg) { #define JANUS_RECORDPLAY_ERROR_RECORDING_EXISTS 420 #define JANUS_RECORDPLAY_ERROR_UNKNOWN_ERROR 499 -#define ntohll(x) ((1==ntohl(1)) ? (x) : ((gint64)ntohl((x) & 0xFFFFFFFF) << 32) | ntohl((x) >> 32)) - /* Plugin implementation */ int janus_recordplay_init(janus_callbacks *callback, const char *config_path) { if(g_atomic_int_get(&stopping)) { @@ -2529,7 +2527,7 @@ janus_recordplay_frame_packet *janus_recordplay_get_frames(const char *dir, cons JANUS_LOG(LOG_WARN, "Missing data timestamp header"); break; } - when = ntohll(when); + when = ntohll((uint64_t)when); offset += sizeof(gint64); len -= sizeof(gint64); /* Generate frame packet and insert in the ordered list */ diff --git a/src/plugins/plugin.h b/src/plugins/plugin.h index 2c3c174024..93d234dc04 100644 --- a/src/plugins/plugin.h +++ b/src/plugins/plugin.h @@ -171,7 +171,7 @@ janus_plugin *create(void) { * Janus instance or it will crash. * */ -#define JANUS_PLUGIN_API_VERSION 102 +#define JANUS_PLUGIN_API_VERSION 103 /*! \brief Initialization of all plugin properties to NULL * @@ -575,6 +575,8 @@ struct janus_plugin_rtp_extensions { uint8_t dd_len; /*! \brief Dependency Descriptor content */ uint8_t dd_content[256]; + /*! \brief Absolute Capture Time timestamp */ + uint64_t abs_capture_ts; }; /*! \brief Helper method to initialise/reset the RTP extensions field * @note This is important because each of the supported extensions may diff --git a/src/postprocessing/janus-pp-rec.c b/src/postprocessing/janus-pp-rec.c index 45422f1478..0a20a3c229 100644 --- a/src/postprocessing/janus-pp-rec.c +++ b/src/postprocessing/janus-pp-rec.c @@ -133,9 +133,6 @@ Usage: janus-pp-rec [OPTIONS] source.mjr #include "pp-srt.h" #include "pp-binary.h" -#define htonll(x) ((1==htonl(1)) ? (x) : ((gint64)htonl((x) & 0xFFFFFFFF) << 32) | htonl((x) >> 32)) -#define ntohll(x) ((1==ntohl(1)) ? (x) : ((gint64)ntohl((x) & 0xFFFFFFFF) << 32) | ntohl((x) >> 32)) - int janus_log_level = 4; gboolean janus_log_timestamps = FALSE; gboolean janus_log_colors = TRUE; @@ -836,7 +833,7 @@ int main(int argc, char *argv[]) { JANUS_LOG(LOG_WARN, "Missing data timestamp header"); break; } - when = ntohll(when); + when = ntohll((uint64_t)when); offset += sizeof(gint64); len -= sizeof(gint64); /* Generate frame packet and insert in the ordered list */ diff --git a/src/record.c b/src/record.c index 050142599c..bd36753e03 100644 --- a/src/record.c +++ b/src/record.c @@ -29,9 +29,6 @@ #include "debug.h" #include "utils.h" -#define htonll(x) ((1==htonl(1)) ? (x) : ((gint64)htonl((x) & 0xFFFFFFFF) << 32) | htonl((x) >> 32)) -#define ntohll(x) ((1==ntohl(1)) ? (x) : ((gint64)ntohl((x) & 0xFFFFFFFF) << 32) | ntohl((x) >> 32)) - /* Info header in the structured recording */ static const char *header = "MJR00002"; @@ -439,7 +436,7 @@ int janus_recorder_save_frame(janus_recorder *recorder, char *buffer, uint lengt } if(recorder->type == JANUS_RECORDER_DATA) { /* If it's data, then we need to prepend timing related info, as it's not there by itself */ - gint64 now = htonll(janus_get_real_time()); + gint64 now = htonll((uint64_t)janus_get_real_time()); res = fwrite(&now, sizeof(gint64), 1, recorder->file); if(res != 1) { JANUS_LOG(LOG_WARN, "Couldn't write data timestamp in .mjr file (%zu != %zu, %s), expect issues post-processing\n", diff --git a/src/rtp.c b/src/rtp.c index 086ff19054..b07667ba61 100644 --- a/src/rtp.c +++ b/src/rtp.c @@ -103,6 +103,8 @@ const char *janus_rtp_header_extension_get_from_id(const char *sdp, int id) { return JANUS_RTP_EXTMAP_TOFFSET; if(strstr(extension, JANUS_RTP_EXTMAP_ABS_SEND_TIME)) return JANUS_RTP_EXTMAP_ABS_SEND_TIME; + if(strstr(extension, JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME)) + return JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME; if(strstr(extension, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC)) return JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC; if(strstr(extension, JANUS_RTP_EXTMAP_MID)) @@ -332,7 +334,7 @@ int janus_rtp_header_extension_parse_dependency_desc(char *buf, int len, int id, return 0; } -int janus_rtp_header_extension_parse_abs_sent_time(char *buf, int len, int id, uint32_t *abs_ts) { +int janus_rtp_header_extension_parse_abs_send_time(char *buf, int len, int id, uint32_t *abs_ts) { char *ext = NULL; uint8_t idlen = 0; if(janus_rtp_header_extension_find(buf, len, id, NULL, NULL, &ext, &idlen) < 0) @@ -363,6 +365,37 @@ int janus_rtp_header_extension_set_abs_send_time(char *buf, int len, int id, uin return 0; } +int janus_rtp_header_extension_parse_abs_capture_time(char *buf, int len, int id, uint64_t *abs_ts) { + char *ext = NULL; + uint8_t idlen = 0; + if(janus_rtp_header_extension_find(buf, len, id, NULL, NULL, &ext, &idlen) < 0) + return -1; + /* a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time */ + if(ext == NULL) + return -2; + if(idlen < 8 || idlen > len-(ext-buf)-1) + return -3; + uint64_t abs64 = 0; + memcpy(&abs64, ext, 8); + if(abs_ts) + *abs_ts = ntohll(abs64); + return 0; +} + +int janus_rtp_header_extension_set_abs_capture_time(char *buf, int len, int id, uint64_t abs_ts) { + char *ext = NULL; + uint8_t idlen = 0; + if(janus_rtp_header_extension_find(buf, len, id, NULL, NULL, &ext, &idlen) < 0) + return -1; + if(ext == NULL) + return -2; + if(idlen < 8 || idlen > len-(ext-buf)-1) + return -3; + uint64_t abs64 = htonll(abs_ts); + memcpy(ext, &abs64, 8); + return 0; +} + int janus_rtp_header_extension_parse_transport_wide_cc(char *buf, int len, int id, uint16_t *transSeqNum) { char *ext = NULL; uint8_t idlen = 0; diff --git a/src/rtp.h b/src/rtp.h index cebb8b3c86..34d698b9fb 100644 --- a/src/rtp.h +++ b/src/rtp.h @@ -78,6 +78,8 @@ typedef struct janus_rtp_header_extension { #define JANUS_RTP_EXTMAP_TOFFSET "urn:ietf:params:rtp-hdrext:toffset" /*! \brief a=extmap:3 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time */ #define JANUS_RTP_EXTMAP_ABS_SEND_TIME "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time" +/*! \brief a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time */ +#define JANUS_RTP_EXTMAP_ABS_CAPTURE_TIME "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" /*! \brief a=extmap:4 urn:3gpp:video-orientation */ #define JANUS_RTP_EXTMAP_VIDEO_ORIENTATION "urn:3gpp:video-orientation" /*! \brief a=extmap:5 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 */ @@ -221,7 +223,7 @@ int janus_rtp_header_extension_parse_dependency_desc(char *buf, int len, int id, * @param[in] id The extension ID to look for * @param[out] abs_ts Variable where the parsed abs-send-time value will be stored * @returns 0 if found, -1 otherwise */ -int janus_rtp_header_extension_parse_abs_sent_time(char *buf, int len, int id, uint32_t *abs_ts); +int janus_rtp_header_extension_parse_abs_send_time(char *buf, int len, int id, uint32_t *abs_ts); /*! \brief Helper to set an abs-send-time RTP extension (http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time) * @param[in] buf The packet data @@ -231,6 +233,22 @@ int janus_rtp_header_extension_parse_abs_sent_time(char *buf, int len, int id, u * @returns 0 if found, -1 otherwise */ int janus_rtp_header_extension_set_abs_send_time(char *buf, int len, int id, uint32_t abs_ts); +/*! \brief Helper to parse an abs-capture-time RTP extension (http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time) + * @param[in] buf The packet data + * @param[in] len The packet data length in bytes + * @param[in] id The extension ID to look for + * @param[out] abs_ts Variable where the parsed abs-capture-time value will be stored + * @returns 0 if found, -1 otherwise */ +int janus_rtp_header_extension_parse_abs_capture_time(char *buf, int len, int id, uint64_t *abs_ts); + +/*! \brief Helper to set an abs-capture-time RTP extension (http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time) + * @param[in] buf The packet data + * @param[in] len The packet data length in bytes + * @param[in] id The extension ID to look for + * @param[out] abs_ts Absolute Send Time value to set + * @returns 0 if found, -1 otherwise */ +int janus_rtp_header_extension_set_abs_capture_time(char *buf, int len, int id, uint64_t abs_ts); + /*! \brief Helper to parse a transport wide sequence number (https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01) * @param[in] buf The packet data * @param[in] len The packet data length in bytes diff --git a/src/utils.h b/src/utils.h index d20e2fde6f..cf4ef987c8 100644 --- a/src/utils.h +++ b/src/utils.h @@ -32,6 +32,14 @@ struct janus_json_parameter { unsigned int flags; }; +#ifndef htonll +#define htonll(x) ((1==htonl(1)) ? (x) : ((guint64)htonl((x) & 0xFFFFFFFF) << 32) | htonl((x) >> 32)) +#endif +#ifndef ntohll +#define ntohll(x) ((1==ntohl(1)) ? (x) : ((guint64)ntohl((x) & 0xFFFFFFFF) << 32) | ntohl((x) >> 32)) +#endif + + /*! \brief Helper to retrieve the system monotonic time, as Glib's * g_get_monotonic_time may not be available (only since 2.28) * @returns The system monotonic time */