Merging upstream version 2.2.0+dfsg.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

22 files changed, 428 insertions, 113 deletions

diff --git a/include/Client.hpp b/include/Client.hpp
index 3848e92..8454fc3 100644
--- a/include/Client.hpp
+++ b/include/Client.hpp
@@ -57,6 +57,7 @@
 #include "Settings.hpp"
 #include "Timestamp.hpp"
 #include "isochronous.hpp"
+#include "iperf_multicast_api.h"
 #include "Mutex.h"
 
 /* ------------------------------------------------------------------- */
@@ -82,7 +83,7 @@ public:
     void mySockInit(void);
     bool isConnected(void) const;
     int SendFirstPayload(void);
-    int BarrierClient(struct BarrierMutex *);
+    bool BarrierClient(struct BarrierMutex *);
     void RunBounceBackTCP(void);
     struct ReportHeader *myJob;
 
@@ -97,8 +98,7 @@ private:
     void FinishTrafficActions(void);
     void AwaitServerFinPacket(void);
     bool InProgress(void);
-    void PostNullEvent(void);
-    void PostNullEvent(bool isFirst);
+    void PostNullEvent(bool isFirst, bool select_retry);
     void AwaitServerCloseEvent(void);
     inline void tcp_shutdown(void);
     bool connected;
@@ -110,13 +110,14 @@ private:
     bool apply_first_udppkt_delay;
     int udp_payload_minimum;
     void myReportPacket(void);
+    void myReportPacket(struct ReportStruct *);
     // TCP plain
     void RunTCP(void);
     // TCP version which supports rate limiting per -b
     void RunRateLimitedTCP(void);
     void RunNearCongestionTCP(void);
+    bool AwaitSelectWrite(void);
 #if HAVE_DECL_TCP_NOTSENT_LOWAT
-    bool AwaitWriteSelectEventTCP(void);
     void RunWriteEventsTCP(void);
 #endif
     // UDP traffic with isochronous and vbr support
@@ -142,6 +143,17 @@ private:
     bool mysock_init_done;
     bool peerclose;
     Timestamp write_start;
+#if HAVE_DECL_SO_MAX_PACING_RATE
+    Timestamp PacingStepTime;
+#endif
+#if HAVE_DECL_TCP_TX_DELAY
+    enum delay_state {NO_DELAY=0, ADD_DELAY};
+    delay_state current_state;
+    int state_tokens[2];
+    inline void apply_txdelay_func (void);
+    Timestamp TcpTxDelayQuantumEnd;
+#endif
+
 }; // end class Client
 
 #endif // CLIENT_H
diff --git a/include/Listener.hpp b/include/Listener.hpp
index 92baad7..89bbcf1 100644
--- a/include/Listener.hpp
+++ b/include/Listener.hpp
@@ -58,6 +58,7 @@
 #include "Thread.h"
 #include "Settings.hpp"
 #include "Timestamp.hpp"
+#include "iperf_multicast_api.h"
 
 class Listener;
 
@@ -82,8 +83,7 @@ private:
     bool apply_client_settings_tcp(thread_Settings *server);
     bool apply_client_settings(thread_Settings *server);
     int client_test_ack(thread_Settings *server);
-    void my_multicast_join(void);
-    void my_listen(void);
+    bool my_listen(void);
     int my_accept(thread_Settings *server);
     int udp_accept(thread_Settings *server);
     int tuntap_accept(thread_Settings *server);
diff --git a/include/Locale.h b/include/Locale.h
index 65e547e..d07bf29 100644
--- a/include/Locale.h
+++ b/include/Locale.h
@@ -79,6 +79,7 @@ extern const char usage_long1[];
 extern const char usage_long2[];
 
 extern const char version[];
+extern const char branch_version[];
 
 /* -------------------------------------------------------------------
  * settings
@@ -96,6 +97,8 @@ extern const char client_report_epoch_start_current[];
 
 extern const char server_pid_port[];
 
+extern const char server_working_load_port[];
+
 extern const char server_pid_portrange[];
 
 extern const char client_pid_port[];
@@ -140,6 +143,8 @@ extern const char client_burstperiod[];
 
 extern const char client_burstperiodcount[];
 
+extern const char client_bbburstperiodcount[];
+
 extern const char client_bounceback[];
 
 extern const char client_bounceback_noqack[];
@@ -147,6 +152,8 @@ extern const char client_bounceback_noqack[];
 extern const char server_burstperiod[];
 
 extern const char client_fq_pacing[];
+
+extern const char client_fq_pacing_step[];
 /* -------------------------------------------------------------------
  * Legacy reports
  * ------------------------------------------------------------------- */
@@ -223,6 +230,16 @@ extern const char report_sumcnt_write_enhanced_write_header[];
 
 extern const char report_bw_write_enhanced_format[];
 
+extern const char report_bw_write_fq_header[];
+
+extern const char report_bw_write_fq_format[];
+
+extern const char report_bw_write_enhanced_fq_header[];
+
+extern const char report_bw_write_enhanced_fq_format[];
+
+extern const char report_bw_write_enhanced_fq_final_format[];
+
 extern const char report_write_enhanced_write_header[];
 
 extern const char report_write_enhanced_write_format[];
@@ -307,14 +324,12 @@ extern const char report_udp_fullduplex_sum_format[];
 
 extern const char report_udp_fullduplex_enhanced_format[];
 
-extern const char report_sumcnt_udp_fullduplex_header[];
-
-extern const char report_sumcnt_udp_fullduplex_format[];
-
 extern const char report_client_bb_bw_header[];
 
 extern const char report_client_bb_bw_format[];
 
+extern const char report_client_bb_bw_final_format[];
+
 extern const char report_client_bb_bw_triptime_format[];
 
 extern const char report_bw_isoch_enhanced_netpwr_header[];
@@ -365,14 +380,20 @@ extern const char reportCSV_peer[];
 
 extern const char reportCSV_bw_format[];
 
+extern const char reportCSV_bw_read_enhanced_header[];
 extern const char reportCSV_bw_read_enhanced_format[];
 
+extern const char reportCSV_bw_write_enhanced_header[];
 extern const char reportCSV_bw_write_enhanced_format[];
 
 extern const char reportCSV_bw_jitter_loss_format[];
 
+extern const char reportCSV_bw_jitter_loss_pps_header[];
 extern const char reportCSV_bw_jitter_loss_pps_format[];
 
+extern const char reportCSV_client_bb_bw_tcp_header[];
+extern const char reportCSV_client_bb_bw_tcp_format[];
+
 /* -------------------------------------------------------------------
  * warnings
  * ------------------------------------------------------------------- */
@@ -431,6 +452,10 @@ extern const char error_starttime_exceeds[];
 
 extern const char error_delaytime_exceeds[];
 
+extern const char report_client_bb_triptime_clocksync_error[];
+
+extern const char report_omitted[] ;
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/include/Makefile.am b/include/Makefile.am
index 033fbd3..bde8185 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -1,2 +1,2 @@
-EXTRA_DIST = Client.hpp Condition.h Extractor.h active_hosts.h packet_ring.h Listener.hpp Locale.h Makefile.am Mutex.h PerfSocket.hpp Reporter.h Server.hpp Settings.hpp SocketAddr.h Thread.h Timestamp.hpp config.win32.h delay.h gettimeofday.h gnu_getopt.h headers.h inet_aton.h service.h snprintf.h util.h version.h histogram.h isochronous.hpp pdfs.h checksums.h payloads.h gettcpinfo.h dscp.h iperf_formattime.h
+EXTRA_DIST = Client.hpp Condition.h Extractor.h active_hosts.h packet_ring.h Listener.hpp Locale.h Makefile.am Mutex.h PerfSocket.hpp Reporter.h Server.hpp Settings.hpp SocketAddr.h Thread.h Timestamp.hpp config.win32.h delay.h gettimeofday.h gnu_getopt.h headers.h inet_aton.h service.h snprintf.h util.h version.h histogram.h isochronous.hpp pdfs.h checksums.h payloads.h gettcpinfo.h dscp.h iperf_formattime.h iperf_multicast_api.h
 DISTCLEANFILES = $(top_builddir)/include/iperf-int.h
diff --git a/include/Makefile.in b/include/Makefile.in
index ae4ef10..d920143 100644
--- a/include/Makefile.in
+++ b/include/Makefile.in
@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.16.2 from Makefile.am.
+# Makefile.in generated by automake 1.16.5 from Makefile.am.
 # @configure_input@
 
-# Copyright (C) 1994-2020 Free Software Foundation, Inc.
+# Copyright (C) 1994-2021 Free Software Foundation, Inc.
 
 # This Makefile.in is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -133,6 +133,8 @@ CCDEPMODE = @CCDEPMODE@
 CFLAGS = @CFLAGS@
 CPP = @CPP@
 CPPFLAGS = @CPPFLAGS@
+CSCOPE = @CSCOPE@
+CTAGS = @CTAGS@
 CXX = @CXX@
 CXXDEPMODE = @CXXDEPMODE@
 CXXFLAGS = @CXXFLAGS@
@@ -143,6 +145,7 @@ ECHO_C = @ECHO_C@
 ECHO_N = @ECHO_N@
 ECHO_T = @ECHO_T@
 EGREP = @EGREP@
+ETAGS = @ETAGS@
 EXEEXT = @EXEEXT@
 GREP = @GREP@
 INSTALL = @INSTALL@
@@ -234,7 +237,7 @@ target_alias = @target_alias@
 top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
-EXTRA_DIST = Client.hpp Condition.h Extractor.h active_hosts.h packet_ring.h Listener.hpp Locale.h Makefile.am Mutex.h PerfSocket.hpp Reporter.h Server.hpp Settings.hpp SocketAddr.h Thread.h Timestamp.hpp config.win32.h delay.h gettimeofday.h gnu_getopt.h headers.h inet_aton.h service.h snprintf.h util.h version.h histogram.h isochronous.hpp pdfs.h checksums.h payloads.h gettcpinfo.h dscp.h iperf_formattime.h
+EXTRA_DIST = Client.hpp Condition.h Extractor.h active_hosts.h packet_ring.h Listener.hpp Locale.h Makefile.am Mutex.h PerfSocket.hpp Reporter.h Server.hpp Settings.hpp SocketAddr.h Thread.h Timestamp.hpp config.win32.h delay.h gettimeofday.h gnu_getopt.h headers.h inet_aton.h service.h snprintf.h util.h version.h histogram.h isochronous.hpp pdfs.h checksums.h payloads.h gettcpinfo.h dscp.h iperf_formattime.h iperf_multicast_api.h
 DISTCLEANFILES = $(top_builddir)/include/iperf-int.h
 all: all-am
 
@@ -274,7 +277,6 @@ ctags CTAGS:
 
 cscope cscopelist:
 
-
 distdir: $(BUILT_SOURCES)
 	$(MAKE) $(AM_MAKEFLAGS) distdir-am
 
diff --git a/include/PerfSocket.hpp b/include/PerfSocket.hpp
index 0214c08..2686013 100644
--- a/include/PerfSocket.hpp
+++ b/include/PerfSocket.hpp
@@ -67,6 +67,8 @@ void SetSocketOptions(struct thread_Settings *inSettings);
 void SetSocketOptionsSendTimeout(struct thread_Settings *mSettings, int timer);
 void SetSocketOptionsReceiveTimeout(struct thread_Settings *mSettings, int timer);
 void SetSocketOptionsIPTos (struct thread_Settings *mSettings, int tos);
+void SetSocketTcpTxDelay (struct thread_Settings *mSettings, int delay);
+void SetSocketBindToDeviceIfNeeded (struct thread_Settings *inSettings);
 void setsock_tcp_mss(int inSock, int inMSS);
 int  getsock_tcp_mss(int inSock);
 #ifdef DEFAULT_PAYLOAD_LEN_PER_MTU_DISCOVERY
diff --git a/include/Reporter.h b/include/Reporter.h
index 6f97418..bb7ec6f 100644
--- a/include/Reporter.h
+++ b/include/Reporter.h
@@ -58,6 +58,7 @@
 #include "histogram.h"
 #include "packet_ring.h"
 #include "gettcpinfo.h"
+#include "payloads.h"
 
 // forward declarations found in Settings.hpp
 struct thread_Settings;
@@ -108,10 +109,24 @@ struct MeanMinMaxStats {
     intmax_t err;
 };
 
+struct ShiftIntCounter {
+    intmax_t current;
+    intmax_t prev;
+};
+
+struct ShiftUintCounter {
+    uintmax_t current;
+    uintmax_t prev;
+};
+
 #define TCPREADBINCOUNT 8
 struct ReadStats {
-    intmax_t cntRead;
-    intmax_t totcntRead;
+    uintmax_t cntRead;
+    uintmax_t cntReadTimeo;
+    uintmax_t cntReadErrLen;
+    struct ShiftUintCounter ReadCnt;
+    struct ShiftUintCounter ReadTimeoCnt;
+    struct ShiftUintCounter ReadErrLenCnt;
     int bins[TCPREADBINCOUNT];
     int totbins[TCPREADBINCOUNT];
     int binsize;
@@ -120,9 +135,10 @@ struct ReadStats {
 struct WriteStats {
     intmax_t WriteCnt;
     intmax_t WriteErr;
+    intmax_t WriteTimeo;
     intmax_t totWriteCnt;
     intmax_t totWriteErr;
-    intmax_t totWriteWarn;
+    intmax_t totWriteTimeo;
     struct iperf_tcpstats tcpstats;
 };
 
@@ -134,13 +150,6 @@ struct WriteStats {
 #define L2LENERR   0x02
 #define L2CSUMERR  0x04
 
-enum WriteErrType {
-    WriteNoErr  = 0,
-    WriteErrAccount,
-    WriteErrFatal,
-    WriteErrNoAccount
-};
-
 struct L2Stats {
     intmax_t cnt;
     intmax_t unknown;
@@ -165,7 +174,8 @@ enum ReportType {
     SUM_REPORT,
     SETTINGS_REPORT,
     CONNECTION_REPORT,
-    SERVER_RELAY_REPORT
+    SERVER_RELAY_REPORT,
+    STRING_REPORT
 };
 
 enum ReportSubType {
@@ -174,6 +184,11 @@ enum ReportSubType {
     TOTALSUM_REPORT
 };
 
+enum TansferIDType {
+    REVERSED = 1,
+    NORMAL
+};
+
 union SendReadStats {
     struct ReadStats read;
     struct WriteStats write;
@@ -186,10 +201,12 @@ struct ReportCommon {
     enum ThreadMode ThreadMode;
     enum ReportMode ReportMode;
     bool KeyCheck;
+    bool Omit;
     int flags;
     int flags_extend;
     int flags_extend2;
     int threads;
+    int working_load_threads;
     unsigned short Port;
     unsigned short PortLast;
     unsigned short BindPort;
@@ -212,6 +229,7 @@ struct ReportCommon {
 #endif
     int winsize_requested;
     unsigned int FQPacingRate;
+    int FQPacingRateStep;
     int HistBins;
     int HistBinsize;
     int HistUnits;
@@ -220,6 +238,8 @@ struct ReportCommon {
     Socklen_t size_peer;
     iperf_sockaddr local;
     Socklen_t size_local;
+    iperf_sockaddr multicast_group;
+    Socklen_t size_multicast_group;
     char* Host;                   // -c
     char* HideHost;
     char* Localhost;              // -B
@@ -227,13 +247,16 @@ struct ReportCommon {
     char* Ifrnametx;
     char* SSMMulticastStr;
     char* Congestion;
+    char* LoadCCA;
     char* transferIDStr;
     char* PermitKey;
     int transferID;
+    int peertransferID;
     double rtt_weight;
     double ListenerTimeout;
     double FPS;
     int bbsize;
+    int bbreplysize;
     int bbhold;
     int bbcount;
     int jitter_binwidth;
@@ -256,16 +279,7 @@ struct ConnectionInfo {
     char peerversion[PEERVERBUFSIZE];
     struct MeanMinMaxStats connect_times;
     struct iperf_tcpstats tcpinitstats;
-};
-
-struct ShiftIntCounter {
-    intmax_t current;
-    intmax_t prev;
-};
-
-struct ShiftUintCounter {
-    uintmax_t current;
-    uintmax_t prev;
+    char connected_cca[TCP_CCA_NAME_MAX];
 };
 
 struct ShiftCounters {
@@ -323,6 +337,7 @@ struct ReportTimeStamps {
     struct timeval packetTime;
     struct timeval prevpacketTime;
     struct timeval prevsendTime;
+    struct timeval prevTime;
     struct timeval nextTime;
     struct timeval intervalTime;
     struct timeval IPGstart;
@@ -333,8 +348,13 @@ struct TransferInfo {
     struct ReportCommon *common;
     struct ReportTimeStamps ts;
     void (*output_handler) (struct TransferInfo *stats);
+    struct SumReport *sumreport;
+    enum ReportType type;
+    enum edgeLevel uplevel;
+    enum edgeLevel downlevel;
     int groupID;
-    int threadcnt;
+    int slot_thread_upcount;  // increments on a thread's first sample into a report interval
+    int slot_thread_downcount; // increments on a thread's interval sum output
     bool isMaskOutput;
     uintmax_t cntBytes;
     intmax_t cntError;
@@ -354,10 +374,13 @@ struct TransferInfo {
     struct histogram *framelatency_histogram;
     struct RunningMMMStats frame; // isochronous frame or msg burst
     struct histogram *bbrtt_histogram;
+    struct histogram *bbowdto_histogram;
+    struct histogram *bbowdfro_histogram;
     struct RunningMMMStats bbrtt;
     struct RunningMMMStats bbowdto;
     struct RunningMMMStats bbowdfro;
     struct RunningMMMStats bbasym;
+    uintmax_t bb_clocksync_error;
     struct MeanMinMaxStats schedule_error;
     struct L2Stats l2counts;
     // Packet and frame state info
@@ -375,16 +398,18 @@ struct TransferInfo {
     double fInP;
     double iInPVar;
     double fInPVar;
+    intmax_t FQPacingRateCurrent;
+    int threadcnt_final;
 };
 
 struct SumReport {
     struct ReferenceMutex reference;
     int threads;
-    int threads_cntr_fsum;
+    int final_thread_upcount;
     struct TransferInfo info;
-    void (*transfer_protocol_sum_handler) (struct TransferInfo *stats, int final);
+    void (*transfer_protocol_sum_handler) (struct TransferInfo *stats, bool final);
     struct BarrierMutex fullduplex_barrier;
-    int sum_fd_set;
+    bool sum_fd_set;
     bool sum_reverse_set;
 };
 
@@ -392,8 +417,8 @@ struct ReporterData {
     // function pointer for per packet processing
     void (*packet_handler_pre_report) (struct ReporterData *data, struct ReportStruct *packet);
     void (*packet_handler_post_report) (struct ReporterData *data, struct ReportStruct *packet);
-    void (*transfer_protocol_handler) (struct ReporterData *data, int final);
-    int (*transfer_interval_handler) (struct ReporterData *data, struct ReportStruct *packet);
+    void (*transfer_protocol_handler) (struct ReporterData *data, bool final);
+    bool (*transfer_interval_handler) (struct ReporterData *data, struct ReportStruct *packet);
 
     struct PacketRing *packetring;
     int reporter_thread_suspends; // used to detect CPU bound systems
@@ -425,15 +450,16 @@ typedef void (* report_statistics)( struct TransferInfo* );
 typedef void (* report_serverstatistics)( struct ConnectionInfo *, struct TransferInfo* );
 
 void SetSumHandlers (struct thread_Settings *inSettings, struct SumReport* sumreport);
-struct SumReport* InitSumReport(struct thread_Settings *inSettings, int inID, int fullduplex);
+struct SumReport* InitSumReport(struct thread_Settings *inSettings, int inID, bool fullduplex);
 struct ReportHeader* InitIndividualReport(struct thread_Settings *inSettings);
 struct ReportHeader* InitConnectionReport(struct thread_Settings *inSettings);
 struct ConnectionInfo* InitConnectOnlyReport(struct thread_Settings *thread);
 struct ReportHeader *InitSettingsReport(struct thread_Settings *inSettings);
 struct ReportHeader* InitServerRelayUDPReport(struct thread_Settings *inSettings, struct server_hdr *server);
+struct ReportHeader* InitStringReport (char *textoutput);
 void PostReport(struct ReportHeader *reporthdr);
 bool ReportPacket (struct ReporterData* data, struct ReportStruct *packet);
-int EndJob(struct ReportHeader *reporthdr,  struct ReportStruct *packet);
+bool EndJob(struct ReportHeader *reporthdr,  struct ReportStruct *packet);
 void FreeReport(struct ReportHeader *reporthdr);
 void FreeSumReport (struct SumReport *sumreport);
 void FreeConnectionReport(struct ConnectionInfo *reporthdr);
@@ -469,34 +495,34 @@ void reporter_handle_packet_bb_server(struct ReporterData *data, struct ReportSt
 // Invoked from the Reporter thread per function vector this_ireport->transfer_interval_handler
 // This is set during Report instantiation (found in Reports.c)
 // These conditionally, e.g per a sample interval, invoke the transfer protocol handlers
-int reporter_condprint_time_interval_report(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_frame_interval_report_client_udp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_frame_interval_report_server_udp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_frame_interval_report_server_tcp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_frame_interval_report_client_tcp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_burst_interval_report_client_udp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_burst_interval_report_server_udp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_burst_interval_report_server_tcp(struct ReporterData *data, struct ReportStruct *packet);
-int reporter_condprint_burst_interval_report_client_tcp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_time_interval_report(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_frame_interval_report_client_udp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_frame_interval_report_server_udp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_frame_interval_report_server_tcp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_frame_interval_report_client_tcp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_burst_interval_report_client_udp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_burst_interval_report_server_udp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_burst_interval_report_server_tcp(struct ReporterData *data, struct ReportStruct *packet);
+bool reporter_condprint_burst_interval_report_client_tcp(struct ReporterData *data, struct ReportStruct *packet);
 
 // Each report type needs specialized updating prior to printing its report
 // These functions realize that updating. They are called by the transfer protocol handler,
 // e.g. (*data->transfer_protocol_handler)(data, 0) found in src/Reporter.c
 // These update the TransferInfo stats struct which is used during output (e.g. the output handler)
-void reporter_transfer_protocol_null(struct ReporterData *data, int final);
-void reporter_transfer_protocol_client_tcp(struct ReporterData *data, int final);
-void reporter_transfer_protocol_client_bb_tcp(struct ReporterData *data, int final);
-void reporter_transfer_protocol_client_udp(struct ReporterData *data, int final);
-void reporter_transfer_protocol_server_tcp(struct ReporterData *data, int final);
-void reporter_transfer_protocol_server_bb_tcp(struct ReporterData *data, int final);
-void reporter_transfer_protocol_server_udp(struct ReporterData *data, int final);
+void reporter_transfer_protocol_null(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_client_tcp(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_client_bb_tcp(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_client_udp(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_server_tcp(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_server_bb_tcp(struct ReporterData *data, bool final);
+void reporter_transfer_protocol_server_udp(struct ReporterData *data, bool final);
 // Function vectors to suppport sum reports
-void reporter_transfer_protocol_sum_client_tcp(struct TransferInfo *stats, int final);
-void reporter_transfer_protocol_sum_server_tcp(struct TransferInfo *stats, int final);
-void reporter_transfer_protocol_sum_client_udp(struct TransferInfo *stats, int final);
-void reporter_transfer_protocol_sum_server_udp(struct TransferInfo *stats, int final);
-void reporter_transfer_protocol_fullduplex_tcp(struct TransferInfo *stats, int final);
-void reporter_transfer_protocol_fullduplex_udp(struct TransferInfo *stats, int final);
+void reporter_transfer_protocol_sum_client_tcp(struct TransferInfo *stats, bool final);
+void reporter_transfer_protocol_sum_server_tcp(struct TransferInfo *stats, bool final);
+void reporter_transfer_protocol_sum_client_udp(struct TransferInfo *stats, bool final);
+void reporter_transfer_protocol_sum_server_udp(struct TransferInfo *stats, bool final);
+void reporter_transfer_protocol_fullduplex_tcp(struct TransferInfo *stats, bool final);
+void reporter_transfer_protocol_fullduplex_udp(struct TransferInfo *stats, bool final);
 
 // Report output print routines invoked by the transfer_protocol handler
 // Bound in Report instantiation and invoked by the transfer protocol
@@ -520,6 +546,7 @@ void tcp_output_burst_write(struct TransferInfo *stats);
 void tcp_output_sum_write(struct TransferInfo *stats);
 void tcp_output_sumcnt_write(struct TransferInfo *stats);
 void tcp_output_write_enhanced (struct TransferInfo *stats);
+void tcp_output_write_enhanced_fq (struct TransferInfo *stats);
 void tcp_output_write_enhanced_isoch (struct TransferInfo *stats);
 void tcp_output_sum_write_enhanced (struct TransferInfo *stats);
 void tcp_output_sumcnt_write_enhanced (struct TransferInfo *stats);
@@ -559,6 +586,7 @@ void udp_output_enhanced_csv(struct TransferInfo *stats);
 void tcp_output_basic_csv(struct TransferInfo *stats);
 void tcp_output_read_enhanced_csv(struct TransferInfo *stats);
 void tcp_output_write_enhanced_csv(struct TransferInfo *stats);
+void tcp_output_write_bb_csv (struct TransferInfo *stats);
 
 // The report output routines that are simpler and aren't related to stats
 void reporter_print_connection_report(struct ConnectionInfo *report);
@@ -571,10 +599,11 @@ void reporter_connect_printf_tcp_final(struct ConnectionInfo *report);
 
 void write_UDP_AckFIN(struct TransferInfo *stats, int len);
 
-int reporter_process_transfer_report (struct ReporterData *this_ireport);
-int reporter_process_report (struct ReportHeader *reporthdr);
+bool reporter_process_transfer_report (struct ReporterData *this_ireport);
+bool reporter_process_report (struct ReportHeader *reporthdr);
 
-void setTransferID(struct thread_Settings *inSettings, int role_reversal);
+void setTransferID(struct thread_Settings *inSettings, enum TansferIDType traffic_direction);
+void updateTransferIDPeer(struct thread_Settings *inSettings);
 void format_ips_port_string (struct TransferInfo *stats, bool sum);
 
 #ifdef __cplusplus
diff --git a/include/Server.hpp b/include/Server.hpp
index cca386a..ee38c3c 100644
--- a/include/Server.hpp
+++ b/include/Server.hpp
@@ -95,6 +95,7 @@ private:
     int SkipFirstPayload(void);
     void ClientReverseFirstRead(void);
     void PostNullEvent(void);
+    inline bool WriteBB(void);
     Timestamp connect_done;
     bool peerclose;
     bool isburst;
@@ -116,6 +117,9 @@ private:
     struct msghdr message;
     char ctrl[CMSG_SPACE(sizeof(struct timeval))];
     struct cmsghdr *cmsg;
+#if HAVE_DECL_MSG_CTRUNC
+    bool ctrunc_warn_enable;
+#endif
 #endif
 #if defined(HAVE_LINUX_FILTER_H) && defined(HAVE_AF_PACKET)
     struct ether_header *eth_hdr;
diff --git a/include/Settings.hpp b/include/Settings.hpp
index 250d3a7..6d87012 100644
--- a/include/Settings.hpp
+++ b/include/Settings.hpp
@@ -77,7 +77,7 @@ extern "C" {
 #define SMALLEST_INTERVAL_SEC 0.005 // 5ms
 #else
 #define SMALLEST_INTERVAL 100 // 100 usec
-#define SMALLEST_INTERVAL_SEC 0.0001 // 5ms
+#define SMALLEST_INTERVAL_SEC 0.0001
 #endif
 
 #define SLOPSECS 2
@@ -94,14 +94,14 @@ extern "C" {
 #define CSVPEERLIMIT ((REPORT_ADDRLEN * 2) + 40)
 #define NEARCONGEST_DEFAULT 0.5
 #define DEFAULT_PERMITKEY_LIFE 20.0 // units is seconds
-#define TESTEXCHANGETIMEOUT (4 * 1000000) // 4 secs, units is microseconds
+#define DEFAULT_TESTEXCHANGETIMEOUT (60 * 1000000) // 60 secs, units is microseconds
 #ifndef MAXTTL
 #define MAXTTL 255
 #endif
 #define DEFAULT_BOUNCEBACK_BYTES 100
 #define SMALL_WRITE_PREFETCH (1024 * 16)
 
-#define SHALLOW_COPY 1
+#define SHALLOW_COPY 1 // This is likley a bug, using a deep copy is safer but overkill
 #define DEEP_COPY 1
 // server/client mode
 enum ThreadMode {
@@ -178,11 +178,12 @@ struct thread_Settings {
     struct thread_Settings *runNow;
     struct thread_Settings *runNext;
     // int's
+    int sosndtimer;
     int mThreads;                   // -P
     int mTOS;                       // -S
     int mRTOS;                      // reflected TOS
     int mTransferID;
-    int mConnectRetries;
+    int mPeerTransferID;
 #if WIN32
     SOCKET mSock;
 #else
@@ -246,12 +247,16 @@ struct thread_Settings {
     Socklen_t size_peer;
     iperf_sockaddr local;
     Socklen_t size_local;
+    iperf_sockaddr multicast_group;
+    Socklen_t size_multicast_group;
+    iperf_sockaddr multicast_group_source;
     nthread_t mTID;
     int incrdstip;
     int incrsrcip;
     int incrsrcport;
     int connectonly_count;
     char* mCongestion;
+    char* mLoadCCA;
     int mHistBins;
     int mHistBinsize;
     int mHistUnits;
@@ -270,6 +275,11 @@ struct thread_Settings {
     int recvflags; // used to set recv flags,e.g. MSG_TRUNC with L
     double mVariance; //vbr variance
     uintmax_t mFQPacingRate;
+#if (HAVE_DECL_SO_MAX_PACING_RATE)
+    int mFQPacingRateStep;
+    uintmax_t mFQPacingRateCurrent;
+    double mFQPacingRateStepInterval;
+#endif
     struct timeval txholdback_timer;
     struct timeval txstart_epoch;
     struct timeval accept_time;
@@ -280,7 +290,7 @@ struct thread_Settings {
     int numreportstructs;
     int32_t peer_version_u;
     int32_t peer_version_l;
-    double connecttime;
+    long barrier_time; // wait in units of microseconds
     double rtt_nearcongest_weight_factor;
     char mPermitKey[MAX_PERMITKEY_LEN + 1]; //add some space for timestamp
     struct timeval mPermitKeyTime;
@@ -289,6 +299,7 @@ struct thread_Settings {
     int tuntapdev;
     int firstreadbytes;
     int mBounceBackBytes;
+    int mBounceBackReplyBytes;
     int mBounceBackBurst;
     int mWorkingLoadThreads; // number of congest threads
     uint32_t mBounceBackHold; // units of usecs
@@ -300,6 +311,15 @@ struct thread_Settings {
     int mWritePrefetch;
 #endif
     int jitter_binwidth;
+#if HAVE_DECL_TCP_TX_DELAY
+    float mTcpTxDelayMean;
+    float mTcpTxDelayProb;
+#endif
+    bool mOmit;
+    int sendfirst_pacing;
+    double connecttime;
+    double connect_retry_time;    // units in seconds
+    unsigned int connect_retry_timer; // units in usecs
 };
 
 /*
@@ -398,6 +418,14 @@ struct thread_Settings {
 #define FLAG_DOMAINV4          0x00008000
 #define FLAG_JITTER_HISTOGRAM  0x00010000
 #define FLAG_UTC            0x00020000
+#define FLAG_LOAD_CCA       0x00040000
+#define FLAG_BURSTSIZE      0x00080000
+#define FLAG_TCPTXDELAY     0x00100000
+#define FLAG_FQPACINGSTEP     0x00200000
+#define FLAG_FQPACINGSTEPINTERVAL 0x00400000
+#define FLAG_SYNCTRANSFERID 0x00800000
+#define FLAG_IGNORESHUTDOWN  0x01000000
+#define FLAG_SETTOS          0x02000000
 
 #define isBuflenSet(settings)      ((settings->flags & FLAG_BUFLENSET) != 0)
 #define isCompat(settings)         ((settings->flags & FLAG_COMPAT) != 0)
@@ -476,6 +504,14 @@ struct thread_Settings {
 #define isWorkingLoadDown(settings)  ((settings->flags_extend2 & FLAG_WORKING_LOAD_DOWN) != 0)
 #define isJitterHistogram(settings)  ((settings->flags_extend2 & FLAG_JITTER_HISTOGRAM) != 0)
 #define isUTC(settings)            ((settings->flags_extend2 & FLAG_UTC) != 0)
+#define isLoadCCA(settings) ((settings->flags_extend2 & FLAG_LOAD_CCA) != 0)
+#define isBurstSize(settings)      ((settings->flags_extend2 & FLAG_BURSTSIZE) != 0)
+#define isTcpTxDelay(settings)     ((settings->flags_extend2 & FLAG_TCPTXDELAY) != 0)
+#define isFQPacingStep(settings)       ((settings->flags_extend2 & FLAG_FQPACINGSTEP) != 0)
+#define isFQPacingStepInterval(settings) ((settings->flags_extend2 & FLAG_FQPACINGSTEPINTERVAL) != 0)
+#define isSyncTransferID(settings) ((settings->flags_extend2 & FLAG_SYNCTRANSFERID) != 0)
+#define isIgnoreShutdown(settings) ((settings->flags_extend2 & FLAG_IGNORESHUTDOWN) != 0)
+#define isSetTOS(settings)         ((settings->flags_extend2 & FLAG_SETTOS) != 0)
 
 #define setBuflenSet(settings)     settings->flags |= FLAG_BUFLENSET
 #define setCompat(settings)        settings->flags |= FLAG_COMPAT
@@ -551,6 +587,14 @@ struct thread_Settings {
 #define setWorkingLoadDown(settings) settings->flags_extend2 |= FLAG_WORKING_LOAD_DOWN
 #define setJitterHistogram(settings) settings->flags_extend2 |= FLAG_JITTER_HISTOGRAM
 #define setUTC(settings)           settings->flags_extend2 |= FLAG_UTC
+#define setLoadCCA(settings)    settings->flags_extend2 |= FLAG_LOAD_CCA
+#define setBurstSize(settings)     settings->flags_extend2 |= FLAG_BURSTSIZE
+#define setTcpTxDelay(settings)    settings->flags_extend2 |= FLAG_TCPTXDELAY
+#define setFQPacingStep(settings) settings->flags_extend2 |= FLAG_FQPACINGSTEP
+#define setFQPacingStepInterval(settings) settings->flags_extend2 |= FLAG_FQPACINGSTEPINTERVAL
+#define setSyncTransferID(settings) settings->flags_extend2 |= FLAG_SYNCTRANSFERID
+#define setIgnoreShutdown(settings) settings->flags_extend2 |= FLAG_IGNORESHUTDOWN
+#define setSetTOS(settings)         settings->flags_extend2 |= FLAG_SETTOS
 
 #define unsetBuflenSet(settings)   settings->flags &= ~FLAG_BUFLENSET
 #define unsetCompat(settings)      settings->flags &= ~FLAG_COMPAT
@@ -625,6 +669,14 @@ struct thread_Settings {
 #define unsetWorkingLoadDown(settings) settings->flags_extend2 &= ~FLAG_WORKING_LOAD_DOWN
 #define unsetJitterHistogram(settings) settings->flags_extend2 &= ~FLAG_JITTER_HISTOGRAM
 #define unsetUTC(settings)           settings->flags_extend2 &= ~FLAG_UTC
+#define unsetLoadCCA(settings)       settings->flags_extend2 &= ~FLAG_LOAD_CCA
+#define unsetBurstSize(settings)     settings->flags_extend2 &= ~FLAG_BURSTSIZE
+#define unsetTcpTxDelay(settings)    settings->flags_extend2 &= ~FLAG_TCPTXDELAY
+#define unsetFQPacingStep(settings)  settings->flags_extend2 &= ~FLAG_FQPACINGSTEP
+#define unsetFQPacingStepInterval(settings) settings->flags_extend2 &= ~FLAG_FQPACINGSTEPINTERVAL
+#define unsetSyncTransferID(settings) settings->flags_extend2 &= ~FLAG_SYNCTRANSFERID
+#define unsetIgnoreShutdown(settings) settings->flags_extend2 &= ~FLAG_IGNORESHUTDOWN
+#define unsetSetTOS(settings)         settings->flags_extend2 &= ~FLAG_SETTOS
 
 // set to defaults
 void Settings_Initialize(struct thread_Settings* main);
@@ -632,6 +684,9 @@ void Settings_Initialize(struct thread_Settings* main);
 // copy structure
 void Settings_Copy(struct thread_Settings* from, struct thread_Settings** into, int copyall);
 
+// grow mBuf
+void Settings_Grow_mBuf(struct thread_Settings *mSettings, int newsize);
+
 // free associated memory
 void Settings_Destroy(struct thread_Settings *mSettings);
 
diff --git a/include/SocketAddr.h b/include/SocketAddr.h
index be391a8..c18c059 100644
--- a/include/SocketAddr.h
+++ b/include/SocketAddr.h
@@ -60,11 +60,13 @@
 extern "C" {
 #endif
 /* ------------------------------------------------------------------- */
+int SockAddr_getAFdomain(iperf_sockaddr *inSockAddr);
 void SockAddr_localAddr(struct thread_Settings *inSettings);
 void SockAddr_remoteAddr(struct thread_Settings *inSettings);
 
 void SockAddr_setHostname(const char* inHostname,
                            iperf_sockaddr *inSockAddr,
+                           Socklen_t *addr_size,
                            int isIPv6);  // DNS lookup
 void SockAddr_getHostname(iperf_sockaddr *inSockAddr,
                            char* outHostname,
diff --git a/include/Timestamp.hpp b/include/Timestamp.hpp
index e0c9f29..044a145 100644
--- a/include/Timestamp.hpp
+++ b/include/Timestamp.hpp
@@ -83,7 +83,7 @@ public:
     /* -------------------------------------------------------------------
      * Create a timestamp, with the given seconds/microseconds
      * ------------------------------------------------------------------- */
-    Timestamp(long sec, long usec) {
+    Timestamp(time_t sec, long usec) {
         set(sec, usec);
     }
 
@@ -111,7 +111,7 @@ public:
     /* -------------------------------------------------------------------
      * Set timestamp to the given seconds/microseconds
      * ------------------------------------------------------------------- */
-    void set(long sec, long usec) {
+    void set(time_t sec, long usec) {
         assert(sec  >= 0);
         assert(usec >= 0  &&  usec < kMillion);
 
@@ -123,14 +123,14 @@ public:
      * Set timestamp to the given seconds
      * ------------------------------------------------------------------- */
     void set(double sec) {
-        mTime.tv_sec  = (long) sec;
+        mTime.tv_sec  = (time_t) sec;
         mTime.tv_usec = (long) ((sec - mTime.tv_sec) * kMillion);
     }
 
     /* -------------------------------------------------------------------
      * return seconds portion of timestamp
      * ------------------------------------------------------------------- */
-    long inline getSecs(void) {
+    time_t inline getSecs(void) {
         return mTime.tv_sec;
     }
 
@@ -240,8 +240,8 @@ public:
      * TODO optimize?
      * ------------------------------------------------------------------- */
     void add(double sec) {
-        mTime.tv_sec  += (long) sec;
-        mTime.tv_usec += (long) ((sec - ((long) sec)) * kMillion);
+        mTime.tv_sec  += (time_t) sec;
+        mTime.tv_usec += (long) ((sec - ((time_t) sec)) * kMillion);
 
         // watch for overflow
         if (mTime.tv_usec >= kMillion) {
diff --git a/include/active_hosts.h b/include/active_hosts.h
index 9635032..757d15c 100644
--- a/include/active_hosts.h
+++ b/include/active_hosts.h
@@ -66,8 +66,10 @@
  */
 struct Iperf_ListEntry {
     iperf_sockaddr host;
-    struct SumReport *sum_report;
+    iperf_sockaddr multicast;
+    struct SumReport *sumreport;
     int thread_count;
+    int gid;
 #if WIN32
     SOCKET socket;
 #else
@@ -78,10 +80,13 @@ struct Iperf_ListEntry {
 
 struct Iperf_Table {
     Mutex my_mutex;
-    struct Iperf_ListEntry *root;
-    int count;
-    int total_count;
+    struct Iperf_ListEntry *sum_root;
+    struct Iperf_ListEntry *flow_root;
     int groupid;
+#if HAVE_THREAD_DEBUG
+    int sum_count;
+    int flow_count;
+#endif
 };
 
 /*
@@ -89,7 +94,6 @@ struct Iperf_Table {
  */
 void Iperf_initialize_active_table (void);
 void Iperf_destroy_active_table (void);
-int Iperf_push_host (struct thread_Settings *agent);
-int Iperf_push_host_port_conditional (struct thread_Settings *agent);
+bool Iperf_push_host (struct thread_Settings *agent);
 void Iperf_remove_host (struct thread_Settings *agent);
 #endif
diff --git a/include/dscp.h b/include/dscp.h
index fe5486e..79d779b 100644
--- a/include/dscp.h
+++ b/include/dscp.h
@@ -44,11 +44,27 @@
  * by Robert J. McMahon (rjmcmahon@rjmcmahon.com, bob.mcmahon@broadcom.com)
  * ------------------------------------------------------------------- */
 
+/*
+ *      7  6  5  4  3  2  1  0
+ *    +--+--+--+--+--+--+--+--+
+ *    |     DSCP        | ECN |
+ *    +--+--+--+--+--+--+--+--+
+ *
+ */
+
 #ifndef DSCP_H
 #define DSCP_H
 #ifdef __cplusplus
 extern "C" {
 #endif
+
+// DSCP bits
+#define DSCP_SHIFT 2
+#define DSCP_BITMASK 0xFC
+#define ECN_BITMASK 0x03
+#define DSCP_VALUE(value) (int)((value & DSCP_BITMASK) >> DSCP_SHIFT)
+#define ECN_VALUE(value) (int)(value & ECN_BITMASK)
+
 int parse_ipqos(const char *cp);
 const char * iptos2str(int iptos);
 #ifdef __cplusplus
diff --git a/include/gettcpinfo.h b/include/gettcpinfo.h
index 995ee20..4eee092 100644
--- a/include/gettcpinfo.h
+++ b/include/gettcpinfo.h
@@ -50,21 +50,28 @@
 
 #include "headers.h"
 
+#define HAVE_TCP_INFLIGHT (HAVE_TCP_STATS && HAVE_STRUCT_TCP_INFO_TCPI_DSACK_DUPS && \
+			   HAVE_STRUCT_TCP_INFO_TCPI_LOST && HAVE_STRUCT_TCP_INFO_TCPI_SACKED \
+			   && HAVE_STRUCT_TCP_INFO_TCPI_UNACKED)
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct iperf_tcpstats {
     bool isValid;
-    int rtt;
+    uint32_t rtt;
     double connecttime;
 #if HAVE_TCP_STATS
-    int cwnd;
-    int rttvar;
-    intmax_t retry;
-    intmax_t retry_prev;
-    intmax_t retry_tot;
-    int mss_negotiated;
+    intmax_t cwnd;
+    uint32_t cwnd_packets;
+    uint32_t rttvar;
+    uint32_t retry;
+    uint32_t retry_prev;
+    uint32_t retry_tot;
+    uint32_t mss_negotiated;
+    uint32_t packets_in_flight;
+    intmax_t bytes_in_flight;
 #endif
 };
 
diff --git a/include/headers.h b/include/headers.h
index 6fb499d..891c46d 100644
--- a/include/headers.h
+++ b/include/headers.h
@@ -248,7 +248,9 @@ SPECIAL_OSF1_EXTERN_C_START
     #include <arpa/inet.h>   /* netinet/in.h must be before this on SunOS */
 SPECIAL_OSF1_EXTERN_C_STOP
 #endif
-#if HAVE_NETINET_TCP_H
+#if HAVE_LINUX_TCP_H && HAVE_DECL_TCP_TX_DELAY
+#include <linux/tcp.h>
+#elif HAVE_NETINET_TCP_H
 #include <netinet/tcp.h>
 #endif
 
diff --git a/include/histogram.h b/include/histogram.h
index 2cd7b3b..4dee5f8 100644
--- a/include/histogram.h
+++ b/include/histogram.h
@@ -1,5 +1,5 @@
 /*---------------------------------------------------------------
- * Copyrighta (c) 2017
+ * Copyright (c) 1999,2000,2001,2002,2003,2023
  * Broadcom Corporation
  * All Rights Reserved.
  *---------------------------------------------------------------
@@ -54,7 +54,8 @@ struct histogram {
     unsigned int bincount;
     unsigned int binwidth;
     unsigned int populationcnt;
-    int final;
+    bool Omit;
+    bool final;
     int maxbin;
     int fmaxbin;
     double maxval;
@@ -73,7 +74,7 @@ struct histogram {
 };
 
 extern struct histogram *histogram_init(unsigned int bincount, unsigned int binwidth, float offset,\
-				   float units, double ci_lower, double ci_upper, unsigned int id, char *name);
+					float units, double ci_lower, double ci_upper, unsigned int id, char *name, bool omit);
 extern void histogram_delete(struct histogram *h);
 extern int histogram_insert(struct histogram *h, float value, struct timeval *ts);
 extern void histogram_clear(struct histogram *h);
diff --git a/include/iperf_multicast_api.h b/include/iperf_multicast_api.h
new file mode 100644
index 0000000..4844598
--- /dev/null
+++ b/include/iperf_multicast_api.h
@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------
+ * Copyright (c) 2023
+ * Broadcom Corporation
+ * All Rights Reserved.
+ *---------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute,
+ * sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ *
+ * Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and
+ * the following disclaimers.
+ *
+ *
+ * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimers in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ *
+ * Neither the name of Broadcom Coporation,
+ * nor the names of its contributors may be used to endorse
+ * or promote products derived from this Software without
+ * specific prior written permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE CONTIBUTORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * ________________________________________________________________
+ *
+ * multicast_join.h
+ * pull out multicast join code for maitainability
+ *
+ * by Robert J. McMahon (rjmcmahon@rjmcmahon.com, bob.mcmahon@broadcom.com)
+ * -------------------------------------------------------------------
+ */
+#ifndef MULTICASTJOIN_H
+#define MULTICASTJOIN_H
+
+#include "headers.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum McastJoinResponse {
+    IPERF_MULTICAST_JOIN_SUCCESS = 1,
+    IPERF_MULTICAST_JOIN_FAIL,
+    IPERF_MULTICAST_JOIN_UNSUPPORTED
+};
+
+#define IPERF_MULTICAST_JOIN_SUCCESS 1
+
+enum McastJoinResponse iperf_multicast_join (struct thread_Settings *inSettings);
+bool iperf_multicast_sendif_v4 (struct thread_Settings *inSettings);
+bool iperf_multicast_sendif_v6 (struct thread_Settings *inSettings);
+
+#ifdef __cplusplus
+} /* end extern "C" */
+#endif
+
+#endif // MULTICASTJOIN_H
diff --git a/include/isochronous.hpp b/include/isochronous.hpp
index 7fb5a92..65609cd 100644
--- a/include/isochronous.hpp
+++ b/include/isochronous.hpp
@@ -63,7 +63,7 @@ namespace Isochronous {
 	unsigned int get(long *);
 	unsigned int get(const Timestamp&) const;
 	unsigned int period_us(void);
-	unsigned int wait_tick(long *sched_err);
+        unsigned int wait_tick(long *sched_err, bool sync_strict);
 	unsigned int wait_sync(long sec, long usec);
 	long getSecs(void);
 	long getUsecs(void);
diff --git a/include/packet_ring.h b/include/packet_ring.h
index 3ac6b91..d97a64d 100644
--- a/include/packet_ring.h
+++ b/include/packet_ring.h
@@ -57,6 +57,26 @@ extern "C" {
 
 #define ACKRING_DEFAULTSIZE 100
 
+enum ReadWriteExtReturnVals {
+    ReadSuccess  = 1,
+    ReadTimeo,
+    ReadTimeoFatal,
+    ReadErrLen,
+    ReadNoAccount,
+    WriteSuccess,
+    WriteSelectRetry,
+    WriteErrAccount,
+    WriteErrFatal,
+    WriteTimeo,
+    WriteNoAccount,
+    NullEvent
+};
+
+enum edgeLevel {
+    LOW = 0,
+    HIGH = 1
+};
+
 struct ReportStruct {
     intmax_t packetID;
     intmax_t packetLen;
@@ -64,8 +84,8 @@ struct ReportStruct {
     struct timeval prevPacketTime;
     struct timeval sentTime;
     struct timeval prevSentTime;
-    int errwrite;
-    int emptyreport;
+    enum ReadWriteExtReturnVals err_readwrite;
+    bool emptyreport;
     int l2errors;
     int l2len;
     int expected_l2len;
@@ -77,7 +97,7 @@ struct ReportStruct {
     uint32_t burstsize;
     uint32_t burstperiod;
     uint32_t remaining;
-    int transit_ready;
+    bool transit_ready;
     int writecnt;
     long write_time;
     bool scheduled;
@@ -85,6 +105,9 @@ struct ReportStruct {
     struct timeval sentTimeRX;
     struct timeval sentTimeTX;
     struct iperf_tcpstats tcpstats;
+#if defined(HAVE_DECL_SO_MAX_PACING_RATE)
+    intmax_t FQPacingRate;
+#endif
 };
 
 struct PacketRing {
@@ -94,10 +117,12 @@ struct PacketRing {
     int producer;
     int consumer;
     int maxcount;
-    int consumerdone;
+    bool consumerdone;
     int awaitcounter;
-    int mutex_enable;
+    bool mutex_enable;
     int bytes;
+    enum edgeLevel uplevel;
+    enum edgeLevel downlevel;
 
     // Use a condition variables
     // o) awake_producer - producer waits for the consumer thread to
@@ -116,6 +141,7 @@ extern void enqueue_ackring(struct PacketRing *pr, struct ReportStruct *metapack
 extern struct ReportStruct *dequeue_ackring(struct PacketRing * pr);
 extern void packetring_free(struct PacketRing *pr);
 extern void free_ackring(struct PacketRing *pr);
+extern enum edgeLevel toggleLevel(enum edgeLevel level);
 #ifdef HAVE_THREAD_DEBUG
 extern int packetring_getcount(struct PacketRing *pr);
 #endif
diff --git a/include/payloads.h b/include/payloads.h
index 30080d9..d1eb59b 100644
--- a/include/payloads.h
+++ b/include/payloads.h
@@ -56,6 +56,7 @@ extern "C" {
  */
 
 #define TAPBYTESSLOP 512
+#define TCP_CCA_NAME_MAX 32
 
 /* key for permit */
 #define HEADER_KEYCHECK     0x10000000
@@ -88,15 +89,22 @@ extern "C" {
 #define RUN_NOW              0x00000001
 #define HEADER16_SMALL_TRIPTIMES 0x0002 // use is 16 bits and not 32 bits
 
+// Num threads highest bit indicates sync id
+#define HEADER_HASTRANSFERID   0x80000000
+#define HEADER_TRANSFERIDMASK  0xFFF00000
+#define HEADER_TRANSFERIDSHIFT 20
+
 // Bounceback flag
 #define HEADER_BBQUICKACK    0x8000
 #define HEADER_BBCLOCKSYNCED 0x4000 // used in the bb header only
 #define HEADER_BBTOS         0x2000
 #define HEADER_BBSTOP        0x1000
+#define HEADER_BBREPLYSIZE   0x0800
 
 // newer flags available per HEADER_EXTEND
 // Below flags are used to pass test settings in *every* UDP packet
 // and not just during the header exchange
+// upper flags (16 bit)
 #define HEADER_ISOCH          0x0001
 #define HEADER_L2ETHPIPV6     0x0002
 #define HEADER_L2LENCHECK     0x0004
@@ -114,9 +122,14 @@ extern "C" {
 #define HEADER_WRITEPREFETCH  0x4000
 #define HEADER_TCPQUICKACK    0x8000
 
+// lower flags (16 bit)
+#define HEADER_CCA          0x8000
+#define HEADER_BARRIER_TIME 0x4000
+
 // later features
 #define HDRXACKMAX 2500000 // default 2.5 seconds, units microseconds
 #define HDRXACKMIN   10000 // default 10 ms, units microsecond
+
 /*
  * Structures used for test messages which
  * are exchanged between the client and the Server/Listener
@@ -230,6 +243,7 @@ struct bounceback_hdr {
     uint32_t bbhold; // up to here is mandatory
     uint32_t bbrtt;
     struct bb_ts bbread_ts;
+    uint32_t bbreplysize;
 };
 
 struct client_hdrext_isoch_settings {
@@ -255,10 +269,10 @@ struct client_hdrext {
     uint32_t lRate;
     uint32_t uRate;
     uint32_t TCPWritePrefetch;
+    uint32_t barrier_usecs;
 };
 
 struct client_hdrext_starttime_fq {
-    uint32_t reserved;
     uint32_t start_tv_sec;
     uint32_t start_tv_usec;
     uint32_t fqratel;
@@ -440,6 +454,11 @@ struct isoch_payload {
     uint32_t reserved;
 };
 
+struct cca_field {
+    uint16_t cca_length;
+    char value[TCP_CCA_NAME_MAX];
+};
+
 struct permitKey {
     uint16_t length;
     char value[MAX_PERMITKEY_LEN];
@@ -457,6 +476,8 @@ struct client_udp_testhdr {
 struct client_udpsmall_testhdr {
     struct UDP_datagram seqno_ts;
     uint16_t flags;
+    uint32_t start_tv_sec;
+    uint32_t start_tv_usec;
 };
 
 struct client_hdr_ack_ts {
@@ -513,7 +534,7 @@ struct client_hdr_ack {
  *                +--------+--------+--------+--------+
  *            15  |        write prefetch   (0.13)    |
  *                +--------+--------+--------+--------+
- *            16  |        reserved (0.14 start)      |
+ *            16  |        barrier wait (usecs)       |
  *                +--------+--------+--------+--------+
  *            17  |        start tv_sec (0.14)        |
  *                +--------+--------+--------+--------+
@@ -539,12 +560,15 @@ struct client_hdr_ack {
  *                +--------+--------+--------+--------+
  *            28  |        BurstIPG                   |
  *                +--------+--------+--------+--------+
+ *            29  | CCAS len value ...
+ *                +--------+--------+--------+--------+
  */
 struct client_tcp_testhdr {
     struct client_hdr_v1 base;
     struct client_hdrext extend;
     struct client_hdrext_starttime_fq start_fq;
     struct client_hdrext_isoch_settings isoch_settings;
+    struct cca_field cca;
     struct permitKey permitkey;
 };
 
@@ -614,7 +638,7 @@ struct server_hdr {
 #define SIZEOF_TCPHDRMSG_V1 (sizeof(struct client_hdr_v1))
 #define SIZEOF_TCPHDRMSG_EXT (sizeof(struct client_tcp_testhdr))
 #define MINMBUFALLOCSIZE (int) (sizeof(struct client_tcp_testhdr)) + TAPBYTESSLOP
-#define MINTRIPTIMEPLAYOAD (int) (sizeof(struct client_udp_testhdr) - sizeof(struct client_hdrext_isoch_settings))
+#define MINTRIPTIMEPAYLOAD (int) (sizeof(struct client_udp_testhdr) - sizeof(struct client_hdrext_isoch_settings))
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/include/util.h b/include/util.h
index 0f28ce4..db0e66f 100644
--- a/include/util.h
+++ b/include/util.h
@@ -63,6 +63,8 @@ extern "C" {
 
 extern int sInterupted;
 
+#define WARNBUFSIZE 256
+
 /* -------------------------------------------------------------------
  * set/getsockopt wrappers for SO_RCVBUF and SO_SNDBUF; TCP_MAXSEG
  * socket.c
@@ -119,6 +121,7 @@ BOOL WINAPI sig_dispatcher(DWORD type);
  * ------------------------------------------------------------------- */
 void warn      (const char *inMessage, const char *inFile, int inLine);
 void warn_errno(const char *inMessage, const char *inFile, int inLine);
+int errno_decode (char *decoded_text, size_t len);
 
 #define FAIL_exit(cond, msg)       \
   do {                                          \
@@ -192,12 +195,27 @@ double bitorbyte_atof(const char *inString);
 intmax_t byte_atoi(const char  *inString);
 uintmax_t bitorbyte_atoi(const char *inString);
 void byte_snprintf(char* outString, int inLen, double inNum, char inFormat);
-
+void make_lower(char *s);
 /*
  * Time macros for C-code (not the include Timestamp.hpp)
  */
 #define rMillion 1000000
 
+
+#ifdef HAVE_CLOCK_GETTIME
+#define TimeGetNow(timeval) do { \
+    struct timespec t1;			\
+    clock_gettime(CLOCK_REALTIME, &t1); \
+    timeval.tv_sec  = t1.tv_sec; \
+    timeval.tv_usec = t1.tv_nsec / 1000; \
+} while (0)
+#else
+#define TimeGetNow(timeval) do { \
+    gettimeofday(&timeval, NULL); \
+} while (0)
+#endif
+
+
 #define TimeZero(timeval) ((timeval.tv_sec == 0) && (timeval.tv_usec == 0))
 
 #define TimeDifference(left, right) ((left.tv_sec  - right.tv_sec) +	\
@@ -217,6 +235,16 @@ void byte_snprintf(char* outString, int inLen, double inNum, char inFormat);
                                     left.tv_sec += right.tv_sec;        \
                                 } while (0)
 
+#define TimeAddIntUsec(left, right)  do {                                    \
+                                    left.tv_usec += right % 1000000;  \
+                                    if (left.tv_usec > rMillion) {    \
+                                        left.tv_usec -= rMillion;       \
+                                        left.tv_sec++;                  \
+                                    }                                   \
+                                    left.tv_sec += right / 1000000;  \
+                                } while (0)
+
+
 /* -------------------------------------------------------------------
  * redirect the stdout to a specified file
  * stdio.c
@@ -260,6 +288,8 @@ void redirect(const char *inOutputFileName);
 #define NONFATALTCPWRITERR(errno) ((errno = WSAGetLastError()) == WSAETIMEDOUT)
 #define FATALUDPWRITERR(errno)  (((errno = WSAGetLastError()) != WSAETIMEDOUT) \
 				 && (errno != WSAECONNREFUSED))
+#define FATALTCPCONNECTERR(errno)  (((errno = WSAGetLastError()) != WSAETIMEDOUT) && \
+				    (errno != WSAECONNREFUSED) && (errno != WSAEWOULDBLOCK))
 #else
 #define FATALTCPREADERR(errno) ((errno != EAGAIN) && (errno != EWOULDBLOCK) && (errno != EINTR))
 #define FATALUDPREADERR(errno) ((errno != EAGAIN) && (errno != EWOULDBLOCK) && \
@@ -268,11 +298,11 @@ void redirect(const char *inOutputFileName);
 #define NONFATALTCPWRITERR(errno)  (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
 #define FATALUDPWRITERR(errno) 	((errno != EAGAIN) && (errno != EWOULDBLOCK) && (errno != EINTR) \
 				 && (errno != ECONNREFUSED) && (errno != ENOBUFS))
+#define FATALTCPCONNECTERR(errno)  ((errno != EAGAIN) && (errno != EWOULDBLOCK) && (errno != EINTR) \
+				   && (errno != ECONNREFUSED))
 #endif
 
-#ifdef WIN32
-#else
-#endif
+#define IPERF_SOCKET_ERROR_NONFATAL   -2
 
 #ifdef __cplusplus
 } /* end extern "C" */
diff --git a/include/version.h b/include/version.h
index d28d4b0..261d3e0 100644
--- a/include/version.h
+++ b/include/version.h
@@ -1,8 +1,9 @@
 #include "config.h"
 #define IPERF_VERSION VERSION
-#define IPERF_VERSION_DATE "14 March 2023"
-#define IPERF_VERSION_MAJORHEX 0x00020001
-#define IPERF_VERSION_MINORHEX 0x00090003
+#define IPERF_VERSION_DATE "10 April 2024"
+#define IPERF_VERSION_MAJORHEX 0x00020002
+#define IPERF_VERSION_MINORHEX 0x00000003
+#define IPERF_BRANCH ""
 
 /*
  * 	case 0: