Index: src/kernel_coalesce.c =================================================================== --- src/kernel_coalesce.c (revision 0) +++ src/kernel_coalesce.c (revision 0) @@ -0,0 +1,308 @@ +/* + * This contains functions used by ndt to disable disable packet + * coalescing on network interfaces. This uses the ETHTOOL API to + * access and modify coalescing settings. + * + * disable_coalesce is intended to be called first followed by + * revert_coalesce to undo any changes made by disable_coalesce. + * + * Author: Richard Sanger + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../config.h" +#include "logging.h" + +#define RX_ON 0x1 +#define RX_IRQ_ON 0x2 +#define TX_ON 0x4 +#define TX_IRQ_ON 0x8 +#define RX_ADAPT_ON 0x10 +#define TX_ADAPT_ON 0x20 + + +/* We need to keep a copy of the ethtool_coalescing for every interface + * that we change. This is the number of coal_settings data structures + * we can keep for the purpose to later revert. 6 seems plenty + * since 'lo' isn't included nor any other interface without coalescing + * support. + */ +#define MAX_IFACES 6 +#define IS_DISABLED(a,b) ((a == 0 || a==1) && (b==0 || b == 1)) + +/* Keep the original coalescing data so we can switch back to this */ +static struct coal_settings{ + /* The original coal data */ + struct ethtool_coalesce coal; + /* The interfaces name, if empty we don't revert this one */ + char name[IFNAMSIZ]; +} eth_coal_org[MAX_IFACES]; + +/** + * Check if coalescing is already off and fill in the new structure as + * required. See linux/ethtool.h header for discriptions of each field + * Note: we cannot set both coalesce_usecs and coalesce_max_frame to 0 + * this is because 0 means to ignore this setting and only use the other + * no 0 one. + * + * @param org (in) - the original coalesce data + * @param new (out) - the new coalesce data which will have all + * coalescing turned off upon return. + * + * @return flags indicating which settings have been changed in 'new'. + * If return value is 0 then coalescing is already off and no changes + * to new have been made. + */ +static __u32 check_disable_coalesce(const struct ethtool_coalesce * org, + struct ethtool_coalesce * new){ + __u32 flags = 0; + + assert(org); + assert(new); + + /* Copy across the orginal settings into new settings */ + memcpy(new, org, sizeof(struct ethtool_coalesce)); + + log_println(6, "--rx_coalesce_usecs : %d\n--rx_max_coalesced_frames : %d", + org->rx_coalesce_usecs, org->rx_max_coalesced_frames); + if(IS_DISABLED(org->rx_coalesce_usecs, org->rx_max_coalesced_frames)){ + log_println(6, " RX coalescing is already disabled"); + } else { + flags |= RX_ON; + new->rx_coalesce_usecs = 0; + new->rx_max_coalesced_frames = 1; + log_println(6, " ndt will attempt to disable RX coalescing"); + } + + log_println(6, "--rx_coalesce_usecs_irq : %d\n--rx_max_coalesced_frames_irq : %d", + org->rx_coalesce_usecs_irq, org->rx_max_coalesced_frames_irq); + if(IS_DISABLED(org->rx_coalesce_usecs_irq, org->rx_max_coalesced_frames_irq)){ + log_println(6, " RX irq coalescing is already disabled"); + } else { + flags |= RX_IRQ_ON; + new->rx_coalesce_usecs_irq = 0; + new->rx_max_coalesced_frames_irq = 1; + log_println(6, " ndt will attempt to disable RX irq coalescing"); + } + + log_println(6, "--tx_coalesce_usecs : %d\n--tx_max_coalesced_frames : %d", + org->tx_coalesce_usecs, org->tx_max_coalesced_frames); + if(IS_DISABLED(org->tx_coalesce_usecs, org->tx_max_coalesced_frames)){ + log_println(6, " TX coalescing is already disabled"); + } else { + flags |= TX_ON; + new->tx_coalesce_usecs = 0; + new->tx_max_coalesced_frames = 1; + log_println(6, " ndt will attempt to disable TX coalescing"); + } + + log_println(6, "--tx_coalesce_usecs_irq : %d\n--tx_max_coalesced_frames_irq : %d", + org->tx_coalesce_usecs_irq, org->tx_max_coalesced_frames_irq); + if(IS_DISABLED(org->tx_coalesce_usecs_irq, org->tx_max_coalesced_frames_irq)){ + log_println(6, " TX irq coalescing is already disabled"); + } else { + flags |= TX_IRQ_ON; + new->tx_coalesce_usecs_irq = 0; + new->tx_max_coalesced_frames_irq = 1; + log_println(6, " ndt will attempt to disable TX irq coalescing"); + } + + log_println(6, "--use_adaptive_rx_coalesce : %d", org->use_adaptive_rx_coalesce); + if(org->use_adaptive_rx_coalesce == 0){ + log_println(6, " Adaptive RX coalescing is already disabled"); + } else { + flags |= RX_ADAPT_ON; + new->use_adaptive_rx_coalesce = 0; + log_println(6, " ndt will attempt to disable RX adaptive coalescing"); + } + + log_println(6, "--use_adaptive_tx_coalesce : %d", org->use_adaptive_tx_coalesce); + if(org->use_adaptive_tx_coalesce == 0){ + log_println(6, " Adaptive TX coalescing is already disabled"); + } else { + flags |= TX_ADAPT_ON; + new->use_adaptive_rx_coalesce = 0; + log_println(6, " ndt will attempt to disable TX adaptive coalescing"); + } + + return flags; +} + +/** + * Where possible disables packet coalecsing on all interfaces. + * This will log information about any interfaces which it fails on. + * + * @param autofix + */ +void disable_coalesce(int autofix){ + struct ethtool_coalesce eth_coal_new; + pcap_if_t *alldevs, *dp; + /* The current item in the eth_coal_org stucture we are filling */ + int curr = 0; + /* We use ifr for the ETHTOOL query ioctl*/ + struct ifreq ifr; + int skfd; + __u32 flags; + memset(&ifr, 0, sizeof(ifr)); + char errbuf[PCAP_ERRBUF_SIZE]; + + /* ioctl needs a file discriptor, make sure we get this without + * needing super user permissions. + */ + if (( skfd = socket(AF_INET, SOCK_DGRAM, 0 ) ) < 0 ) + { + log_println(0, "disable_coalesce - Socket error: %s", strerror(errno)); + return; + } + + /* Get all the interfaces using pcap since it only lists + * the interfaces it can actually open. + */ + if (pcap_findalldevs(&alldevs, errbuf) != 0) { + log_println(0, "disable_coalesce - pcap_findalldevs error: %s", errbuf); + close(skfd); + return; + } + + for(dp = alldevs; dp != NULL; dp = dp->next){ + /* Ignore any it's a silly device to try and change */ + if( strcmp("any", dp->name) == 0 || dp->flags & PCAP_IF_LOOPBACK) + continue; + + log_println(6, "Found interface %s", dp->name); + /* Try get current coalesce settings */ + eth_coal_org[curr].coal.cmd = ETHTOOL_GCOALESCE; + strncpy(ifr.ifr_name, dp->name, strlen(dp->name)); + ifr.ifr_data = (char *) ð_coal_org[curr].coal; + + if (ioctl(skfd, SIOCETHTOOL, &ifr)) + { + switch(errno){ + case EPERM: + log_println(1, "I can not read coalesce settings for %s. " + "You should try running ndt as a super user: %s", + dp->name, strerror(errno)); + break; + case EOPNOTSUPP: + /* This is actually ok, the card doesn't support coalescing" */ + log_println(6, "Network interface %s doesn't support" + " coalescing: %s", dp->name, strerror(errno)); + continue; + default: + log_println(1, "I can not read coalesce settings for" + " %s: %s", dp->name, strerror(errno)); + break; + } + log_println(0, "WARNING - Link speed detection may be inaccurate for %s.", dp->name); + continue; + } + + /* Check what the current coalescing settings are and change if necessary */ + flags = check_disable_coalesce(ð_coal_org[curr].coal, ð_coal_new); + + if(flags != 0){ + if (!autofix){ + log_println(0, "Warning: Link speed detection may be inaccurate for %s.", dp->name); + continue; + } + + if (curr == MAX_IFACES) + log_println(0, "Ran out of space to store coalescing settings," + " consider rebuiling with an increased MAX_IFACES kernel_coalesce.c"); + + /* Only change coalesce settings if they are actually enabled */ + eth_coal_new.cmd = ETHTOOL_SCOALESCE; + ifr.ifr_data = (char *) ð_coal_new; + if (ioctl(skfd, SIOCETHTOOL, &ifr)) + { + switch(errno){ + case EPERM: + log_println(1, "I can not set new coalesce settings for %s. " + "You should try running ndt as a super user: %s" + , dp->name, strerror(errno)); + break; + case EOPNOTSUPP: + /* We should have already checked this */ + log_println(1, "Network interface %s doesn't support coalescing" + "However it appeared to when querying for settings: %s" + , dp->name, strerror(errno)); + break; + default: + log_println(1, "I can not set coalesce settings" + " for %s: %s.", dp->name, strerror(errno)); + break; + } + log_println(0, "Warning: Link speed detection may be inaccurate for %s.", dp->name); + continue; + } + log_println(6, "Successfully disabled coalescing for %s.", dp->name); + /* Mark the slot */ + strncpy(eth_coal_org[curr].name, dp->name, strlen(dp->name)); + curr++; + } + } + + if (alldevs != NULL) + pcap_freealldevs(alldevs); + close(skfd); + return; +} + +/** + * Revert the coalesce setting to their original state before ndt was + * launched. Called after disable_coalesce to revert all of the changes. + * This will log information about any interfaces which it fails on. + */ +void revert_coalesce(){ + struct ifreq ifr; + int skfd = -1; + int curr; + memset(&ifr, 0, sizeof(ifr)); + + /* ioctl needs a file discriptor, make sure we get this without + * needing super user permissions. + */ + if (( skfd = socket( AF_INET, SOCK_DGRAM, 0 ) ) < 0 ) + { + log_println(0, "revert_coalesce - Socket error: %s", strerror(errno)); + return; + } + + for(curr = 0; curr < MAX_IFACES; curr++){ + /* Check if we are at the last valid entry */ + if(eth_coal_org[curr].name[0] == 0) + break; + + /* Unset coalesce on this interface */ + strncpy(ifr.ifr_name, eth_coal_org[curr].name, + strlen(eth_coal_org[curr].name)); + + eth_coal_org[curr].coal.cmd = ETHTOOL_SCOALESCE; + ifr.ifr_data = (char *) ð_coal_org; + if (ioctl(skfd, SIOCETHTOOL, &ifr)) + { + log_println(0, "I can not undo coalescing settings for %s: %s", + ifr.ifr_name, strerror(errno)); + continue; + } + log_println(6, "Successfully reverted coalesce settings for %s.", ifr.ifr_name); + /* Free the slot */ + eth_coal_org[curr].name[0] = 0; + } + + close(skfd); + return; +} Index: src/kernel_coalesce.h =================================================================== --- src/kernel_coalesce.h (revision 0) +++ src/kernel_coalesce.h (revision 0) @@ -0,0 +1,14 @@ +/* + * This contains function declaraions for needed by ndt to disable + * packet coalescing on network interfaces. + * + * Author: Richard Sanger + */ + +#ifndef SRC_KERNEL_COALESCE_H_ +#define SRC_KERNEL_COALESCE_H_ + +void disable_coalesce(int autofix); +void revert_coalesce(); + +#endif // SRC_KERNEL_COALESCE_H_ Index: src/kernel_config.c =================================================================== --- src/kernel_config.c (revision 0) +++ src/kernel_config.c (revision 0) @@ -0,0 +1,250 @@ +/* + * This contains functions used by ndt to tweak kernel settings. + * This uses the /proc/sys interface. + * + * Author: Richard Sanger + */ +#include +#include +#include +#include +#include "logging.h" + +/* Storage for the congestion control method */ +static char oldCongestionControl[128]; + +/* The name for reno that we write to the proc interface */ +#define RENO "reno" + +/** + * Reverses swap_to_reno by changing the congestion control to what + * it was before swap_to_reno was called. + * + * @return 0 if successfully otherwise -1 + */ +int revert_congestion(){ + FILE *fdwrite; + size_t len; + + /* Check if we need to change it back */ + if(oldCongestionControl[0] != 0){ + fdwrite = fopen("/proc/sys/net/ipv4/tcp_congestion_control", "w"); + if(fdwrite == NULL){ + log_println(0, "tcp_congestion_control Cannot open file for writing" + " - probably need to be super user: %s", strerror(errno)); + return -1; + } + /* Write the original value back to tcp_congestion_control */ + len = fwrite(oldCongestionControl, sizeof(char), strlen(oldCongestionControl), fdwrite); + if(len != strlen(oldCongestionControl)){ + log_println(0, "tcp_congestion_control Write failed: %s", + strerror(errno)); + fclose(fdwrite); + return -1; + } + if (fclose(fdwrite)){ + log_println(0, "tcp_congestion_control Write failed: %s", + strerror(errno)); + return -1; + } + oldCongestionControl[0] = 0; + log_println(6, "Successfully reverted tcp_congestion_control to its original value"); + } + return 0; +} + +/** + * Swaps the kernel to reno tcp congestion control. The man pages say + * that reno is a guaranteed tcp congestion control on every linux + * system. + * @param autofix if not 0 this function will try and change the settings + * + * @return 0 if successfully otherwise -1 + */ +int check_kernel(int autofix){ + FILE *fdread, *fdwrite; + size_t len; + fdwrite = fdread = NULL; + + /* Try read the current value, if it is already reno we don't need to change it */ + fdread = fopen("/proc/sys/net/ipv4/tcp_congestion_control", "r"); + if(fdread == NULL){ + log_println(0, "tcp_congestion_control Cannot open file for reading: %s", + strerror(errno)); + goto error_handle; + } + /* Store the current value into oldCongestionControl */ + len = fread((void *)oldCongestionControl, sizeof(char), sizeof(oldCongestionControl), fdread); + if(!feof(fdread)){ + log_println(0, "tcp_congestion_control fread failed: %s", + strerror(errno)); + goto error_handle; + } else { + fclose(fdread); + fdread = NULL; + } + + if(len == sizeof(oldCongestionControl)){ + log_println(0, "ERROR original congestion control string is to long"); + goto error_handle; + } else { + /* Null terminate */ + oldCongestionControl[len] = 0; + } + + /* Check if it's already reno */ + if(strncmp(RENO, oldCongestionControl, sizeof(RENO)-1) == 0){ + log_println(6, "Tcp_congestion_control value is already reno no change needed"); + oldCongestionControl[0] = 0; + return 0; + } + + log_println(6, "The original tcp_congestion_control value is %s", + oldCongestionControl); + + /* We need to change to reno */ + if(!autofix){ + log_println(0, "Warning: tcp_congestion_control should be set to '" + RENO "' ndt results may be incorrect"); + oldCongestionControl[0] = 0; + return 0; + } + + fdwrite = fopen("/proc/sys/net/ipv4/tcp_congestion_control", "w"); + if(fdwrite == NULL){ + log_println(0, "Tcp_congestion_control fopen(write) - probably not super user: %s", strerror(errno)); + goto error_handle; + } + len = fwrite(RENO,sizeof(char), sizeof(RENO)-1, fdwrite); + if(len != sizeof(RENO)-1){ + log_println(0, "Tcp_congestion_control fwrite: %s", strerror(errno)); + goto error_handle; + } + if (fclose(fdwrite)){ + if(errno == ENOENT){ + /* Trying to set a bad value will give us this error */ + log_println(0, "'" RENO "' tcp congestion control not available" + " on this system??: %s", strerror(errno)); + } else { + log_println(0, "tcp congestion control fclose(fdwrite): %s", strerror(errno)); + } + goto error_handle; + } + /* Success - Kernel has accpected and changed to reno */ + log_println(6, "Successfully changed tcp_congestion_control to " RENO); + return 0; + + /* Print warnings and close all the file handles if and error occurs */ +error_handle: + /* We have failed reading */ + if(fdread == NULL && fdwrite == NULL){ + log_println(0, "Warning: Could not determine current tcp_congestion_control" + ", this should be set to 'reno' or else ndt results may be incorrect."); + oldCongestionControl[0] = 0; + return -1; + } + /* Set close the read handle if its still open */ + if(fdread != NULL){ + fclose(fdread); + fdread = NULL; + } + /* We have failed writing */ + if(fdwrite != NULL){ + fclose(fdwrite); + fdwrite = NULL; + return -1; + } + oldCongestionControl[0] = 0; + log_println(0, "Warning: Failed to change congestion tcp_congestion_control to '" + RENO "' ndt results may be incorrect"); + return -1; +} + + +static char module_already_loaded = 1; +#define TCP_ESTATS_MODULE "tcp_estats_nl" + +/** + * Search for the TCP_ESTATS_MODULE in /proc/module aka the list of + * running modules + * + * @return 0 not running, 1 running, -1 error. + */ +static int search_for_module_tcpe(){ + int loaded = 0; + FILE *mods_fd; + char buf[256]; + mods_fd = fopen("/proc/modules", "r"); + + if(mods_fd == NULL){ + log_println(0, "/proc/modules Cannot open file for reading: %s", + strerror(errno)); + return -1; + } + + /* Search for the tcp_estats module line */ + while(fgets(buf, 256, mods_fd)){ + if(strncmp(TCP_ESTATS_MODULE " ", buf , sizeof(TCP_ESTATS_MODULE " ") - 1) + == 0){ + loaded = 1; + break; + } + } + + fclose(mods_fd); + return loaded; +} + +/** + * Check if tcp_estats module is loaded and will load if not. (Only + * relevent to web10G) + * + * @return 0 on success, -1 upon failure either to load the module or + * determine if the module is running. + */ +int start_tcpe(){ + int code = search_for_module_tcpe(); + + if(code == -1) + return -1; + + if(code == 1){ + module_already_loaded = 1; + return 0; + } + module_already_loaded = 0; + /* Module is not loaded try and load it */ + log_println(1, "Warning: tcp_estats module not loaded, attempting to load"); + system("modprobe " TCP_ESTATS_MODULE ); + if(search_for_module_tcpe() == 1) + return 0; + + log_println(0, "Error: tcp_estats module could not be loaded"); + return -1; +} + +/** + * Unload tcp_estats module if it was previously unloaded (Only + * relevent to web10G). If the module was previously loaded + * success code is returned and this function does nothing. + * + * Only should be called after start_tcpe(). + * + * @return 0 on success, -1 upon failure either to unload the module. + */ +int revert_tcpe(){ + if( module_already_loaded ) + return 0; + + /* We had to load the module so unload it*/ + log_println(1, "Warning: tcp_estats needs to be unloaded, attempting to unload"); + system("modprobe -r " TCP_ESTATS_MODULE ); + if( search_for_module_tcpe() == 0){ + log_println(1, "Successfully unloaded tcp_estats "); + return 0; + } + + log_println(0, "Warning: tcp_estats failed to unload '"); + return -1; +} + Index: src/kernel_config.h =================================================================== --- src/kernel_config.h (revision 0) +++ src/kernel_config.h (revision 0) @@ -0,0 +1,16 @@ +/* + * This contains functions declaractions used by ndt to tweak kernel settings. + * This uses the /proc/sys interface. + * + * Author: Richard Sanger + */ + +#ifndef SRC_KERNEL_CONFIG_H_ +#define SRC_KERNEL_CONFIG_H_ + +int check_kernel(int autofix); +int revert_congestion(); +int start_tcpe(); +int revert_tcpe(); + +#endif // SRC_KERNEL_CONFIG_H_ Index: src/usage.c =================================================================== --- src/usage.c (revision 796) +++ src/usage.c (working copy) @@ -69,6 +69,7 @@ printf(" -L, --log_dir DIR - specify the base directory for snaplog and tcpdump files\n"); printf(" (default %s/serverdata)\n", BASEDIR); printf(" -S, --logfacility #F - specify syslog facility name\n"); + printf(" -k, --kernelconfig - Automatically configure kernel settings for best results\n"); printf(" Note: this doesn't enable 'syslog'\n\n"); #ifdef EXPERIMENTAL_ENABLED printf(" Experimental code:\n\n"); Index: src/Makefile.am =================================================================== --- src/Makefile.am (revision 796) +++ src/Makefile.am (working copy) @@ -56,7 +56,7 @@ web100srv_SOURCES = web100srv.c web100-util.c web100-pcap.c web100-admin.c runningtest.c \ network.c usage.c utils.c mrange.c logging.c testoptions.c ndtptestconstants.c \ protocol.c test_sfw_srv.c test_meta_srv.c ndt_odbc.c strlutils.c heuristics.c \ - test_c2s_srv.c test_s2c_srv.c test_mid_srv.c + test_c2s_srv.c test_s2c_srv.c test_mid_srv.c kernel_coalesce.c kernel_config.c web100srv_LDFLAGS = $(NDTLDFLAGS) $(I2UTILLDFLAGS) web100srv_LDADD = $(NDTLIBS) $(I2UTILLIBS) $(I2UTILLIBDEPS) -lpthread $(ZLIB) web100srv_CPPFLAGS ='-DBASEDIR="$(ndtdir)"' @@ -88,5 +88,5 @@ EXTRA_DIST = clt_tests.h logging.h mrange.h network.h protocol.h testoptions.h test_sfw.h test_meta.h \ troute.h tr-tree.h usage.h utils.h varinfo.h web100-admin.h web100srv.h ndt_odbc.h runningtest.h ndtptestconstants.h \ - heuristics.h strlutils.h test_results_clt.h tests_srv.h + heuristics.h strlutils.h test_results_clt.h tests_srv.h kernel_coalesce.h kernel_config.h Index: src/web100srv.c =================================================================== --- src/web100srv.c (revision 796) +++ src/web100srv.c (working copy) @@ -86,6 +86,8 @@ #include "strlutils.h" #include "heuristics.h" #include "tests_srv.h" +#include "kernel_coalesce.h" +#include "kernel_config.h" static char lgfn[FILENAME_SIZE]; // log file name static char wvfn[FILENAME_SIZE]; // file name of web100-variables list @@ -166,6 +168,7 @@ { "record", 0, 0, 'r' }, { "syslog", 0, 0, 's' }, { "tcpdump", 0, 0, 't' }, { "version", 0, 0, 'v' }, { "gzip", 0, 0, 'z' }, { "config", 1, 0, 'c' }, + { "kernelconfig", 0, 0, 'k'}, #ifdef EXPERIMENTAL_ENABLED { "avoidsndblockup", 0, 0, 306}, { "snaplog", 0, 0, 307}, @@ -457,6 +460,13 @@ exit(-2); break; case SIGINT: + if(getpid() == ndtpid){ + revert_coalesce(); + revert_congestion(); +//#ifndef FORCE_WEB100 +// revert_tcpe(); +//#endif + } exit(0); case SIGTERM: if (getpid() == ndtpid) { @@ -1451,6 +1461,7 @@ int ctlsockfd = -1; int c, chld_pipe[2]; int i, loopcnt, t_opts = 0; + int kernel_config = 0; struct sockaddr_storage cli_addr; struct sigaction new; web100_agent* agent; @@ -1516,7 +1527,7 @@ argc, argv, GETOPT_LONG_INET6( - GETOPT_LONG_EXP("adhmoqrstvzc:x:b:f:i:l:u:e:p:T:A:S:")), + GETOPT_LONG_EXP("adhmoqrstvzc:x:b:f:i:l:u:e:p:T:A:S:k")), long_options, 0)) != -1) { switch (c) { case 'c': @@ -1555,7 +1566,7 @@ argc, argv, GETOPT_LONG_INET6( - GETOPT_LONG_EXP("adhmoqrstvzc:x:b:f:i:l:u:e:p:T:A:S:")), + GETOPT_LONG_EXP("adhmoqrstvzc:x:b:f:i:l:u:e:p:T:A:S:k")), long_options, 0)) != -1) { switch (c) { case '4': @@ -1715,6 +1726,9 @@ case '?': short_usage(argv[0], ""); break; + case 'k': + kernel_config = 1; + break; } } @@ -1889,7 +1903,18 @@ // RAC 7/14/09 get_iflist(); + disable_coalesce(kernel_config); + check_kernel(kernel_config); +//#ifndef FORCE_WEB100 +// if(start_tcpe() == -1){ +// log_println(0, "Warning: Web10G module cannot be loaded into kernel\n" +// "\t - This is expected if Web10G is built-in to the kernel\n" +// "\t - If not either sudo modprobe tcp_estats_nl yourself\n" +// "\t - or run this program as superuser\n"); +// } +//#endif + for (i = 0; iflist.speed[i] > 0; i++) log_println(4, "Generated iflist with device=%s and if_speed=%d", iflist.name[i], iflist.speed[i]); @@ -2755,6 +2780,11 @@ break; } } + revert_coalesce(); + revert_congestion(); +#ifndef FORCE_WEB100 + revert_tcpe(); +#endif } /**