http bandwidth drops periodically

[anuradhapriyankara]

Hi,
I am developing a traffic monitoring application and planning to use warp17 for testing it. However, I have experienced that warp17 application traffic bandwidth drops periodically. Here is a snapshot I captured by my monitoring application. I am expecting to have a flat curve without periodic drops.

I have previously tested my application with real world traffic, so I don't think there is a problem with my application.

I am using a custom python script based on test_1_http_4M.py to create 275 http sessions belongs to 11 URLs(Applications).
My test setup use two X710 NICs with 6GB hugepage memory.

[chandrasheker]

I think you need to modify your uptime to infinity, I think this is configured as 10 seconds.

[anuradhapriyankara]

I think you need to modify your uptime to infinity, I think this is configured as 10 seconds.

Thanks for the reply. I don't think uptime is the problem here. I have set stop time to 36000(10 hours) on below line of python script for both client and server,

scrit = TestCriteria(tc_crit_type=SRV_UP, tc_srv_up=stop_time)

[davvore33]

Hi @anuradhapriyankara, could you please give us the whole configuration, either way python commands or cli commands is fine

[anuradhapriyankara]

Hi @anuradhapriyankara, could you please give us the whole configuration, either way python commands or cli commands is fine

Hi, @davvore33 Sorry for the extreme delay.
I have faced this problem first with warp17 v1.6. So thinking the older version was the problem I have upgraded to v1.8. However the problem still exists.
I cannot publish my original code due to company policy, but I was able to replicate the problem using test_1_http_4M.py

Here is my code,

`
import sys
import time
import os
import argparse

sys.path.append('../../python')
sys.path.append('../../api/generated/py')
sys.path.append('../../ut/lib')

from warp17_api import *

.# We hijack a bit the back2back definitions that we use for testing.
.# Ideally the user should have his own ways scripting the configured values.
from b2b_setup import *

from warp17_common_pb2 import *
from warp17_l3_pb2 import *
from warp17_server_pb2 import *
from warp17_client_pb2 import *
from warp17_app_pb2 import *
from warp17_app_http_pb2 import *
from warp17_test_case_pb2 import *
from warp17_service_pb2 import *

def die(msg):
sys.stderr.write(msg + ' Should cleanup but we just exit..\n')
sys.exit(1)

def init_test(w17_binary):
# Configure the WARP17 mandatory environment variables. These could also be
# set outside the script (through normal env variables) but for this example
# we do it here..

# Save the hostname and tcp port WARP17 listens on. Also the env.
global warp17_host
global warp17_port
global env

env = Warp17Env(path=os.path.join(os.path.dirname(__file__),
                                  './test_1_http_4M.ini'))

# First start WARP17 on the local machine, default IP and port.
warp17_host = env.get_host_name()
warp17_port = env.get_rpc_port()

# Ask for the output log to go to /tmp/test_1_api_example.out
# Returns the process id of WARP17.
# This will throw an exception if the output file can't be created or if
# WARP17 can't be started.
warp17_pid = warp17_start(env=env, exec_file=w17_binary,
                          output_args=Warp17OutputArgs(out_file='/tmp/test_1_api_example_2.out'))

# Now wait for WARP17 to finish initializing
# This will exit if WARP17 fails to initialize
warp17_wait(env)
return warp17_pid

def configure_client_port():
# Configure 2 client IP interfaces and no default gateway on port 0.
# First the default gw
pcfg = b2b_port_add(eth_port=0, def_gw =Ip(ip_version=IPV4, ip_v4=0))

intf1 = (Ip(ip_version=IPV4, ip_v4=b2b_ipv4(eth_port=0, intf_idx=0)),
         Ip(ip_version=IPV4, ip_v4=b2b_mask(eth_port=0, intf_idx=0)),
         b2b_count(eth_port=0, intf_idx=0))
intf2 = (Ip(ip_version=IPV4, ip_v4=b2b_ipv4(eth_port=0, intf_idx=1)),
         Ip(ip_version=IPV4, ip_v4=b2b_mask(eth_port=0, intf_idx=1)),
         b2b_count(eth_port=0, intf_idx=1))
b2b_port_add_intfs(pcfg, [intf1, intf2])

# Ask WARP17 to add them to the config.
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'ConfigurePort', pcfg)
if err.e_code != 0:
    die('Error configuring port 0.')

# Prepare the L4 Client config (eth_port 0)
# source ports in the range: [10001, 30000]
# destination ports in the range: [101, 200]
# In total (per client IP): 2M sessions.
# In total: 4M sessions
sport_range=L4PortRange(l4pr_start=10001, l4pr_end=10275)
dport_range=L4PortRange(l4pr_start=80, l4pr_end=80)
l4_ccfg = L4Client(l4c_proto=TCP,
                   l4c_tcp_udp=TcpUdpClient(tuc_sports=sport_range,
                                            tuc_dports=dport_range))

rate_ccfg = RateClient(rc_open_rate=Rate(),  # no rate limiting
                       rc_close_rate=Rate(), # no rate limiting
                       rc_send_rate=Rate(r_value=30000))  # no rate limiting

# Prepare the HTTP Client config
http_ccfg = App(app_proto=HTTP_CLIENT,
                app_http_client=HttpClient(hc_req_method=GET,
                                           hc_req_object_name='/index.html',
                                           hc_req_host_name='www.facebook.com',
                                           hc_req_size=2048)) # configure HTTP requests of size 2K

# Prepare the Client test case criteria.
# Let the test case run for one hour.
ccrit = TestCriteria(tc_crit_type=RUN_TIME, tc_run_time_s=3600)

# Put the whole test case config together.
ccfg = TestCase(tc_type=CLIENT, tc_eth_port=0,
                tc_id=0,
                tc_client=Client(cl_src_ips=b2b_sips(eth_port=0, ip_count=2),
                                 cl_dst_ips=b2b_dips(eth_port=0, ip_count=1),
                                 cl_l4=l4_ccfg,
                                 cl_rates=rate_ccfg),
                tc_app=http_ccfg,
                tc_criteria=ccrit,
                tc_async=True)

# Ask WARP17 to add the test case config
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'ConfigureTestCase', ccfg)
if err.e_code != 0:
    die('Error configuring client test case.')

print 'Clients configured successfully!\n'

def configure_server_port():
# Configure 1 server IP interface and no default gateway on port 1.
# First the default gw
pcfg = b2b_port_add(eth_port=1, def_gw=Ip(ip_version=IPV4, ip_v4=0))

intf1 = (Ip(ip_version=IPV4, ip_v4=b2b_ipv4(eth_port=1, intf_idx=0)),
         Ip(ip_version=IPV4, ip_v4=b2b_mask(eth_port=1, intf_idx=0)),
         b2b_count(eth_port=0, intf_idx=0))
b2b_port_add_intfs(pcfg, [intf1])

# Ask WARP17 to add them to the config.
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'ConfigurePort', pcfg)
if err.e_code != 0:
    die('Error configuring port 1.')

# Prepare the L4 Server config (eth_port 1)
# ports in the range: [101, 200]
port_range=L4PortRange(l4pr_start=80, l4pr_end=80)
l4_scfg = L4Server(l4s_proto=TCP,
                   l4s_tcp_udp=TcpUdpServer(tus_ports=port_range))

# Prepare the HTTP Server config
http_scfg = App(app_proto=HTTP_SERVER,
                app_http_server=HttpServer(hs_resp_code=OK_200,
                                           hs_resp_size=2048))

# The server test case criteria is to have all servers in listen state.
# However, server test cases are special and keep running even after the
# PASS criteria is met
scrit = TestCriteria(tc_crit_type=SRV_UP, tc_srv_up=3600)

# Put the whole test case config together
scfg = TestCase(tc_type=SERVER, tc_eth_port=1, tc_id=0,
                tc_server=Server(srv_ips=b2b_sips(eth_port=1, ip_count=1),
                                 srv_l4=l4_scfg),
                tc_app=http_scfg,
                tc_criteria=scrit,
                tc_async=False)

# Ask WARP17 to add the test case config
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'ConfigureTestCase', scfg)
if err.e_code != 0:
    die('Error configuring server test case.')

print 'Servers configured successfully!\n'

def start_client_port():
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'PortStart',
PortArg(pa_eth_port=0))
if err.e_code != 0:
die('Error starting client test cases.')

print 'Clients started successfully!\n'

def start_server_port():
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'PortStart',
PortArg(pa_eth_port=1))
if err.e_code != 0:
die('Error starting server test cases.')

print 'Servers started successfully!\n'

def check_stats():
# Just check client stats a couple of times (once a second and stop
# afterwards)..
for i in range(0, 200):
client_result = warp17_method_call(warp17_host, warp17_port, Warp17_Stub,
'GetTestStatus',
TestCaseArg(tca_eth_port=0,
tca_test_case_id=0))
if client_result.tsr_error.e_code != 0:
die('Error fetching client test case stats.')

    print 'Client test case state: ' + str(client_result.tsr_state) + '\n'
    print 'Global stats:'
    print client_result.tsr_stats
    print 'Rate stats:'
    print client_result.tsr_rate_stats
    print 'HTTP Client stats:'
    print client_result.tsr_app_stats.as_http

    time.sleep(1)

def stop_client_port():
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'PortStop',
PortArg(pa_eth_port=0))
if err.e_code != 0:
die('Error stopping client test cases.')

print 'Clients stopped successfully!\n'

def stop_server_port():
err = warp17_method_call(warp17_host, warp17_port, Warp17_Stub, 'PortStop',
PortArg(pa_eth_port=1))
if err.e_code != 0:
die('Error stopping server test cases.')

print 'Servers stopped successfully!\n'

def run_test():
''' Assumes a back to back topology with two 40G ports. '''
''' Port 0 emulates clients and port 1 emulates servers. '''

parser = argparse.ArgumentParser()

parser.add_argument('-f', '--file', help='path to warp17 binary',
                    required=True)
args = parser.parse_args()

warp17_pid = init_test(args.file)
configure_client_port()
configure_server_port()

start_server_port()
start_client_port()

check_stats()

stop_client_port()
stop_server_port()

# Cleanup: Ask WARP17 to stop
warp17_stop(env, warp17_pid)

if name == 'main':
run_test()
`

And here is the test_1_http_4M.ini

`

[DEFAULT]
.# Mask of cores to be assigned to DPDK
coremask=0xF
.# Number of memory channels to use
nchan=1
.# Hugepages memory to be used (in MB)
memory=2000

.# TCB pool size (by default 10M TCBs)
tcb-pool-sz=200

.# Disable UDP packet pool since we don't use it
ucb-pool-sz=0

mbuf-pool-sz=384

.# List of ports to be used. These must be already added to the DPDK driver.
ports=0000:65:00.0 0000:65:00.1

.# Default qmap
qmap-default=max-c

[0000:82:00.0]
.# This is here for reference as the default qmap will be used
.# Comment out qmap-default from DEFAULT if this one should be used
qmap=0x42

[0000:83:00.0]
.# This is here for reference as the default qmap will be used
.# Comment out qmap-default from DEFAULT if this one should be used
qmap=0x42`

The problem seem to be associated with the rate_ccfg. I have tested using multiple rc_send_rate and following were the results,

rc_send_rate = 30000

rc_send_rate = 300000

rc_send_rate = Rate() - Unlimited

The drop are not seems to be periodic. When the rate is around 30000 - 40000 the curve is flat.