diff --git a/.gitmodules b/.gitmodules index bcef25b35..2e9c43593 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,5 @@ [submodule "libmoon"] path = libmoon - url = https://github.com/libmoon/libmoon + url = https://github.com/brentondwalker/libmoon.git + branch = master + diff --git a/examples/ipt-rate-set-crc.lua b/examples/ipt-rate-set-crc.lua new file mode 100644 index 000000000..d14613c4b --- /dev/null +++ b/examples/ipt-rate-set-crc.lua @@ -0,0 +1,194 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local hist = require "histogram" +local log = require "log" +local timer = require "timer" +local limiter = require "software-ratecontrol" +local stats = require "stats" + +local ETH_DST = "11:12:13:14:15:16" + + +function master(txPort, rxPort, pattern, threads, pktSize, rate, waitTime) + if not rxPort or not txPort or not threads or not pktSize or not rate then + errorf("usage: txPort rxPort pattern threads pktSize rate [waitTime]") + return + end + + --if (((pktSize+26)*8) > (1000*ipt)) then + -- errorf("requested packet size and inter-packet time is infeasible") + --end + + --local rxDev = device.config{ port = rxPort, rxDescs = 4096, dropEnable = true } + --local txDev = device.config{ port = txPort, txQueues = threads, disableOffloads = true, dropEnable = false } + -- rxDev:wait() + -- txDev:wait() +<<<<<<< HEAD + --device.waitForLinks() +======= + device.waitForLinks() +>>>>>>> d1110772cdaf2ee21f1845884b2565922ef3bb12 + + local queue = rxDev:getRxQueue(0) + queue:enableTimestampsAllPackets() + local total = 0 + local bufs = memory.createBufArray() + local times = {} + local sizes = {} + local timer = timer:new(waitTime) + + local txDev = device.config({port = args.txDev, txQueues = 2, rxQueues = 2, disableOffloads = true}) + local rxDev = device.config({port = args.rxDev, txQueues = 2, rxQueues = 2}) + device.waitForLinks() + + mg.startTask("loadSlave", txDev, rxDev, txDev:getTxQueue(0), args.rate, PKT_SIZE) + mg.startTask("timerSlave", txDev:getTxQueue(1), rxDev:getRxQueue(1), PKT_SIZE) + mg.waitForTasks() + + for i = 1, threads do + -- local pktRate = (1000.0 * rate) / pktSize + mg.startTask("loadSlave", txDev:getTxQueue(i - 1), txDev, rate, i, threads, pktSize) + mg.startTask("timerSlave", txDev:getTxQueue(i), rxDev:getRxQueue(i), PKT_SIZE) + end + + local hsz = hist:create() + local last = 0 + while mg.running() do + local n = queue:recv(bufs) + for i = 1, n do + --if timer:expired() then + local ts = bufs[i]:getTimestamp() + times[#times + 1] = ts + local sz = bufs[i].pkt_len + -- print(i, sz, ts, ts-last) + last = ts + sizes[#sizes + 1] = sz + hsz:update(sz) + --end + end + total = total + n + bufs:free(n) + end + + mg.waitForTasks() + + hsz:print() + hsz:save("sizes.csv") + --for i, v in ipairs(sizes) do + -- print(i,v) + --end + + local pkts = rxDev:getRxStats(port) + local h = hist:create() + local last + for i, v in ipairs(times) do + if last then + local diff = v - last + h:update(diff) + end + last = v + end + + h:print() + h:save("histogram.csv") + log[(pkts - total > 0 and "warn" or "info")](log, "Lost packets: " .. pkts - total + .. " (this can happen if the NIC still receives data after this script stops the receive loop)") + +end + +function timerSlave(txQueue, rxQueue, size) + local timestamper = ts:newTimestamper(txQueue, rxQueue) + local hist = histogram:new() + -- wait for a second to give the other task a chance to start + mg.sleepMillis(1000) + local rateLimiter = timer:new(0.001) + while mg.running() do + rateLimiter:reset() + hist:update(timestamper:measureLatency(size)) + rateLimiter:busyWait() + end + hist:print() + hist:save("histogram.csv") +end + + + +function loadSlave2(queue, txDev, rate, threadId, numThreads, pktSize) + local ETH_DST = "11:12:13:14:15:16" + local PKT_SIZE = 60 + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethSrc = txDev, + ethDst = ETH_DST, + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + while mg.running() do + bufs:alloc(PKT_SIZE) + queue:send(bufs) + end +end + + + +function loadSlave333(queue, txDev, rate, threadId, numThreads, pktSize) + local ETH_DST = "11:12:13:14:15:16" + --local PKT_SIZE = 60 + + -- doing crc rate control requires us to know the link speed. + -- it is given in Mbps, just like the rate argument + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local mem = memory.createMemPool{n=4096, func=function(buf) + --local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethSrc = txDev, + ethDst = ETH_DST, + ethType = 0x1234 + } + end} + + -- larger batch size is useful when sending it through a rate limiter + local bufs = mem:bufArray() --(128) + -- local dist = pattern == "poisson" and poissonDelay or function(x) return x end + while mg.running() do + bufs:alloc(pktSize) + for _, buf in ipairs(bufs) do + --buf:setDelay(dist(10^10 / numThreads / 8 / (rate * 10^6) - pktSize - 24)) + -- --buf:setDelay(1000000) + buf:setDelay((pktSize+24) * (linkspeed/rate - 1) ) + end + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + queue:sendWithDelay(bufs, rate * numThreads) + --queue:send(bufs) + end +end + +function loadSlave(queue, dev, rate, threadId, numThreads, size) + print("using packet size "..size) + local linkspeed = dev:getLinkStatus().speed + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + local txStats = stats:newManualTxCounter(dev, "plain") + while mg.running() do + bufs:alloc(size) + for _, buf in ipairs(bufs) do + buf:setDelay((size+24) * (linkspeed/rate - 1) ) + end + txStats:updateWithSize(queue:sendWithDelay(bufs), size) + end + txStats:finalize() +end + + + diff --git a/examples/ipt-rate-set.lua b/examples/ipt-rate-set.lua new file mode 100644 index 000000000..25292a71b --- /dev/null +++ b/examples/ipt-rate-set.lua @@ -0,0 +1,92 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local hist = require "histogram" +local log = require "log" +local timer = require "timer" +local limiter = require "software-ratecontrol" + +local ETH_DST = "11:12:13:14:15:16" + + +function master(txPort, rxPort, pattern, threads, pktSize, ipt, waitTime) + if not rxPort or not txPort or not threads or not pktSize or not ipt then + errorf("usage: txPort rxPort threads pktSize interpacket_time(us) [waitTime]") + end + + if (((pktSize+26)*8) > (1000*ipt)) then + errorf("requested packet size and inter-packet time is infeasible") + end + + local rxDev = device.config{ port = rxPort, rxDescs = 4096, dropEnable = true } + local txDev = device.config{port = txPort, txQueues = threads, disableOffloads = true} + -- rxDev:wait() + -- txDev:wait() + device.waitForLinks() + + local queue = rxDev:getRxQueue(0) + queue:enableTimestampsAllPackets() + local total = 0 + local bufs = memory.createBufArray() + local times = {} + local timer = timer:new(waitTime) + + for i = 1, threads do + -- local pktRate = (1000.0 * rate) / pktSize + local rateLimiter = limiter:new(txDev:getTxQueue(i - 1), pattern, 1000*ipt) + mg.startTask("loadSlave", txDev:getTxQueue(i - 1), txDev, rateLimiter, i, threads, pktSize) + end + + while mg.running() do + local n = queue:recv(bufs) + for i = 1, n do + if timer:expired() then + local ts = bufs[i]:getTimestamp() + times[#times + 1] = ts + end + end + total = total + n + bufs:free(n) + end + + mg.waitForTasks() + + local pkts = rxDev:getRxStats(port) + local h = hist:create() + local last + for i, v in ipairs(times) do + if last then + local diff = v - last + h:update(diff) + end + last = v + end + h:print() + h:save("histogram.csv") + log[(pkts - total > 0 and "warn" or "info")](log, "Lost packets: " .. pkts - total + .. " (this can happen if the NIC still receives data after this script stops the receive loop)") +end + + + +function loadSlave(queue, txDev, rateLimiter, threadId, numThreads, pktSize) + local mem = memory.createMemPool(4096, function(buf) + buf:getEthernetPacket():fill{ + ethSrc = txDev, + ethDst = ETH_DST, + ethType = 0x1234 + } + end) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = mem:bufArray(128) + while mg.running() do + bufs:alloc(pktSize) + rateLimiter:send(bufs) + end +end + + + + diff --git a/examples/l2-cbr-ipt-runner.sh b/examples/l2-cbr-ipt-runner.sh new file mode 100755 index 000000000..b7d8ec37b --- /dev/null +++ b/examples/l2-cbr-ipt-runner.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +hist_base=$1 + +if [ "$#" -lt 2 ] || [ ${hist_base:0:1} == "-" ] ; then + echo "usage: l2-cbr-ipt-runner.sh " + exit +fi + +declare rates=("10" "20" "40" "80" "160" "320" "640" "1000") +declare sizes=("1400" "688" "332" "154" "65" "60") + +# get the path where this script is +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo $dir + +for size in "${sizes[@]}" +do + for rate in "${rates[@]}" + do + sudo $dir/../build/MoonGen $dir/../examples/l2-cbr-load-ipt.lua -r $rate -s $size "${@:2}" + $dir/../normalizer-ipt.pl histogram.csv > $hist_base-r$rate-s$size.csv + #echo $rate $hist_base"-r"$rate"-s"$size".csv" + #echo "${@:2}" + done +done + + + diff --git a/examples/l2-cbr-latency-runner.sh b/examples/l2-cbr-latency-runner.sh new file mode 100755 index 000000000..8ff4c1d10 --- /dev/null +++ b/examples/l2-cbr-latency-runner.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +hist_base=$1 + +if [ "$#" -lt 2 ] || [ ${hist_base:0:1} == "-" ] ; then + echo "usage: l2-cbr-latency-runner.sh " + exit +fi + +declare rates=("10" "20" "40" "80") +# "160" "320" "640" "1000") +declare sizes=("1400" "688" "332" "154" "65" "60") + +# get the path where this script is +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo $dir + +for size in "${sizes[@]}" +do + for rate in "${rates[@]}" + do + sudo $dir/../build/MoonGen $dir/../examples/l2-cbr-load-latency.lua -r $rate -s $size "${@:2}" + $dir/../normalizer.pl histogram.csv > $hist_base-r$rate-s$size.csv + #echo $rate $hist_base"-r"$rate"-s"$size".csv" + #echo "${@:2}" + done +done + + + diff --git a/examples/l2-cbr-load-ipt.lua b/examples/l2-cbr-load-ipt.lua new file mode 100644 index 000000000..00115d9b3 --- /dev/null +++ b/examples/l2-cbr-load-ipt.lua @@ -0,0 +1,105 @@ +local mg = require "moongen" +local memory = require "memory" +local ts = require "timestamping" +local device = require "device" +local stats = require "stats" +local timer = require "timer" +local histogram = require "histogram" +local log = require "log" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Generates traffic based on a poisson process with CRC-based rate control.") + parser:argument("txDev", "Device to transmit from."):args(1):convert(tonumber) + parser:argument("rxDev", "Device to receive from."):args(1):convert(tonumber) + parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:option("-s --size", "Packet size to use (min=60, max~~1500)"):args(1):default(60):convert(tonumber) + parser:option("-n --numpackets", "Number of packets to sample (default = 0 = run forever)"):args(1):default(0):convert(tonumber) +end + +function master(args) + local txDev = device.config({port = args.txDev, txQueues = 2, rxQueues = 2}) + local rxDev = device.config({port = args.rxDev, rxDescs = 4096, dropEnable = false }) + PKT_SIZE = math.max(60, args.size) + print("using packet size "..PKT_SIZE) + device.waitForLinks() + + mg.startTask("iptSlave", rxDev:getRxQueue(0), args.numpackets) + mg.startTask("loadSlave", txDev, txDev:getTxQueue(0), args.rate, PKT_SIZE) + mg.waitForTasks() +end + +function loadSlave(dev, queue, rate, size) + + -- doing crc rate control requires us to know the link speed. + -- it is given in Mbps, just like the rate argument + local linkspeed = dev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + local txStats = stats:newManualTxCounter(dev, "plain") + local numsent = 0 + while mg.running() do + bufs:alloc(size) + for _, buf in ipairs(bufs) do + -- this script uses Mpps instead of Mbit (like the other scripts) + --buf:setDelay((10^10 / 8 / (rate * 10^6) - size - 24)) + buf:setDelay((size+24) * (linkspeed/rate - 1) ) + --buf:setRate(rate*10) -- rate in Mpps on gigabit ethernet + -- from crc-ratecontrol.lua: + -- delay The time to wait before this packet \(in bytes, i.e. 1 == 0.8 nanoseconds on 10 GbE\) + -- self.udata64 = 10^10 / 8 / (rate * 10^6) - self.pkt_len - 24 + -- key code from stats.lua (accounting for preamble + inter-packet gap = 8+12?): + -- local mpps = (pkts - self.total) / elapsed / 10^6 + -- local mbit = (bytes - self.totalBytes) / elapsed / 10^6 * 8 + -- local wireRate = mbit + (mpps * 20 * 8) + + numsent = numsent + 1 + end + txStats:updateWithSize(queue:sendWithDelay(bufs), size) + + end + print("sent packets: "..numsent) + txStats:finalize() +end + +function iptSlave(queue, numpackets) + queue:enableTimestampsAllPackets() + local total = 0 + local bufs = memory.createBufArray() + local times = {} + local numrx = 0 + while mg.running() and (numpackets == 0 or numrx < numpackets) do + local n = queue:recv(bufs) + for i = 1, n do + local ts = bufs[i]:getTimestamp() + times[#times + 1] = ts + --print(ts) + numrx = numrx + 1 + end + total = total + n + bufs:free(n) + end + mg.stop() + print("captured packets: "..numrx) + local h = histogram:create() + local last + for i, v in ipairs(times) do + --print(i,v) + if last then + local diff = v - last + h:update(diff) + end + last = v + end + --h:print() + h:save("histogram.csv") +end + + diff --git a/examples/l2-cbr-load-latency-newconf.lua b/examples/l2-cbr-load-latency-newconf.lua new file mode 100644 index 000000000..fd7e6d5f1 --- /dev/null +++ b/examples/l2-cbr-load-latency-newconf.lua @@ -0,0 +1,92 @@ +local mg = require "moongen" +local memory = require "memory" +local ts = require "timestamping" +local device = require "device" +local stats = require "stats" +local timer = require "timer" +local histogram = require "histogram" +local log = require "log" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Generates traffic based on a poisson process with CRC-based rate control.") + parser:argument("txDev", "Device to transmit from."):args(1):convert(tonumber) + parser:argument("rxDev", "Device to receive from."):args(1):convert(tonumber) + parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2) + parser:option("-s --size", "Packet size to use (min=60, max~~1500)"):args(1):default(60) + parser:option("-n --numpackets", "Number of packets to sample (default = 0 = run forever)"):args(1):default(0):convert(tonumber) +end + +function master(args) + local txDev = device.config({port = args.txDev, txQueues = 2, rxQueues = 2, txDescs = 1024, numBufs = 1024}) + local rxDev = device.config({port = args.rxDev, txQueues = 2, rxQueues = 2, rxDescs = 1024, numBufs = 1024}) + PKT_SIZE = math.max(60, tonumber(args.size)) + print("using packet size "..PKT_SIZE) + device.waitForLinks() + + mg.startTask("loadSlave", txDev, rxDev, txDev:getTxQueue(0), args.rate, PKT_SIZE) + mg.startTask("timerSlave", txDev:getTxQueue(1), rxDev:getRxQueue(1), PKT_SIZE, args.numpackets) + mg.waitForTasks() +end + +function loadSlave(txDev, rxDev, queue, rate, size) + + -- doing crc rate control requires us to know the link speed. + -- it is given in Mbps, just like the rate argument + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + local rxStats = stats:newDevRxCounter(rxDev, "plain") + local txStats = stats:newManualTxCounter(txDev, "plain") + while mg.running() do + bufs:alloc(size) + for _, buf in ipairs(bufs) do + -- this script uses Mpps instead of Mbit (like the other scripts) + --buf:setDelay((10^10 / 8 / (rate * 10^6) - size - 24)) + buf:setDelay((size+24) * (linkspeed/rate - 1) ) + --buf:setRate(rate*10) -- rate in Mpps on gigabit ethernet + -- from crc-ratecontrol.lua: + -- delay The time to wait before this packet \(in bytes, i.e. 1 == 0.8 nanoseconds on 10 GbE\) + -- self.udata64 = 10^10 / 8 / (rate * 10^6) - self.pkt_len - 24 + -- key code from stats.lua (accounting for preamble + inter-packet gap = 8+12?): + -- local mpps = (pkts - self.total) / elapsed / 10^6 + -- local mbit = (bytes - self.totalBytes) / elapsed / 10^6 * 8 + -- local wireRate = mbit + (mpps * 20 * 8) + end + txStats:updateWithSize(queue:sendWithDelay(bufs), size) + rxStats:update() + --txStats:update() + end + rxStats:finalize() + txStats:finalize() +end + +function timerSlave(txQueue, rxQueue, size, numpackets) + numpackets = numpackets or 0 + local timestamper = ts:newTimestamper(txQueue, rxQueue) + local hist = histogram:new() + -- wait for a second to give the other task a chance to start + mg.sleepMillis(1000) + local rateLimiter = timer:new(0.001) + local pktCount = 0 + while mg.running() and (numpackets == 0 or pktCount < numpackets) do + rateLimiter:reset() + local measurement, num = timestamper:measureLatency(size) + --print(measurement, num) + hist:update(measurement) + pktCount = pktCount + 1 + rateLimiter:busyWait() + end + mg.stop() + print("latency measurements: "..pktCount) + hist:print() + hist:save("histogram-newconf.csv") +end + diff --git a/examples/l2-cbr-load-latency.lua b/examples/l2-cbr-load-latency.lua new file mode 100644 index 000000000..7bbeb5cc7 --- /dev/null +++ b/examples/l2-cbr-load-latency.lua @@ -0,0 +1,101 @@ +local mg = require "moongen" +local memory = require "memory" +local ts = require "timestamping" +local device = require "device" +local stats = require "stats" +local timer = require "timer" +local histogram = require "histogram" +local log = require "log" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Generates traffic based on a poisson process with CRC-based rate control.") + parser:argument("txDev", "Device to transmit from."):args(1):convert(tonumber) + parser:argument("rxDev", "Device to receive from."):args(1):convert(tonumber) + parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2) + parser:option("-s --size", "Packet size to use (min=60, max~~1500)"):args(1):default(60) + parser:option("-n --numpackets", "Number of packets to sample (default = 0 = run forever)"):args(1):default(0):convert(tonumber) + parser:option("-w --maxwait", "Max time (in ms) to wait got timer packets to come back (default=100)"):args(1):default(100):convert(tonumber) +end + +function master(args) + local txDev = device.config({port = args.txDev, txQueues = 2, rxQueues = 2}) + local rxDev = device.config({port = args.rxDev, txQueues = 2, rxQueues = 2}) + PKT_SIZE = math.max(60, tonumber(args.size)) + print("using packet size "..PKT_SIZE) + device.waitForLinks() + + mg.startTask("loadSlave", txDev, rxDev, txDev:getTxQueue(0), args.rate, PKT_SIZE) + mg.startTask("timerSlave", txDev:getTxQueue(1), rxDev:getRxQueue(1), PKT_SIZE, args.numpackets, args.maxwait) + mg.waitForTasks() +end + +function loadSlave(txDev, rxDev, queue, rate, size) + + -- doing crc rate control requires us to know the link speed. + -- it is given in Mbps, just like the rate argument + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + local rxStats = stats:newDevRxCounter(rxDev, "plain") + local txStats = stats:newManualTxCounter(txDev, "plain") + while mg.running() do + bufs:alloc(size) + for _, buf in ipairs(bufs) do + -- this script uses Mpps instead of Mbit (like the other scripts) + --buf:setDelay((10^10 / 8 / (rate * 10^6) - size - 24)) + buf:setDelay((size+24) * (linkspeed/rate - 1) ) + --buf:setRate(rate*10) -- rate in Mpps on gigabit ethernet + -- from crc-ratecontrol.lua: + -- delay The time to wait before this packet \(in bytes, i.e. 1 == 0.8 nanoseconds on 10 GbE\) + -- self.udata64 = 10^10 / 8 / (rate * 10^6) - self.pkt_len - 24 + -- key code from stats.lua (accounting for preamble + inter-packet gap = 8+12?): + -- local mpps = (pkts - self.total) / elapsed / 10^6 + -- local mbit = (bytes - self.totalBytes) / elapsed / 10^6 * 8 + -- local wireRate = mbit + (mpps * 20 * 8) + end + txStats:updateWithSize(queue:sendWithDelay(bufs), size) + rxStats:update() + --txStats:update() + end + rxStats:finalize() + txStats:finalize() +end + + +-- in order to pass a maxWait parameter to timestamper:measureLatency we need +-- to also pass a packet modifying function. This one does nothing. +function dummyModifier(buf) + return false +end + + +function timerSlave(txQueue, rxQueue, size, numpackets, maxWait) + numpackets = numpackets or 0 + local timestamper = ts:newTimestamper(txQueue, rxQueue) + local hist = histogram:new() + -- wait for a second to give the other task a chance to start + mg.sleepMillis(1000) + local rateLimiter = timer:new(0.001) + local pktCount = 0 + while mg.running() and (numpackets == 0 or pktCount < numpackets) do + rateLimiter:reset() + local measurement, num = timestamper:measureLatency(size, dummyModifier, maxWait) + --print(measurement, num) + hist:update(measurement) + pktCount = pktCount + 1 + rateLimiter:busyWait() + end + mg.stop() + print("latency measurements: "..pktCount) + hist:print() + hist:save("histogram.csv") +end + diff --git a/examples/l2-cbr-load-purelatency.lua b/examples/l2-cbr-load-purelatency.lua new file mode 100644 index 000000000..05e39f8fb --- /dev/null +++ b/examples/l2-cbr-load-purelatency.lua @@ -0,0 +1,97 @@ +local mg = require "moongen" +local memory = require "memory" +local ts = require "timestamping" +local device = require "device" +local stats = require "stats" +local timer = require "timer" +local histogram = require "histogram" +local log = require "log" + +local PKT_SIZE = 60 + +-- +-- This version does not run the CBR generator thread, so the only thing going out/in in the timestamp frames. +-- + + +function configure(parser) + parser:description("Generates traffic based on a poisson process with CRC-based rate control.") + parser:argument("txDev", "Device to transmit from."):args(1):convert(tonumber) + parser:argument("rxDev", "Device to receive from."):args(1):convert(tonumber) + parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2) + parser:option("-s --size", "Packet size to use (min=60, max~~1500)"):args(1):default(60) + parser:option("-n --numpackets", "Number of packets to sample (default = 0 = run forever)"):args(1):default(0):convert(tonumber) +end + +function master(args) + local txDev = device.config({port = args.txDev, txQueues = 2, rxQueues = 2}) + local rxDev = device.config({port = args.rxDev, txQueues = 2, rxQueues = 2}) + PKT_SIZE = math.max(60, tonumber(args.size)) + print("using packet size "..PKT_SIZE) + device.waitForLinks() + + --mg.startTask("loadSlave", txDev, rxDev, txDev:getTxQueue(0), args.rate, PKT_SIZE) + mg.startTask("timerSlave", txDev:getTxQueue(1), rxDev:getRxQueue(1), PKT_SIZE, args.numpackets) + mg.waitForTasks() +end + +function loadSlave(txDev, rxDev, queue, rate, size) + + -- doing crc rate control requires us to know the link speed. + -- it is given in Mbps, just like the rate argument + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local mem = memory.createMemPool(function(buf) + buf:getEthernetPacket():fill{ + ethType = 0x1234 + } + end) + local bufs = mem:bufArray() + local rxStats = stats:newDevRxCounter(rxDev, "plain") + local txStats = stats:newManualTxCounter(txDev, "plain") + while mg.running() do + bufs:alloc(size) + for _, buf in ipairs(bufs) do + -- this script uses Mpps instead of Mbit (like the other scripts) + --buf:setDelay((10^10 / 8 / (rate * 10^6) - size - 24)) + buf:setDelay((size+24) * (linkspeed/rate - 1) ) + --buf:setRate(rate*10) -- rate in Mpps on gigabit ethernet + -- from crc-ratecontrol.lua: + -- delay The time to wait before this packet \(in bytes, i.e. 1 == 0.8 nanoseconds on 10 GbE\) + -- self.udata64 = 10^10 / 8 / (rate * 10^6) - self.pkt_len - 24 + -- key code from stats.lua (accounting for preamble + inter-packet gap = 8+12?): + -- local mpps = (pkts - self.total) / elapsed / 10^6 + -- local mbit = (bytes - self.totalBytes) / elapsed / 10^6 * 8 + -- local wireRate = mbit + (mpps * 20 * 8) + end + txStats:updateWithSize(queue:sendWithDelay(bufs), size) + rxStats:update() + --txStats:update() + end + rxStats:finalize() + txStats:finalize() +end + +function timerSlave(txQueue, rxQueue, size, numpackets) + numpackets = numpackets or 0 + local timestamper = ts:newTimestamper(txQueue, rxQueue) + local hist = histogram:new() + -- wait for a second to give the other task a chance to start + mg.sleepMillis(1000) + local rateLimiter = timer:new(0.001) + local pktCount = 0 + while mg.running() and (numpackets == 0 or pktCount < numpackets) do + rateLimiter:reset() + local measurement, num = timestamper:measureLatency(size) + --print(measurement, num) + hist:update(measurement) + pktCount = pktCount + 1 + rateLimiter:busyWait() + end + mg.stop() + print("latency measurements: "..pktCount) + hist:print() + hist:save("histogram.csv") +end + diff --git a/examples/l2-forward-bsring-hybrid-latency-rate.lua b/examples/l2-forward-bsring-hybrid-latency-rate.lua new file mode 100644 index 000000000..10268ce97 --- /dev/null +++ b/examples/l2-forward-bsring-hybrid-latency-rate.lua @@ -0,0 +1,212 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local stats = require "stats" +local log = require "log" +local limiter = require "software-ratecontrol" +local pipe = require "pipe" +local ffi = require "ffi" +local libmoon = require "libmoon" +local histogram = require "histogram" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Forward traffic between interfaces with moongen rate control") + parser:option("-d --dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + --parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:option("-r --rate", "Forwarding rates in Mbps (two values for two links)"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + parser:option("-l --latency", "Fixed emulated latency (in ms) on the link."):args(2):convert(tonumber):default({0,0}) + parser:option("-x --xlatency", "Extra exponentially distributed latency, in addition to the fixed latency (in ms)."):args(2):convert(tonumber):default({0,0}) + parser:option("-q --queuedepth", "Maximum number of bytes to hold in the delay line"):args(2):convert(tonumber):default({0,0}) + parser:option("-o --loss", "Rate of packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-c --concealedloss", "Rate of concealed packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-u --catchuprate", "After a concealed loss, this rate will apply to the backed-up frames."):args(2):convert(tonumber):default({0,0}) + return parser:parse() +end + + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {}, + rxDescs = 4096, + dropEnable = true, + disableOffloads = true + } + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- create the ring buffers + -- should set the size here, based on the line speed and latency, and maybe desired queue depth + local qdepth1 = args.queuedepth[1] + if qdepth1 < 1 then + qdepth1 = math.floor((args.latency[1] * args.rate[1] * 1000)/8) + end + local qdepth2 = args.queuedepth[2] + if qdepth2 < 1 then + qdepth2 = math.floor((args.latency[2] * args.rate[2] * 1000)/8) + end + local ring1 = pipe:newBytesizedRing(qdepth1) + local ring2 = pipe:newBytesizedRing(qdepth2) + + -- start the forwarding tasks + for i = 1, args.threads do + mg.startTask("forward", ring1, args.dev[1]:getTxQueue(i - 1), args.dev[1], args.rate[1], args.latency[1], args.xlatency[1], args.loss[1], args.concealedloss[1], args.catchuprate[1]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("forward", ring2, args.dev[2]:getTxQueue(i - 1), args.dev[2], args.rate[2], args.latency[2], args.xlatency[2], args.loss[2], args.concealedloss[2], args.catchuprate[2]) + end + end + + -- start the receiving/latency tasks + for i = 1, args.threads do + mg.startTask("receive", ring1, args.dev[2]:getRxQueue(i - 1), args.dev[2]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("receive", ring2, args.dev[1]:getRxQueue(i - 1), args.dev[1]) + end + end + + mg.waitForTasks() +end + + +function receive(ring, rxQueue, rxDev) + local bufs = memory.createBufArray() + local count = 0 + local count_hist = histogram:new() + local ringsize_hist = histogram:new() + local ringbytes_hist = histogram:new() + local ts = 0 + while mg.running() do + count = rxQueue:recv(bufs) + count_hist:update(count) + for iix=1,count do + local buf = bufs[iix] + --if buf:hasTimestamp() then + -- ts = buf:getTimestamp() + --end + ts = limiter:get_tsc_cycles() + buf.udata64 = ts + end + if count > 0 then + local num_added = pipe:sendToBytesizedRing(ring.ring, bufs, count) + if (num_added < count) then + print("failed to add packets to bsring "..num_added.." "..count) + end + ringsize_hist:update(pipe:countBytesizedRing(ring.ring)) + ringbytes_hist:update(pipe:bytesusedBytesizedRing(ring.ring)) + --print("ring count/usage: ",pipe:countBytesizedRing(ring.ring),pipe:bytesusedBytesizedRing(ring.ring),count) + end + end + count_hist:print() + count_hist:save("rxq-pkt-count-distribution-histogram-"..rxDev["id"]..".csv") + ringsize_hist:print() + ringsize_hist:save("rxq-ringsize-distribution-histogram-"..rxDev["id"]..".csv") + ringbytes_hist:print() + ringbytes_hist:save("rxq-ringbytes-distribution-histogram-"..rxDev["id"]..".csv") +end + + +function forward(ring, txQueue, txDev, rate, latency, xlatency, lossrate, clossrate, catchuprate) + print("forward with rate "..rate.." and latency "..latency.." and loss rate "..lossrate.." and clossrate "..clossrate.." and catchuprate "..catchuprate) + local numThreads = 1 + + local count_hist = histogram:new() + local size_hist = histogram:new() + + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local tsc_hz = libmoon:getCyclesFrequency() + local tsc_hz_ms = tsc_hz / 1000 + local tsc_hz_us = tsc_hz / 1000000 + print("tsc_hz = "..tsc_hz) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = memory.createBufArray() --memory:bufArray() --(128) + local count = 0 + + -- when there is a concealed loss, the backed-up packets can + -- catch-up at line rate + local catchup_mode = false + + while mg.running() do + -- receive one or more packets from the queue + count = pipe:recvFromBytesizedRing(ring.ring, bufs, 1) + count_hist:update(count) + + for iix=1,count do + local buf = bufs[iix] + + -- get the buf's arrival timestamp and compare to current time + local arrival_timestamp = buf.udata64 + + -- emulate extra exponential random delay + local extraDelay = 0.0 + if (xlatency > 0) then + extraDelay = -math.log(math.random())*xlatency + end + + -- emulate concealed losses + local closses = 0 + while (math.random() < clossrate) do + closses = closses + 1 + if (catchuprate > 0) then + catchup_mode = true + --print "entering catchup mode!" + end + end + local send_time = arrival_timestamp + (((closses+1)*latency + extraDelay) * tsc_hz_ms) + + local cur_time = limiter:get_tsc_cycles() + -- spin/wait until it is time to send this frame + -- this assumes frame order is preserved + + while cur_time < send_time do + catchup_mode = false + if not mg.running() then + return + end + cur_time = limiter:get_tsc_cycles() + end + + local pktSize = buf.pkt_len + 24 + size_hist:update(buf.pkt_len) + if (catchup_mode) then + --print "operating in catchup mode!" + --print("catchup setting delay to "..((pktSize) * (linkspeed/rate - 1)).." on buf ",buf) + buf:setDelay((pktSize) * (linkspeed/catchuprate - 1)) + else + buf:setDelay((pktSize) * (linkspeed/rate - 1)) + end + + --if count > 0 then + if count > 0 then + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + txQueue:sendWithDelayLoss(bufs, rate * numThreads, lossrate, count) + end + end -- if count > 0 then + end -- while mg.running() do + count_hist:print() + count_hist:save("pkt-count-distribution-histogram-"..tonumber(txDev["id"])..".csv") + size_hist:print() + size_hist:save("pkt-size-distribution-histogram-"..tonumber(txDev["id"])..".csv") +end + + + + + + + + diff --git a/examples/l2-forward-bstxring-hybrid-latency-rate.lua b/examples/l2-forward-bstxring-hybrid-latency-rate.lua new file mode 100644 index 000000000..1e465a1b8 --- /dev/null +++ b/examples/l2-forward-bstxring-hybrid-latency-rate.lua @@ -0,0 +1,214 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local stats = require "stats" +local log = require "log" +local limiter = require "software-ratecontrol" +local pipe = require "pipe" +local ffi = require "ffi" +local libmoon = require "libmoon" +local histogram = require "histogram" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Forward traffic between interfaces with moongen rate control") + parser:option("-d --dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + --parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:option("-r --rate", "Forwarding rates in Mbps (two values for two links)"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + parser:option("-l --latency", "Fixed emulated latency (in ms) on the link."):args(2):convert(tonumber):default({0,0}) + parser:option("-x --xlatency", "Extra exponentially distributed latency, in addition to the fixed latency (in ms)."):args(2):convert(tonumber):default({0,0}) + parser:option("-q --queuedepth", "Maximum number of bytes to hold in the delay line"):args(2):convert(tonumber):default({0,0}) + parser:option("-o --loss", "Rate of packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-c --concealedloss", "Rate of concealed packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-u --catchuprate", "After a concealed loss, this rate will apply to the backed-up frames."):args(2):convert(tonumber):default({0,0}) + return parser:parse() +end + + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + print(i,dev) + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {}, + rxDescs = 4096, + dropEnable = true, + disableOffloads = true + } + print(args.dev[i]["id"]) + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- create the ring buffers + -- should set the size here, based on the line speed and latency, and maybe desired queue depth + local qdepth1 = args.queuedepth[1] + if qdepth1 < 1 then + qdepth1 = math.floor((args.latency[1] * args.rate[1] * 1000)/8) + end + local qdepth2 = args.queuedepth[2] + if qdepth2 < 1 then + qdepth2 = math.floor((args.latency[2] * args.rate[2] * 1000)/8) + end + local ring1 = pipe:newBytesizedtxRing(qdepth1, -1, args.dev[1]["id"]) + local ring2 = pipe:newBytesizedtxRing(qdepth2, -1, args.dev[2]["id"]) + + -- start the forwarding tasks + for i = 1, args.threads do + mg.startTask("forward", ring1, args.dev[1]:getTxQueue(i - 1), args.dev[1], args.rate[1], args.latency[1], args.xlatency[1], args.loss[1], args.concealedloss[1], args.catchuprate[1]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("forward", ring2, args.dev[2]:getTxQueue(i - 1), args.dev[2], args.rate[2], args.latency[2], args.xlatency[2], args.loss[2], args.concealedloss[2], args.catchuprate[2]) + end + end + + -- start the receiving/latency tasks + for i = 1, args.threads do + mg.startTask("receive", ring1, args.dev[2]:getRxQueue(i - 1), args.dev[2]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("receive", ring2, args.dev[1]:getRxQueue(i - 1), args.dev[1]) + end + end + + mg.waitForTasks() +end + + +function receive(ring, rxQueue, rxDev) + local bufs = memory.createBufArray() + local count = 0 + local count_hist = histogram:new() + local ringsize_hist = histogram:new() + local ringbytes_hist = histogram:new() + local ts = 0 + while mg.running() do + count = rxQueue:recv(bufs) + --count_hist:update(count) + for iix=1,count do + local buf = bufs[iix] + --if buf:hasTimestamp() then + -- ts = buf:getTimestamp() + --end + ts = limiter:get_tsc_cycles() + buf.udata64 = ts + end + if count > 0 then + local num_added = pipe:sendToBytesizedtxRing(ring.ring, bufs, count) + if (num_added < count) then + --print("failed to add packets to bstxring "..num_added.." "..count) + end + --ringsize_hist:update(pipe:countBytesizedRing(ring.ring)) + --ringbytes_hist:update(pipe:bytesusedBytesizedRing(ring.ring)) + --print("ring count/usage: ",pipe:countBytesizedRing(ring.ring),pipe:bytesusedBytesizedRing(ring.ring),count) + end + end + count_hist:print() + count_hist:save("rxq-pkt-count-distribution-histogram-"..rxDev["id"]..".csv") + ringsize_hist:print() + ringsize_hist:save("rxq-ringsize-distribution-histogram-"..rxDev["id"]..".csv") + ringbytes_hist:print() + ringbytes_hist:save("rxq-ringbytes-distribution-histogram-"..rxDev["id"]..".csv") +end + + +function forward(ring, txQueue, txDev, rate, latency, xlatency, lossrate, clossrate, catchuprate) + print("forward with rate "..rate.." and latency "..latency.." and loss rate "..lossrate.." and clossrate "..clossrate.." and catchuprate "..catchuprate) + local numThreads = 1 + + local count_hist = histogram:new() + local size_hist = histogram:new() + + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local tsc_hz = libmoon:getCyclesFrequency() + local tsc_hz_ms = tsc_hz / 1000 + local tsc_hz_us = tsc_hz / 1000000 + print("tsc_hz = "..tsc_hz) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = memory.createBufArray() --memory:bufArray() --(128) + local count = 0 + + -- when there is a concealed loss, the backed-up packets can + -- catch-up at line rate + local catchup_mode = false + + while mg.running() do + -- receive one or more packets from the queue + count = pipe:recvFromBytesizedtxRing(ring.ring, bufs, 1) + --count_hist:update(count) + + for iix=1,count do + local buf = bufs[iix] + + -- get the buf's arrival timestamp and compare to current time + local arrival_timestamp = buf.udata64 + + -- emulate extra exponential random delay + local extraDelay = 0.0 + if (xlatency > 0) then + extraDelay = -math.log(math.random())*xlatency + end + + -- emulate concealed losses + local closses = 0 + while (math.random() < clossrate) do + closses = closses + 1 + if (catchuprate > 0) then + catchup_mode = true + --print "entering catchup mode!" + end + end + local send_time = arrival_timestamp + (((closses+1)*latency + extraDelay) * tsc_hz_ms) + + local cur_time = limiter:get_tsc_cycles() + -- spin/wait until it is time to send this frame + -- this assumes frame order is preserved + + while cur_time < send_time do + catchup_mode = false + if not mg.running() then + return + end + cur_time = limiter:get_tsc_cycles() + end + + local pktSize = buf.pkt_len + 24 + --size_hist:update(buf.pkt_len) + if (catchup_mode) then + --print "operating in catchup mode!" + --print("catchup setting delay to "..((pktSize) * (linkspeed/rate - 1)).." on buf ",buf) + buf:setDelay((pktSize) * (linkspeed/catchuprate - 1)) + else + buf:setDelay((pktSize) * (linkspeed/rate - 1)) + end + + --if count > 0 then + if count > 0 then + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + txQueue:sendWithDelayLoss(bufs, rate * numThreads, lossrate, count) + end + end -- if count > 0 then + end -- while mg.running() do + count_hist:print() + count_hist:save("pkt-count-distribution-histogram-"..tonumber(txDev["id"])..".csv") + size_hist:print() + size_hist:save("pkt-size-distribution-histogram-"..tonumber(txDev["id"])..".csv") +end + + + + + + + + diff --git a/examples/l2-forward-hybrid-latency-rate-loss.lua b/examples/l2-forward-hybrid-latency-rate-loss.lua new file mode 100644 index 000000000..88b1e39f0 --- /dev/null +++ b/examples/l2-forward-hybrid-latency-rate-loss.lua @@ -0,0 +1,195 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local stats = require "stats" +local log = require "log" +local limiter = require "software-ratecontrol" +local pipe = require "pipe" +local ffi = require "ffi" +local libmoon = require "libmoon" +local histogram = require "histogram" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Forward traffic between interfaces with moongen rate control") + parser:option("-d --dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + --parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:option("-r --rate", "Forwarding rates in Mbps (two values for two links)"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + parser:option("-l --latency", "Fixed emulated latency (in ms) on the link."):args(2):convert(tonumber):default({0,0}) + parser:option("-x --xlatency", "Extra exponentially distributed latency, in addition to the fixed latency (in ms)."):args(2):convert(tonumber):default({0,0}) + parser:option("-q --queuedepth", "Maximum number of packets to hold in the delay line"):args(2):convert(tonumber):default({0,0}) + parser:option("-o --loss", "Rate of packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-c --concealedloss", "Rate of concealed packet drops"):args(2):convert(tonumber):default({0,0}) + parser:option("-u --catchuprate", "After a concealed loss, this rate will apply to the backed-up frames."):args(2):convert(tonumber):default({0,0}) + return parser:parse() +end + + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {}, + --rxDescs = 4096, + dropEnable = true, + disableOffloads = true + } + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- create the ring buffers + -- should set the size here, based on the line speed and latency, and maybe desired queue depth + local qdepth1 = args.queuedepth[1] + if qdepth1 < 1 then + qdepth1 = math.floor((args.latency[1] * args.rate[1] * 1000)/672) + end + local qdepth2 = args.queuedepth[2] + if qdepth2 < 1 then + qdepth2 = math.floor((args.latency[2] * args.rate[2] * 1000)/672) + end + local ring1 = pipe:newPktsizedRing(qdepth1) + local ring2 = pipe:newPktsizedRing(qdepth2) + + -- start the forwarding tasks + for i = 1, args.threads do + mg.startTask("forward", ring1, args.dev[1]:getTxQueue(i - 1), args.dev[1], args.rate[1], args.latency[1], args.xlatency[1], args.loss[1], args.concealedloss[1], args.catchuprate[1]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("forward", ring2, args.dev[2]:getTxQueue(i - 1), args.dev[2], args.rate[2], args.latency[2], args.xlatency[2], args.loss[2], args.concealedloss[2], args.catchuprate[2]) + end + end + + -- start the receiving/latency tasks + for i = 1, args.threads do + mg.startTask("receive", ring1, args.dev[2]:getRxQueue(i - 1), args.dev[2]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("receive", ring2, args.dev[1]:getRxQueue(i - 1), args.dev[1]) + end + end + + mg.waitForTasks() +end + + +function receive(ring, rxQueue, rxDev) + --print("receive thread...") + + local bufs = memory.createBufArray() + local count = 0 + local count_hist = histogram:new() + local ringsize_hist = histogram:new() + local ringbytes_hist = histogram:new() + while mg.running() do + count = rxQueue:recv(bufs) + count_hist:update(count) + --print("receive thread count="..count) + for iix=1,count do + local buf = bufs[iix] + local ts = limiter:get_tsc_cycles() + buf.udata64 = ts + end + if count > 0 then + pipe:sendToPktsizedRing(ring.ring, bufs, count) + --print("ring count: ",pipe:countPacketRing(ring.ring)) + ringsize_hist:update(pipe:countPktsizedRing(ring.ring)) + end + end + count_hist:print() + count_hist:save("rxq-pkt-count-distribution-histogram-"..rxDev["id"]..".csv") + ringsize_hist:print() + ringsize_hist:save("rxq-ringsize-distribution-histogram-"..rxDev["id"]..".csv") +end + +function forward(ring, txQueue, txDev, rate, latency, xlatency, lossrate, clossrate, catchuprate) + print("forward with rate "..rate.." and latency "..latency.." and loss rate "..lossrate.." and clossrate "..clossrate.." and catchuprate "..catchuprate) + local numThreads = 1 + + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local tsc_hz = libmoon:getCyclesFrequency() + local tsc_hz_ms = tsc_hz / 1000 + print("tsc_hz = "..tsc_hz) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = memory.createBufArray() --memory:bufArray() --(128) + local count = 0 + + -- when there is a concealed loss, the backed-up packets can + -- catch-up at line rate + local catchup_mode = false + + + while mg.running() do + -- receive one or more packets from the queue + --local count = rxQueue:recv(bufs) + --print("calling pipe:recvFromPktsizedRing(ring.ring, bufs)") + count = pipe:recvFromPktsizedRing(ring.ring, bufs, 1) + --print("call returned.") + for iix=1,count do + local buf = bufs[iix] + + -- get the buf's arrival timestamp and compare to current time + --local arrival_timestamp = buf:getTimestamp() + local arrival_timestamp = buf.udata64 + local extraDelay = 0.0 + if (xlatency > 0) then + extraDelay = -math.log(math.random())*xlatency + end + -- emulate concealed losses + local closses = 0 + while (math.random() < clossrate) do + closses = closses + 1 + if (catchuprate > 0) then + catchup_mode = true + --print "entering catchup mode!" + end + end + local send_time = arrival_timestamp + (((closses+1)*latency + extraDelay) * tsc_hz_ms) + local cur_time = limiter:get_tsc_cycles() + --print("timestamps", arrival_timestamp, send_time, cur_time) + -- spin/wait until it is time to send this frame + -- this assumes frame order is preserved + while limiter:get_tsc_cycles() < send_time do + catchup_mode = false + if not mg.running() then + return + end + end + + local pktSize = buf.pkt_len + 24 + if (catchup_mode) then + --print "operating in catchup mode!" + buf:setDelay((pktSize) * (linkspeed/catchuprate - 1)) + else + buf:setDelay((pktSize) * (linkspeed/rate - 1)) + end + end + + --print("count="..tostring(count)) + + --if count > 0 then + if count > 0 then + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + txQueue:sendWithDelayLoss(bufs, rate * numThreads, lossrate, count) + --print("sendWithDelay() returned") + end + end +end + + + + + + + + diff --git a/examples/l2-forward-hybrid-latency-rate.lua b/examples/l2-forward-hybrid-latency-rate.lua new file mode 100644 index 000000000..876725033 --- /dev/null +++ b/examples/l2-forward-hybrid-latency-rate.lua @@ -0,0 +1,171 @@ +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local stats = require "stats" +local hist = require "histogram" +local log = require "log" +local limiter = require "software-ratecontrol" +local pipe = require "pipe" +local ffi = require "ffi" +local libmoon = require "libmoon" + +local PKT_SIZE = 60 + +function configure(parser) + parser:description("Forward traffic between interfaces with moongen rate control") + parser:option("-d --dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + --parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:option("-r --rate", "Forwarding rates in Mbps (two values for two links)"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + parser:option("-l --latency", "Fixed emulated latency (in ms) on the link."):args(2):convert(tonumber):default(0) + parser:option("-q --queuedepth", "Maximum number of packets to hold in the delay line"):args(2):convert(tonumber):default(0) + return parser:parse() +end + + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {}, + rxDescs = 4096, + dropEnable = true, + disableOffloads = true + } + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- create the ring buffers + -- should set the size here, based on the line speed and latency, and maybe desired queue depth + local qdepth1 = args.queuedepth[1] + if qdepth1 < 1 then + qdepth1 = math.floor((args.latency[1] * args.rate[1] * 1000)/672) + end + local qdepth2 = args.queuedepth[2] + if qdepth2 < 1 then + qdepth2 = math.floor((args.latency[2] * args.rate[2] * 1000)/672) + end + local ring1 = pipe:newPacketRing(qdepth1) + local ring2 = pipe:newPacketRing(qdepth2) + + -- start the forwarding tasks + for i = 1, args.threads do + mg.startTask("forward", ring1, args.dev[1]:getTxQueue(i - 1), args.dev[1], args.rate[1], args.latency[1]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("forward", ring2, args.dev[2]:getTxQueue(i - 1), args.dev[2], args.rate[2], args.latency[2]) + end + end + + -- start the receiving/latency tasks + for i = 1, args.threads do + mg.startTask("receive", ring1, args.dev[2]:getRxQueue(i - 1), args.dev[2]) + if args.dev[1] ~= args.dev[2] then + mg.startTask("receive", ring2, args.dev[1]:getRxQueue(i - 1), args.dev[1]) + end + end + + mg.waitForTasks() +end + + +function receive(ring, rxQueue, rxDev) + --print("receive thread...") + + local bufs = memory.createBufArray() + local count = 0 + while mg.running() do + count = rxQueue:recv(bufs) + --print("receive thread count="..count) + for _, buf in ipairs(bufs) do + if (buf ~= nil) then + --if buf:hasTimestamp() then + -- local ts = buf:getTimestamp() + -- buf.udata64 = ts + -- print("set timestamp to getTimestamp=", ts) + --else + local ts = limiter:get_tsc_cycles() + buf.udata64 = ts + --print("set timestamp to get_tsc_cycles=", ts, count) + --end + end + end + if count > 0 then + pipe:sendToPacketRing(ring.ring, bufs, count) + --print("ring count: ",pipe:countPacketRing(ring.ring)) + end + end +end + + +function forward(ring, txQueue, txDev, rate, latency) + print("forward with rate "..rate.." and latency "..latency) + local numThreads = 1 + + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + local tsc_hz = libmoon:getCyclesFrequency() + local tsc_hz_ms = tsc_hz / 1000 + print("tsc_hz = "..tsc_hz) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = memory.createBufArray() --memory:bufArray() --(128) + local count = 0 + + while mg.running() do + -- receive one or more packets from the queue + --local count = rxQueue:recv(bufs) + --print("calling pipe:recvFromPacketRing(ring.ring, bufs)") + count = pipe:recvFromPacketRing(ring.ring, bufs, 1) + --print("call returned.") + if count > 0 then + --print("count=", count) + + for _, buf in ipairs(bufs) do + + if (buf ~= nil) then + --print("buf ~= nil") + -- get the buf's arrival timestamp and compare to current time + --local arrival_timestamp = buf:getTimestamp() + local arrival_timestamp = buf.udata64 + local send_time = arrival_timestamp + (latency * tsc_hz_ms) + local cur_time = limiter:get_tsc_cycles() + --print("timestamps", arrival_timestamp, send_time, cur_time) + -- spin/wait until it is time to send this frame + -- this assumes frame order is preserved + while limiter:get_tsc_cycles() < send_time do + if not mg.running() then + return + end + end + + local pktSize = buf.pkt_len + 24 + buf:setDelay((pktSize) * (linkspeed/rate - 1) ) + end + end + --print("count="..tostring(count)) + + --if count > 0 then + --if count > 0 then + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + txQueue:sendWithDelay(bufs, rate * numThreads, count) + --print("sendWithDelay() returned") + end + end +end + + + + + + + + diff --git a/examples/l2-forward-rate-crc.lua b/examples/l2-forward-rate-crc.lua new file mode 100755 index 000000000..3f95058ce --- /dev/null +++ b/examples/l2-forward-rate-crc.lua @@ -0,0 +1,136 @@ +--- Forward packets between two ports +-- local lm = require "libmoon" +local mg = require "moongen" +local memory = require "memory" +local device = require "device" +local ts = require "timestamping" +local histogram = require "histogram" +local stats = require "stats" +local log = require "log" +local timer = require "timer" + +-- local limiter = require "software-ratecontrol" + +function configure(parser) + parser:description("Forward traffic between interfaces with moongen rate control") + parser:argument("dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + --parser:option("-r --rate", "Transmit rate in Mpps."):args(1):default(2):convert(tonumber) + parser:argument("rate", "Forwarding rates in Mbps (two values for two links)"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + return parser:parse() +end + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {}, + rxDescs = 4096, + dropEnable = true, + disableOffloads = true + } + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- start forwarding tasks + for i = 1, args.threads do + print("dev is ",tonumber(args.dev[1]["id"])) + --rateLimiter1 = limiter:new(args.dev[2]:getTxQueue(i - 1), "cbr", 1 / args.rate[1] * 1000) + mg.startTask("forward", args.dev[1]:getRxQueue(i - 1), args.dev[2]:getTxQueue(i - 1), args.dev[2], args.rate[1]) + -- bidirectional fowarding only if two different devices where passed + if args.dev[1] ~= args.dev[2] then + mg.startTask("forward", args.dev[2]:getRxQueue(i - 1), args.dev[1]:getTxQueue(i - 1), args.dev[1], args.rate[2]) + end + end + mg.waitForTasks() +end + +function forward(rxQueue, txQueue, txDev, rate) + print("forward with rate "..rate) + local ETH_DST = "11:12:13:14:15:16" + local pattern = "cbr" + local numThreads = 1 + + local count_hist = histogram:new() + local size_hist = histogram:new() + + local linkspeed = txDev:getLinkStatus().speed + print("linkspeed = "..linkspeed) + + -- larger batch size is useful when sending it through a rate limiter + local bufs = memory.createBufArray() --memory:bufArray() --(128) + local dist = pattern == "poisson" and poissonDelay or function(x) return x end + while mg.running() do + -- receive one or more packets from the queue + local count = rxQueue:recv(bufs) + + count_hist:update(count) + + -- send out all received bufs on the other queue + -- the bufs are free'd implicitly by this function + -- txQueue:sendN(bufs, count) + + -- There is a problem here when we are just forwarding packets + -- if the link is idle, when the packet arrives, there should be no delay + -- we need to keep track of the size and tx time of the last pkt sent, and then + -- look up the current time in order to compute the remaining delay to add to the current packet + -- the approach here only applies when we have an unlimited stream of packets + -- should all that be implemented here, or in sendWithDelay? + -- + -- The answer is to change the way CRC rate control works + -- When a packet is to be sent, it should be sent immediately, but followed up with + -- a series of bad packets that take up the extra time the packet /should/ have taken + -- at the desired emulated rate. + -- This approach will cause some issues when the desired speed is between 500-1000, + -- and the good packets are small. In that case the size of the bad packet we need may + -- often be less than the min frame size. As long as the rate is less than 1/2 line + -- speed, this should never be a problem, though. + -- + -- Coming back to this later, and I disagree with what I wrote before. The dummy + -- packets shouldbe sent out before the actual packet. Otherwise the good packet + -- could be complete received before it would have even completed sending at the + -- reduced rate. + for iix=1,count do + local buf = bufs[iix] + if (buf ~= nil) then -- should not be necessary + local pktSize = buf.pkt_len + 24 + --print("forwarding packet of size ",pktSize) + --buf:setDelay(dist(10^10 / numThreads / 8 / (rate * 10^6) - pktSize - 24)) + --buf:setDelay((pktSize+24) * (linkspeed/rate - 1) ) + size_hist:update(buf.pkt_len) + buf:setDelay((pktSize) * (linkspeed/rate - 1) ) + end + end + + -- the rate here doesn't affect the result afaict. It's just to help decide the size of the bad pkts + txQueue:sendWithDelay(bufs, rate * numThreads, count) + --txQueue:sendWithDelay(bufs) + end + + count_hist:print() + count_hist:save("pkt-count-distribution-histogram-"..tonumber(txDev["id"])..".csv") + size_hist:print() + size_hist:save("pkt-size-distribution-histogram-"..tonumber(txDev["id"])..".csv") +end + + +function forward2(rxQueue, txQueue, rateLimiter) + -- a bufArray is just a list of buffers that we will use for batched forwarding + local bufs = memory.bufArray() + while mg.running() do -- check if Ctrl+c was pressed + -- receive one or more packets from the queue + local count = rxQueue:recv(bufs) + -- send out all received bufs on the other queue + -- the bufs are free'd implicitly by this function + -- txQueue:sendN(bufs, count) + rateLimiter:send(bufs) + end +end + diff --git a/examples/l2-forward-rate.lua b/examples/l2-forward-rate.lua new file mode 100644 index 000000000..a84ccff81 --- /dev/null +++ b/examples/l2-forward-rate.lua @@ -0,0 +1,57 @@ +--- Forward packets between two ports +local lm = require "libmoon" +local device = require "device" +local stats = require "stats" +local log = require "log" +local memory = require "memory" +local limiter = require "software-ratecontrol" + +function configure(parser) + parser:argument("dev", "Devices to use, specify the same device twice to echo packets."):args(2):convert(tonumber) + parser:argument("rate", "Forwarding rate"):args(2):convert(tonumber) + parser:option("-t --threads", "Number of threads per forwarding direction using RSS."):args(1):convert(tonumber):default(1) + return parser:parse() +end + +function master(args) + -- configure devices + for i, dev in ipairs(args.dev) do + args.dev[i] = device.config{ + port = dev, + txQueues = args.threads, + rxQueues = args.threads, + rssQueues = 0, + rssFunctions = {} + } + end + device.waitForLinks() + + -- print stats + stats.startStatsTask{devices = args.dev} + + -- start forwarding tasks + for i = 1, args.threads do + rateLimiter1 = limiter:new(args.dev[2]:getTxQueue(i - 1), "cbr", 1 / args.rate[1] * 1000) + lm.startTask("forward", args.dev[1]:getRxQueue(i - 1), args.dev[2]:getTxQueue(i - 1), rateLimiter1) + -- bidirectional fowarding only if two different devices where passed + if args.dev[1] ~= args.dev[2] then + rateLimiter2 = limiter:new(args.dev[1]:getTxQueue(i - 1), "cbr", 1 / args.rate[2] * 1000) + lm.startTask("forward", args.dev[2]:getRxQueue(i - 1), args.dev[1]:getTxQueue(i - 1), rateLimiter2) + end + end + lm.waitForTasks() +end + +function forward(rxQueue, txQueue, rateLimiter) + -- a bufArray is just a list of buffers that we will use for batched forwarding + local bufs = memory.bufArray() + while lm.running() do -- check if Ctrl+c was pressed + -- receive one or more packets from the queue + local count = rxQueue:recv(bufs) + -- send out all received bufs on the other queue + -- the bufs are free'd implicitly by this function + -- txQueue:sendN(bufs, count) + rateLimiter:send(bufs) + end +end + diff --git a/examples/timestamps.lua b/examples/timestamps.lua new file mode 100644 index 000000000..0f02ae88a --- /dev/null +++ b/examples/timestamps.lua @@ -0,0 +1,28 @@ +--- This script can be used to measure timestamping precision and accuracy. +-- Connect cables of different length between two ports (or a fiber loopback cable on a single port) to use this. + +local dpdk = require "dpdk" +local ts = require "timestamping" +local device = require "device" +local hist = require "histogram" + +function master(txPort, rxPort) + if not txPort or not rxPort then + errorf("usage: txPort rxPort") + end + local txDev = device.config(txPort) + local rxDev = device.config(rxPort) + device.waitForLinks() + runTest(txDev:getTxQueue(0), rxDev:getRxQueue(0)) +end + +function runTest(txQueue, rxQueue) + local timestamper = ts:newTimestamper(txQueue, rxQueue) + local hist = hist:new() + while dpdk.running() do + hist:update(timestamper:measureLatency()) + end + hist:save("histogram.csv") + hist:print() +end + diff --git a/libmoon b/libmoon index 1dde068b9..719ee3bde 160000 --- a/libmoon +++ b/libmoon @@ -1 +1 @@ -Subproject commit 1dde068b9da43a416a23a371e9394c6d209991cd +Subproject commit 719ee3bde1877418916e69585e16aa7fbb4d590c diff --git a/lua/crc-ratecontrol.lua b/lua/crc-ratecontrol.lua index 3173f1152..ee485d065 100644 --- a/lua/crc-ratecontrol.lua +++ b/lua/crc-ratecontrol.lua @@ -3,6 +3,7 @@ local pkt = require "packet" local memory = require "memory" local ffi = require "ffi" local log = require "log" +local limiter = require "software-ratecontrol" local txQueue = device.__txQueuePrototype local device = device.__devicePrototype @@ -10,6 +11,7 @@ local C = ffi.C ffi.cdef[[ void moongen_send_all_packets_with_delay_bad_crc(uint8_t port_id, uint16_t queue_id, struct rte_mbuf** load_pkts, uint16_t num_pkts, struct mempool* pool, uint32_t min_pkt_size); + void moongen_send_all_packets_with_delay_bad_crc_loss(uint8_t port_id, uint16_t queue_id, struct rte_mbuf** load_pkts, uint16_t num_pkts, struct mempool* pool, uint32_t min_pkt_size, double loss_rate); ]] local mempool @@ -47,6 +49,45 @@ function txQueue:sendWithDelay(bufs, targetRate, n) return bufs.size end +--- Send rate-controlled packets by filling gaps with invalid packets. +-- @param bufs +-- @param targetRate optional, hint to the driver which total rate you are trying to achieve. +-- increases precision at low non-cbr rates +-- @param lossRate bernoulli probability of dropping any particular frame +-- @param n optional, number of packets to send (defaults to full bufs) +function txQueue:sendWithDelayLoss(bufs, targetRate, lossRate, n) + if not self.dev.crcPatch then + log:fatal("Driver does not support disabling the CRC flag. This feature requires a patched driver.") + end + targetRate = targetRate or 14.88 + self.used = true + mempool = mempool or memory.createMemPool{ + func = function(buf) + -- this is tcp packet because the netfpga/OSNT system we use for testing this + -- cannot handle all-zero packets properly (filters get confused) + -- the actual contents of the packets don't matter since their CRC is invalid anways + local pkt = buf:getTcpPacket() + pkt:fill() + end + } + n = n or bufs.size + local avgPacketSize = 1.25 / (targetRate * 2) * 1000 + local minPktSize = self.dev.minPacketSize or 64 + local maxPktRate = self.dev.maxPacketRate or 14.88 + -- allow smaller packets at low rates + if targetRate < maxPktRate / 2 then + minPktSize = minPktSize + 20 + else + minPktSize = math.floor(10 * 10^9 / 10^6 / 8 / maxPktRate) + end + local tsc_hz_us = 2666 + local presend_time = limiter:get_tsc_cycles() + C.moongen_send_all_packets_with_delay_bad_crc_loss(self.id, self.qid, bufs.array, n, mempool, minPktSize, lossRate) + local postsend_time = limiter:get_tsc_cycles() + return bufs.size +end + + --- Set the time to wait before the packet is sent for software rate-controlled send methods. --- @param delay The time to wait before this packet \(in bytes, i.e. 1 == 0.8 nanoseconds on 10 GbE\) function pkt:setDelay(delay) diff --git a/lua/software-ratecontrol.lua b/lua/software-ratecontrol.lua index c1db06b69..28f425a38 100644 --- a/lua/software-ratecontrol.lua +++ b/lua/software-ratecontrol.lua @@ -21,6 +21,7 @@ ffi.cdef[[ void mg_rate_limiter_main_loop(struct rte_ring* ring, uint8_t device, uint16_t queue, uint32_t link_speed, struct limiter_control* ctl); void mg_rate_limiter_cbr_main_loop(struct rte_ring* ring, uint8_t device, uint16_t queue, uint32_t target, struct limiter_control* ctl); void mg_rate_limiter_poisson_main_loop(struct rte_ring* ring, uint8_t device, uint16_t queue, uint32_t target, uint32_t link_speed, struct limiter_control* ctl); + uint64_t mg_rate_limiter_get_tsc_cycles(); ]] local mod = {} @@ -80,6 +81,11 @@ function mod:new(queue, mode, delay) return obj end +--- get the current TSC clock +function mod:get_tsc_cycles() + return C.mg_rate_limiter_get_tsc_cycles() +end + function __MG_RATE_LIMITER_MAIN(ring, devId, qid, mode, delay, speed, ctl) if mode == "cbr" then diff --git a/scripts/normalizer-ipt.pl b/scripts/normalizer-ipt.pl new file mode 100755 index 000000000..db2f0dc13 --- /dev/null +++ b/scripts/normalizer-ipt.pl @@ -0,0 +1,29 @@ +#!/usr/bin/env perl + +use strict; + +if (@ARGV != 1) { + print "usage: $0 \n"; + exit(0); +} + +my $FH; +open($FH, "<$ARGV[0]") || die "ERROR: unable to open $ARGV[0]\n"; +my @lines = <$FH>; +close($FH); + +chomp(@lines); + +my $total = 0; +foreach my $l (@lines) { + my ($latency, $count) = split(/[\,\s]+/,$l); + $total += $count; +} + +foreach my $l (@lines) { + my ($latency, $count) = split(/[\,\s]+/,$l); + print "$latency\t$count\t".((1.0*$count)/(1.0*$total))."\n"; +} + + + diff --git a/scripts/normalizer.pl b/scripts/normalizer.pl new file mode 100755 index 000000000..5b1be77c2 --- /dev/null +++ b/scripts/normalizer.pl @@ -0,0 +1,39 @@ +#!/usr/bin/env perl + +use strict; + +if (@ARGV != 1) { + print "usage: $0 \n"; + exit(0); +} + +my $FH; +open($FH, "<$ARGV[0]") || die "ERROR: unable to open $ARGV[0]\n"; +my @lines = <$FH>; +close($FH); + +chomp(@lines); + +# some scripts record latency in msec and some in usec. +# just guess which it is based on the first latency value. +#my $t_factor = 1.0e-3; +my $t_factor = 1; +my ($latency, $count) = split(/[\,\s]+/,$lines[0]); +#if ($latency > 100000000) { +# $t_factor = 1.0e-6; +#} + +my $total = 0; +foreach my $l (@lines) { + ($latency, $count) = split(/[\,\s]+/,$l); + $total += $count; +} + +print "# latency\tcount\tpdf\t (time-factor: $t_factor)q\n"; +foreach my $l (@lines) { + ($latency, $count) = split(/[\,\s]+/,$l); + print "".($latency*$t_factor)."\t$count\t".((1.0*$count)/(1.0*$total))."\n"; +} + + + diff --git a/scripts/tailizer.pl b/scripts/tailizer.pl new file mode 100755 index 000000000..2e7bebd56 --- /dev/null +++ b/scripts/tailizer.pl @@ -0,0 +1,36 @@ +#!/usr/bin/env perl + +use strict; + +if (@ARGV == 0) { + print "usage: $0 \n"; + exit(0); +} + +foreach my $fname (@ARGV) { + my $FH; + open($FH, "<$fname") || die "ERROR: unable to open $fname for reading\n"; + my @lines = <$FH>; + close($FH); + + chomp(@lines); + + my $total = 0; + foreach my $l (@lines) { + my ($latency, $count) = split(/[\,\s]+/,$l); + $total += $count; + } + + open($FH, ">$fname") || die "ERROR: unable to open $fname for writing\n"; + my $running_sum = 0.0; + foreach my $l (@lines) { + my ($latency, $count) = split(/[\,\s]+/,$l); + $running_sum += $count; + my $cdf = $running_sum/$total; + my $ccdf = ($total - $running_sum)/$total; + print $FH "$latency\t$count\t".((1.0*$count)/(1.0*$total))."\t$cdf\t$ccdf\n"; + } + close($FH); +} + + diff --git a/src/crc-rate-limiter.c b/src/crc-rate-limiter.c index ba9afe152..9a144c2d5 100644 --- a/src/crc-rate-limiter.c +++ b/src/crc-rate-limiter.c @@ -1,8 +1,10 @@ #include +#include #include #include #include #include +#include #include "device.h" @@ -95,3 +97,56 @@ void moongen_send_all_packets_with_delay_bad_crc(uint8_t port_id, uint16_t queue return; } +void moongen_send_all_packets_with_delay_bad_crc_loss(uint8_t port_id, uint16_t queue_id, struct rte_mbuf** load_pkts, uint16_t num_pkts, struct rte_mempool* pool, uint32_t min_pkt_size, double loss_rate) { + const int BUF_SIZE = 128; + struct rte_mbuf* pkts[BUF_SIZE]; + int send_buf_idx = 0; + uint32_t num_bad_pkts = 0; + uint32_t num_bad_bytes = 0; + if (num_pkts>0) { + } + for (uint16_t i = 0; i < num_pkts; i++) { + struct rte_mbuf* pkt; + pkt = load_pkts[i]; + + // desired inter-frame spacing is encoded in the hash 'usr' field + uint32_t delay = (uint32_t) pkt->udata64; + + if (pkt->udata64 > 0x0fffffff) { + printf("WARNING: moongen_send_all_packets_with_delay_bad_crc_loss: bad value in udata64 %lx\n",pkt->udata64); + delay = 0; + } + + // step 1: generate delay-packets + while (delay > 0) { + struct rte_mbuf* bad_pkt = get_delay_pkt_bad_crc(pool, &delay, min_pkt_size); + if (bad_pkt) { + num_bad_pkts++; + // packet size: [MAC, CRC] to be consistent with HW counters + num_bad_bytes += bad_pkt->pkt_len; + pkts[send_buf_idx++] = bad_pkt; + } + if (send_buf_idx >= BUF_SIZE) { + dpdk_send_all_packets(port_id, queue_id, pkts, send_buf_idx); + send_buf_idx = 0; + } + } + // step 2: send the packet + // include random losses + if ((double)rand()/RAND_MAX >= loss_rate) { + pkts[send_buf_idx++] = pkt; + } else { + // if the packet is not going to be sent, we have to free the mbuf. + rte_pktmbuf_free(pkt); + } + if (send_buf_idx >= BUF_SIZE || i + 1 == num_pkts) { // don't forget to send the last batch + dpdk_send_all_packets(port_id, queue_id, pkts, send_buf_idx); + send_buf_idx = 0; + } + } + // atomic as multiple threads may use the same stats register from multiple queues + __sync_fetch_and_add(&bad_pkts_sent[port_id], num_bad_pkts); + __sync_fetch_and_add(&bad_bytes_sent[port_id], num_bad_bytes); + return; +} + diff --git a/src/software-rate-limiter.cpp b/src/software-rate-limiter.cpp index 83104061f..9f99aa557 100644 --- a/src/software-rate-limiter.cpp +++ b/src/software-rate-limiter.cpp @@ -40,6 +40,13 @@ namespace rate_limiter { }; }; static_assert(sizeof(limiter_control) == 16, "struct size mismatch"); + + /* + * We may need to read the current rte_get_tsc_cycles() from lua + */ + static inline uint64_t get_tsc_cycles() { + return rte_get_tsc_cycles(); + } /* * Arbitrary time software rate control main @@ -158,5 +165,9 @@ extern "C" { void mg_rate_limiter_main_loop(rte_ring* ring, uint8_t device, uint16_t queue, uint32_t link_speed, rate_limiter::limiter_control* ctl) { rate_limiter::main_loop(ring, device, queue, link_speed, ctl); } + + uint64_t mg_rate_limiter_get_tsc_cycles() { + return rate_limiter::get_tsc_cycles(); + } }