From 4be6f6d3a40aff2f2663157f754556c19e689b39 Mon Sep 17 00:00:00 2001 From: Alexey Indeev Date: Mon, 6 Oct 2014 13:44:37 -0700 Subject: [PATCH] #136 added poll for when packets are all sent completely --- kernel/linux/example/pcapsend.c | 1 - kernel/linux/include/linux/quick_tx.h | 26 +++++++++++++++++++---- kernel/linux/quick_tx/quick_tx_main.c | 12 +++++++---- kernel/linux/quick_tx/quick_tx_mmap.c | 3 +++ kernel/linux/quick_tx/quick_tx_worker.c | 28 ++++++++++++++++--------- src/tcpreplay.c | 2 -- src/tcpreplay_api.c | 11 ++++++++++ 7 files changed, 62 insertions(+), 21 deletions(-) diff --git a/kernel/linux/example/pcapsend.c b/kernel/linux/example/pcapsend.c index 551b09fba..3394d3f78 100644 --- a/kernel/linux/example/pcapsend.c +++ b/kernel/linux/example/pcapsend.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include diff --git a/kernel/linux/include/linux/quick_tx.h b/kernel/linux/include/linux/quick_tx.h index 8831b6a71..7a67c80d4 100644 --- a/kernel/linux/include/linux/quick_tx.h +++ b/kernel/linux/include/linux/quick_tx.h @@ -22,28 +22,29 @@ #define DMA_COHERENT 1 #ifndef __KERNEL__ + #include #include #include #include #include -#include #include #include #include #include #include -#include #include #include #include #include #include -#include +#include #include #include #include #include +#include + #define __u64 u_int64_t #define __u32 u_int32_t @@ -169,6 +170,7 @@ struct quick_tx_dev { wait_queue_head_t kernel_lookup_q; wait_queue_head_t user_mem_q; wait_queue_head_t user_lookup_q; + wait_queue_head_t user_done_q; struct mutex mtx; #ifdef DMA_COHERENT @@ -210,6 +212,10 @@ extern const struct quick_tx_ops quick_tx_default_ops; extern const struct quick_tx_ops quick_tx_virtio_net_ops; extern const struct quick_tx_ops quick_tx_e1000_ops; +extern int quick_tx_napi_poll_direct(struct quick_tx_dev *dev, struct napi_struct *napi); +extern void quick_tx_set_napi_ops(struct quick_tx_dev *dev); +extern void quick_tx_unset_napi_ops(struct quick_tx_dev *dev); + extern void quick_tx_calc_mbps(struct quick_tx_dev *dev); extern void quick_tx_print_stats(struct quick_tx_dev *dev); extern inline int quick_tx_free_skb(struct quick_tx_dev* dev, bool free_skb); @@ -291,6 +297,7 @@ struct quick_tx_shared_data { __u8 producer_wait_mem_flag; __u8 producer_wait_lookup_flag; + __u8 producer_wait_done_flag; __u8 consumer_wait_lookup_flag; } __attribute__((aligned(8))); @@ -310,6 +317,7 @@ struct quick_tx_shared_data { #define POLL_DMA POLLOUT #define POLL_LOOKUP POLLIN +#define POLL_DONE_TX 0x100 #ifndef __KERNEL__ struct quick_tx { @@ -529,7 +537,7 @@ static inline void __poll_for(struct quick_tx* dev, short events, __u8 *flag) { pfd.events = events; pfd.fd = dev->fd; - *flag = 1; + *flag = 0; wmb(); poll(&pfd, 1, 1000); @@ -546,6 +554,10 @@ static inline void __poll_for_lookup(struct quick_tx* dev) { __poll_for(dev, POLL_LOOKUP, &dev->data->producer_wait_lookup_flag); } +static inline void __poll_for_done_tx(struct quick_tx* dev) { + __poll_for(dev, POLL_DONE_TX, &dev->data->producer_wait_done_flag); +} + /** * Send packet on quick_tx device * @param qtx pointer to a quick_tx structure @@ -623,6 +635,12 @@ static inline int quick_tx_send_packet(struct quick_tx* dev, const void* buffer, } } + +static inline void quick_tx_wait_for_tx_complete(struct quick_tx* dev) { + __wake_up_module(dev); + __poll_for_done_tx(dev); +} + /* * Call this function to close the QuickTX device * @param qtx pointer to a quick_tx structure diff --git a/kernel/linux/quick_tx/quick_tx_main.c b/kernel/linux/quick_tx/quick_tx_main.c index 573e623ee..8664da4b5 100644 --- a/kernel/linux/quick_tx/quick_tx_main.c +++ b/kernel/linux/quick_tx/quick_tx_main.c @@ -27,7 +27,7 @@ DEFINE_MUTEX(init_mutex); #define E1000E_NAME "e1000e" #define E1000_NAME "e1000" -const char *netdev_drivername(const struct net_device *dev) +const char *quick_tx_netdev_drivername(const struct net_device *dev) { const struct device_driver *driver; const struct device *parent; @@ -45,15 +45,15 @@ const char *netdev_drivername(const struct net_device *dev) static void quick_tx_set_ops(struct quick_tx_dev *dev) { - if (!strncmp(netdev_drivername(dev->netdev), VIRTIO_NET_NAME, strlen(VIRTIO_NET_NAME))) { + if (!strncmp(quick_tx_netdev_drivername(dev->netdev), VIRTIO_NET_NAME, strlen(VIRTIO_NET_NAME))) { dev->ops = &quick_tx_virtio_net_ops; qtx_error("Set %s operations", VIRTIO_NET_NAME); return; - } else if (!strncmp(netdev_drivername(dev->netdev), E1000E_NAME, strlen(E1000E_NAME))) { + } else if (!strncmp(quick_tx_netdev_drivername(dev->netdev), E1000E_NAME, strlen(E1000E_NAME))) { dev->ops = &quick_tx_default_ops; qtx_error("Set %s operations", "default"); return; - } else if (!strncmp(netdev_drivername(dev->netdev), E1000_NAME, strlen(E1000_NAME))) { + } else if (!strncmp(quick_tx_netdev_drivername(dev->netdev), E1000_NAME, strlen(E1000_NAME))) { dev->ops = &quick_tx_e1000_ops; qtx_error("Set %s operations", E1000_NAME); return; @@ -115,12 +115,15 @@ static unsigned int quick_tx_poll(struct file *file, poll_table *wait) poll_wait(file, &dev->user_mem_q, wait); poll_wait(file, &dev->user_lookup_q, wait); + poll_wait(file, &dev->user_done_q, wait); smp_rmb(); if (dev->shared_data->producer_wait_mem_flag) mask |= (POLL_DMA); if (dev->shared_data->producer_wait_lookup_flag) mask |= (POLL_LOOKUP); + if (dev->shared_data->producer_wait_done_flag) + mask |= (POLL_DONE_TX); mutex_unlock(&dev->mtx); @@ -229,6 +232,7 @@ static int quick_tx_init(void) init_waitqueue_head(&dev->user_mem_q); init_waitqueue_head(&dev->user_lookup_q); + init_waitqueue_head(&dev->user_done_q); init_waitqueue_head(&dev->kernel_lookup_q); mutex_init(&dev->mtx); diff --git a/kernel/linux/quick_tx/quick_tx_mmap.c b/kernel/linux/quick_tx/quick_tx_mmap.c index 89df2ab97..3159c7440 100644 --- a/kernel/linux/quick_tx/quick_tx_mmap.c +++ b/kernel/linux/quick_tx/quick_tx_mmap.c @@ -62,6 +62,8 @@ void quick_tx_vm_master_close(struct vm_area_struct *vma) dev->num_skb_freed = 0; dev->num_skb_alloced = 0; dev->numsleeps = 0; + dev->time_start_tx = ktime_set(0, 0); + dev->time_end_tx = ktime_set(0, 0); /* kfree the memory allocated for master page */ kfree(dev->data); @@ -161,6 +163,7 @@ int quick_tx_mmap_master(struct file * file, struct vm_area_struct * vma) { dev->shared_data->smp_cache_bytes = SMP_CACHE_BYTES; dev->shared_data->prefix_len = NET_SKB_PAD; dev->shared_data->postfix_len = sizeof(struct skb_shared_info); + dev->shared_data->producer_wait_done_flag = 1; dev->shared_data->num_pages_per_block = 2 * (PAGE_ALIGN(dev->netdev->mtu) >> PAGE_SHIFT); dev->quit_work = false; diff --git a/kernel/linux/quick_tx/quick_tx_worker.c b/kernel/linux/quick_tx/quick_tx_worker.c index b9bcbb5f2..42456487a 100644 --- a/kernel/linux/quick_tx/quick_tx_worker.c +++ b/kernel/linux/quick_tx/quick_tx_worker.c @@ -33,6 +33,10 @@ inline void quick_tx_wake_up_user_lookup(struct quick_tx_dev *dev) { quick_tx_set_flag_wake_up_queue(&dev->user_lookup_q, &dev->shared_data->producer_wait_lookup_flag); } +inline void quick_tx_wake_up_user_done_tx(struct quick_tx_dev *dev) { + quick_tx_set_flag_wake_up_queue(&dev->user_done_q, &dev->shared_data->producer_wait_done_flag); +} + inline void quick_tx_wake_up_kernel_lookup(struct quick_tx_dev *dev) { quick_tx_set_flag_wake_up_queue(&dev->kernel_lookup_q, &dev->shared_data->consumer_wait_lookup_flag); } @@ -253,14 +257,13 @@ void inline quick_tx_wait_free_skb(struct quick_tx_dev *dev) { } } -static void inline quick_tx_finish_work(struct quick_tx_dev *dev, struct netdev_queue *txq) +static void inline quick_tx_finish_work(struct quick_tx_dev *dev, struct netdev_queue *txq, bool do_calc) { /* flush all remaining SKB's in the list before exiting */ quick_tx_do_xmit(NULL, txq, dev, 0, true); - dev->time_end_tx = ktime_get_real(); - - qtx_error("All packets have been transmitted successfully, exiting."); + if (ktime_to_ns(dev->time_end_tx) == 0) + dev->time_end_tx = ktime_get_real(); /* wait until cleaning the SKB list is finished * as well before exiting so we do not have any memory leaks */ @@ -269,11 +272,10 @@ static void inline quick_tx_finish_work(struct quick_tx_dev *dev, struct netdev_ dev->ops->wait_free_skb(dev); } - qtx_error("Done freeing free_skb_list"); - - quick_tx_calc_mbps(dev); - quick_tx_print_stats(dev); - + if (do_calc) { + quick_tx_calc_mbps(dev); + quick_tx_print_stats(dev); + } } void quick_tx_worker(struct work_struct *work) @@ -352,8 +354,14 @@ void quick_tx_worker(struct work_struct *work) data->lookup_consumer_index = (data->lookup_consumer_index + 1) % LOOKUP_TABLE_SIZE; entry = data->lookup_table + data->lookup_consumer_index; } else { + if (dev->shared_data->producer_wait_done_flag == 0) { + quick_tx_finish_work(dev, txq, false); + wmb(); + quick_tx_wake_up_user_done_tx(dev); + } + if (unlikely(dev->quit_work)) { - quick_tx_finish_work(dev, txq); + quick_tx_finish_work(dev, txq, true); break; } #ifdef QUICK_TX_DEBUG diff --git a/src/tcpreplay.c b/src/tcpreplay.c index c09a0fd36..41387958d 100644 --- a/src/tcpreplay.c +++ b/src/tcpreplay.c @@ -140,8 +140,6 @@ main(int argc, char *argv[]) } if (ctx->stats.bytes_sent > 0) { - if (gettimeofday(&ctx->stats.end_time, NULL) < 0) - errx(-1, "gettimeofday() failed: %s", strerror(errno)); packet_stats(&ctx->stats); if (ctx->options->flow_stats) flow_stats(ctx, ctx->options->unique_ip diff --git a/src/tcpreplay_api.c b/src/tcpreplay_api.c index 8b99dd936..5cf098221 100644 --- a/src/tcpreplay_api.c +++ b/src/tcpreplay_api.c @@ -1216,6 +1216,17 @@ tcpreplay_replay(tcpreplay_t *ctx) } ctx->running = false; + +#ifdef HAVE_QUICK_TX + /* flush any remaining netmap packets */ + if (ctx->options->quick_tx) + quick_tx_wait_for_tx_complete(ctx->intf1->qtx_dev); +#endif + + if (ctx->stats.bytes_sent > 0) { + if (gettimeofday(&ctx->stats.end_time, NULL) < 0) + errx(-1, "gettimeofday() failed: %s", strerror(errno)); + } return 0; }