#include <rdma/rdma_cma.h>
#include <rdma/rsocket.h>
+#include "common.h"
struct test_size_param {
int size;
static void *buf;
static volatile uint8_t *poll_byte;
-static void size_str(char *str, size_t ssize, long long size)
-{
- long long base, fraction = 0;
- char mag;
-
- if (size >= (1 << 30)) {
- base = 1 << 30;
- mag = 'g';
- } else if (size >= (1 << 20)) {
- base = 1 << 20;
- mag = 'm';
- } else if (size >= (1 << 10)) {
- base = 1 << 10;
- mag = 'k';
- } else {
- base = 1;
- mag = '\0';
- }
-
- if (size / base < 10)
- fraction = (size % base) * 10 / base;
- if (fraction) {
- snprintf(str, ssize, "%lld.%lld%c", size / base, fraction, mag);
- } else {
- snprintf(str, ssize, "%lld%c", size / base, mag);
- }
-}
-
-static void cnt_str(char *str, size_t ssize, long long cnt)
-{
- if (cnt >= 1000000000)
- snprintf(str, ssize, "%lldb", cnt / 1000000000);
- else if (cnt >= 1000000)
- snprintf(str, ssize, "%lldm", cnt / 1000000);
- else if (cnt >= 1000)
- snprintf(str, ssize, "%lldk", cnt / 1000);
- else
- snprintf(str, ssize, "%lld", cnt);
-}
-
static void show_perf(void)
{
char str[32];
(usec / iterations) / (transfer_count * 2));
}
-static int size_to_count(int size)
-{
- if (size >= 1000000)
- return 100;
- else if (size >= 100000)
- return 1000;
- else if (size >= 10000)
- return 10000;
- else if (size >= 1000)
- return 100000;
- else
- return 1000000;
-}
-
static void init_latency_test(int size)
{
char sstr[5];
transfer_count = size_to_count(transfer_size);
}
-static void format_buf(void *buf, int size)
-{
- uint8_t *array = buf;
- static uint8_t data;
- int i;
-
- for (i = 0; i < size; i++)
- array[i] = data++;
-}
-
-static int verify_buf(void *buf, int size)
-{
- static long long total_bytes;
- uint8_t *array = buf;
- static uint8_t data;
- int i;
-
- for (i = 0; i < size; i++, total_bytes++) {
- if (array[i] != data++) {
- printf("data verification failed byte %lld\n", total_bytes);
- return -1;
- }
- }
- return 0;
-}
-
-static int do_poll(struct pollfd *fds)
-{
- int ret;
-
- do {
- ret = rpoll(fds, 1, poll_timeout);
- } while (!ret);
-
- return ret == 1 ? 0 : ret;
-}
-
static int send_msg(int size)
{
struct pollfd fds;
for (offset = 0; offset < size; ) {
if (use_async) {
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
return ret;
}
for (offset = 0; offset < size; ) {
if (use_async) {
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
return ret;
}
for (offset = 0; offset < size; ) {
if (use_async) {
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
return ret;
}
int val, ret;
memset(&hints, 0, sizeof hints);
- hints.ai_flags = RAI_PASSIVE;
+ hints.ai_flags = AI_PASSIVE;
ret = getaddrinfo(src_addr, port, &hints, &res);
if (ret) {
perror("getaddrinfo");
fds.fd = lrs;
fds.events = POLLIN;
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret) {
perror("rpoll");
return ret;
if (ret && (errno == EINPROGRESS)) {
fds.fd = rs;
fds.events = POLLOUT;
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
goto close;
#include <rdma/rdma_cma.h>
#include <rdma/rsocket.h>
+include "common.h"
struct test_size_param {
int size;
#define rs_getsockopt(s,l,n,v,ol) \
use_rs ? rgetsockopt(s,l,n,v,ol) : getsockopt(s,l,n,v,ol)
-static void size_str(char *str, size_t ssize, long long size)
-{
- long long base, fraction = 0;
- char mag;
-
- if (size >= (1 << 30)) {
- base = 1 << 30;
- mag = 'g';
- } else if (size >= (1 << 20)) {
- base = 1 << 20;
- mag = 'm';
- } else if (size >= (1 << 10)) {
- base = 1 << 10;
- mag = 'k';
- } else {
- base = 1;
- mag = '\0';
- }
-
- if (size / base < 10)
- fraction = (size % base) * 10 / base;
- if (fraction) {
- snprintf(str, ssize, "%lld.%lld%c", size / base, fraction, mag);
- } else {
- snprintf(str, ssize, "%lld%c", size / base, mag);
- }
-}
-
-static void cnt_str(char *str, size_t ssize, long long cnt)
-{
- if (cnt >= 1000000000)
- snprintf(str, ssize, "%lldb", cnt / 1000000000);
- else if (cnt >= 1000000)
- snprintf(str, ssize, "%lldm", cnt / 1000000);
- else if (cnt >= 1000)
- snprintf(str, ssize, "%lldk", cnt / 1000);
- else
- snprintf(str, ssize, "%lld", cnt);
-}
-
static void show_perf(void)
{
char str[32];
(usec / iterations) / (transfer_count * 2));
}
-static int size_to_count(int size)
-{
- if (size >= 1000000)
- return 100;
- else if (size >= 100000)
- return 1000;
- else if (size >= 10000)
- return 10000;
- else if (size >= 1000)
- return 100000;
- else
- return 1000000;
-}
-
static void init_latency_test(int size)
{
char sstr[5];
transfer_count = size_to_count(transfer_size);
}
-static void format_buf(void *buf, int size)
-{
- uint8_t *array = buf;
- static uint8_t data;
- int i;
-
- for (i = 0; i < size; i++)
- array[i] = data++;
-}
-
-static int verify_buf(void *buf, int size)
-{
- static long long total_bytes;
- uint8_t *array = buf;
- static uint8_t data;
- int i;
-
- for (i = 0; i < size; i++, total_bytes++) {
- if (array[i] != data++) {
- printf("data verification failed byte %lld\n", total_bytes);
- return -1;
- }
- }
- return 0;
-}
-
-static int do_poll(struct pollfd *fds)
-{
- int ret;
-
- do {
- ret = rs_poll(fds, 1, poll_timeout);
- } while (!ret);
-
- return ret == 1 ? 0 : ret;
-}
-
static int send_xfer(int size)
{
struct pollfd fds;
for (offset = 0; offset < size; ) {
if (use_async) {
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
return ret;
}
for (offset = 0; offset < size; ) {
if (use_async) {
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
return ret;
}
int val, ret;
memset(&hints, 0, sizeof hints);
- hints.ai_flags = RAI_PASSIVE;
+ hints.ai_flags = AI_PASSIVE;
ret = getaddrinfo(src_addr, port, &hints, &res);
if (ret) {
perror("getaddrinfo");
fds.fd = lrs;
fds.events = POLLIN;
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret) {
perror("rpoll");
return ret;
if (ret && (errno == EINPROGRESS)) {
fds.fd = rs;
fds.events = POLLOUT;
- ret = do_poll(&fds);
+ ret = do_poll(&fds, poll_timeout);
if (ret)
goto close;
--- /dev/null
+/*
+ * Copyright (c) 2012 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <netdb.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include <rdma/rdma_cma.h>
+#include <rdma/rsocket.h>
+
+static int test_size[] = {
+ { 1 << 6 },
+ { 1 << 7 }, { (1 << 7) + (1 << 6) },
+ { 1 << 8 }, { (1 << 8) + (1 << 7) },
+ { 1 << 9 }, { (1 << 9) + (1 << 8) },
+ { 1 << 10 }, { (1 << 10) + (1 << 9) },
+};
+#define TEST_CNT (sizeof test_size / sizeof test_size[0])
+
+enum {
+ msg_op_start,
+ msg_op_data,
+ msg_op_echo,
+ msg_op_end
+};
+
+struct message {
+ uint8_t op;
+ uint8_t id;
+ uint16_t reserved;
+ uint32_t data;
+ uint8_t buf[2048];
+};
+
+#define CTRL_MSG_SIZE 16
+
+struct client {
+ uint64_t recvcnt;
+};
+
+static struct client clients[256];
+static int id;
+
+static int rs;
+static int use_rs = 1;
+static int use_async;
+static int flags = MSG_DONTWAIT;
+static int poll_timeout;
+static int custom;
+static int echo;
+static int transfer_size = 1000;
+static int transfer_count = 1000;
+static int buffer_size;
+static char test_name[10] = "custom";
+static char *port = "7174";
+static char *dst_addr;
+static char *src_addr;
+static union socket_addr addr;
+static socklen_t addrlen;
+static struct timeval start, end;
+static struct message msg;
+
+#define rs_socket(f,t,p) use_rs ? rsocket(f,t,p) : socket(f,t,p)
+#define rs_bind(s,a,l) use_rs ? rbind(s,a,l) : bind(s,a,l)
+#define rs_connect(s,a,l) use_rs ? rconnect(s,a,l) : connect(s,a,l)
+#define rs_close(s) use_rs ? rclose(s) : close(s)
+#define rs_recvfrom(s,b,l,f,a,al) \
+ use_rs ? rrecvfrom(s,b,l,f,a,al) : recvfrom(s,b,l,f,a,al)
+#define rs_sendto(s,b,l,f,a,al) \
+ use_rs ? rsendto(s,b,l,f,a,al) : sendto(s,b,l,fa,al)
+#define rs_poll(f,n,t) use_rs ? rpoll(f,n,t) : poll(f,n,t)
+#define rs_fcntl(s,c,p) use_rs ? rfcntl(s,c,p) : fcntl(s,c,p)
+#define rs_setsockopt(s,l,n,v,ol) \
+ use_rs ? rsetsockopt(s,l,n,v,ol) : setsockopt(s,l,n,v,ol)
+#define rs_getsockopt(s,l,n,v,ol) \
+ use_rs ? rgetsockopt(s,l,n,v,ol) : getsockopt(s,l,n,v,ol)
+
+static void show_perf(void)
+{
+ char str[32];
+ float usec;
+ long long bytes;
+ int transfers;
+
+ usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec);
+ if (echo) {
+ transfers = transfer_count;
+ bytes = (long long) transfer_count * transfer_size * 2;
+ } else {
+ transfers = ntohl(msg.data);
+ bytes = (long long) transfers * transfer_size;
+ }
+
+ /* name size transfers bytes seconds Gb/sec usec/xfer */
+ printf("%-10s", test_name);
+ size_str(str, sizeof str, transfer_size);
+ printf("%-8s", str);
+ cnt_str(str, sizeof str, transfers);
+ printf("%-8s", str);
+ size_str(str, sizeof str, bytes);
+ printf("%-8s", str);
+ printf("%8.2fs%10.2f%11.2f\n",
+ usec / 1000000., (bytes * 8) / (1000. * usec),
+ (usec / iterations) / (transfers * 2));
+}
+
+static void init_latency_test(int size)
+{
+ char sstr[5];
+
+ size_str(sstr, sizeof sstr, size);
+ snprintf(test_name, sizeof test_name, "%s_lat", sstr);
+ transfer_size = size;
+ transfer_count = size_to_count(transfer_size) / 10;
+ echo = 1;
+}
+
+static void init_bandwidth_test(int size)
+{
+ char sstr[5];
+
+ size_str(sstr, sizeof sstr, size);
+ snprintf(test_name, sizeof test_name, "%s_bw", sstr);
+ transfer_size = size;
+ transfer_count = size_to_count(transfer_size);
+ echo = 0;
+}
+
+static void set_options(int rs)
+{
+ int val;
+
+ if (buffer_size) {
+ rs_setsockopt(rs, SOL_SOCKET, SO_SNDBUF, (void *) &buffer_size,
+ sizeof buffer_size);
+ rs_setsockopt(rs, SOL_SOCKET, SO_RCVBUF, (void *) &buffer_size,
+ sizeof buffer_size);
+ } else {
+ val = 1 << 19;
+ rs_setsockopt(rs, SOL_SOCKET, SO_SNDBUF, (void *) &val, sizeof val);
+ rs_setsockopt(rs, SOL_SOCKET, SO_RCVBUF, (void *) &val, sizeof val);
+ }
+
+ if (flags & MSG_DONTWAIT)
+ rs_fcntl(rs, F_SETFL, O_NONBLOCK);
+}
+
+static ssize_t svr_send(struct message *msg, size_t size,
+ union socket_addr *addr, socklen_t addrlen)
+{
+ struct pollfd fds;
+ ssize_t ret;
+
+ if (use_async) {
+ fds.fd = rs;
+ fds.events = POLLOUT;
+ }
+
+ do {
+ if (use_async) {
+ ret = do_poll(&fds, poll_timeout);
+ if (ret)
+ return ret;
+ }
+
+ ret = rs_sendto(rs, msg, size, flags, &addr->sa, addrlen);
+ } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
+
+ if (ret < 0)
+ perror("rsend");
+
+ return ret;
+}
+
+static ssize_t svr_recv(struct message *msg, size_t size,
+ union socket_addr *addr, socklen_t *addrlen)
+{
+ struct pollfd fds;
+ ssize_t ret;
+
+ if (use_async) {
+ fds.fd = rs;
+ fds.events = POLLIN;
+ }
+
+ do {
+ if (use_async) {
+ ret = do_poll(fds, poll_timeout);
+ if (ret)
+ return ret;
+ }
+
+ ret = rs_recvfrom(rs, msg, size, flags, &addr->sa, addrlen);
+ } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
+
+ if (ret < 0)
+ perror("rrecv");
+
+ return ret;
+}
+
+static int svr_process(struct message *msg, size_t size,
+ union socket_addr *addr, socklen_t addrlen)
+{
+ ssize_t ret;
+
+ switch (msg->op) {
+ case msg_op_start:
+ memset(&clients[id], 0, sizeof clients[id]);
+ msg.id = id++;
+ break;
+ case msg_op_echo:
+ clients[msg->id].recvcnt++;
+ break;
+ case msg_op_end:
+ msg->data = htonl(clients[msg->id].recvcnt);
+ break;
+ default:
+ clients[msg->id].recvcnt++;
+ return 0;
+ }
+
+ ret = svr_send(msg, size, addr, addrlen);
+ return (ret == size) ? 0 : (int) ret;
+}
+
+static int svr_bind(void)
+{
+ struct addrinfo hints, *res;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_socktype = SOCK_DGRAM;
+ ret = getaddrinfo(src_addr, port, &hints, &res);
+ if (ret) {
+ perror("getaddrinfo");
+ return ret;
+ }
+
+ rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+ if (rs < 0) {
+ perror("rsocket");
+ ret = rs;
+ goto out;
+ }
+
+ set_options(rs);
+ ret = rs_bind(rs, res->ai_addr, res->ai_addrlen);
+ if (ret) {
+ perror("rbind");
+ rs_close(rs);
+ }
+
+out:
+ free(res);
+ return ret;
+}
+
+static int svr_run(void)
+{
+ size_t len;
+ int ret;
+
+ ret = svr_bind();
+ while (!ret) {
+ addrlen = sizeof addr;
+ len = svr_recv(&msg, sizeof msg, &addr, &addrlen);
+ if (len < 0)
+ return len;
+
+ ret = svr_process(&msg, len, &addr, addrlen);
+ }
+ return ret;
+}
+
+static ssize_t client_send(struct message *msg, size_t size)
+{
+ struct pollfd fds;
+ int ret;
+
+ if (use_async) {
+ fds.fd = rs;
+ fds.events = POLLOUT;
+ }
+
+ do {
+ if (use_async) {
+ ret = do_poll(&fds, poll_timeout);
+ if (ret)
+ return ret;
+ }
+
+ ret = rs_send(rs, msg, size, flags);
+ } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
+
+ if (ret < 0)
+ perror("rsend");
+
+ return ret;
+}
+
+static ssize_t client_recv(struct message *msg, size_t size, int timeout)
+{
+ struct pollfd fds;
+ int ret;
+
+ if (timeout) {
+ fds.fd = rs;
+ fds.events = POLLIN;
+
+ ret = rs_poll(fds, 1, timeout);
+ if (ret <= 0)
+ return ret;
+ }
+
+ ret = rs_recv(rs, msg, size, flags);
+ if (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN))
+ perror("rrecv");
+
+ return ret;
+}
+
+static int client_send_recv(struct message *msg, size_t size)
+{
+ int ret;
+
+ do {
+ ret = client_send(msg, size);
+ if (ret != size)
+ return ret;
+
+ ret = client_recv(msg, size, 1);
+ } while (ret <= 0);
+
+ return ret;
+}
+
+static int run_test(void)
+{
+ int ret, i, t;
+
+ msg.op = msg_op_start;
+ ret = client_send_recv(&msg, CTRL_MSG_SIZE);
+ if (ret != CTRL_MSG_SIZE)
+ goto out;
+
+ msg.op = echo ? htonl(msg_op_echo) : htonl(msg_op_data);
+ gettimeofday(&start, NULL);
+ for (i = 0; i < transfer_count; i++) {
+ ret = echo ? client_send_recv(&msg, transfer_size) :
+ client_send(&msg, transfer_size);
+ if (ret != transfer_size)
+ goto out;
+ }
+
+ msg.op = msg_op_end;
+ ret = client_send_recv(&msg, CTRL_MSG_SIZE);
+ if (ret != CTRL_MSG_SIZE)
+ goto out;
+
+ gettimeofday(&end, NULL);
+ show_perf();
+ ret = 0;
+
+out:
+ return ret;
+}
+
+static int client_connect(void)
+{
+ struct addrinfo hints, *res;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_socktype = SOCK_DGRAM;
+ ret = getaddrinfo(dst_addr, port, hints, &res);
+ if (ret) {
+ perror("getaddrinfo");
+ return ret;
+ }
+
+ rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+ if (rs < 0) {
+ perror("rsocket");
+ ret = rs;
+ goto out;
+ }
+
+ set_options(rs);
+ ret = rs_connect(rs, res->ai_addr, res->ai_addrlen);
+ if (ret) {
+ perror("rconnect");
+ rs_close(rs);
+ }
+
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
+static int client_run(void)
+{
+ int i, ret;
+
+ printf("%-10s%-8s%-8s%-8s%8s %10s%13s\n",
+ "name", "bytes", "xfers", "total", "time", "Gb/sec", "usec/xfer");
+
+ ret = client_connect();
+ if (ret)
+ return ret;
+
+ if (!custom) {
+ for (i = 0; i < TEST_CNT; i++) {
+ init_latency_test(test_size[i]);
+ run_test();
+ }
+ for (i = 0; i < TEST_CNT; i++) {
+ init_bandwidth_test(test_size[i]);
+ run_test();
+ }
+ } else {
+ run_test();
+ }
+ rs_close(rs);
+
+ return ret;
+}
+
+static int set_test_opt(char *optarg)
+{
+ if (strlen(optarg) == 1) {
+ switch (optarg[0]) {
+ case 's':
+ use_rs = 0;
+ break;
+ case 'a':
+ use_async = 1;
+ break;
+ case 'b':
+ flags = 0;
+ break;
+ case 'n':
+ flags = MSG_DONTWAIT;
+ break;
+ case 'e':
+ echo = 1;
+ break;
+ default:
+ return -1;
+ }
+ } else {
+ if (!strncasecmp("socket", optarg, 6)) {
+ use_rs = 0;
+ } else if (!strncasecmp("async", optarg, 5)) {
+ use_async = 1;
+ } else if (!strncasecmp("block", optarg, 5)) {
+ flags = 0;
+ } else if (!strncasecmp("nonblock", optarg, 8)) {
+ flags = MSG_DONTWAIT;
+ } else if (!strncasecmp("echo", optarg, 4)) {
+ echo = 1;
+ } else {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "s:b:B:C:S:p:T:")) != -1) {
+ switch (op) {
+ case 's':
+ dst_addr = optarg;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'B':
+ buffer_size = atoi(optarg);
+ break;
+ case 'C':
+ custom = 1;
+ transfer_count = atoi(optarg);
+ break;
+ case 'S':
+ custom = 1;
+ transfer_size = atoi(optarg);
+ if (transfer_size < CTRL_MSG_SIZE) {
+ printf("size must be at least %d bytes\n",
+ CTRL_MSG_SIZE);
+ exit(1);
+ }
+ break;
+ case 'p':
+ port = optarg;
+ break;
+ case 'T':
+ if (!set_test_opt(optarg))
+ break;
+ /* invalid option - fall through */
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-B buffer_size]\n");
+ printf("\t[-C transfer_count]\n");
+ printf("\t[-S transfer_size]\n");
+ printf("\t[-p port_number]\n");
+ printf("\t[-T test_option]\n");
+ printf("\t s|sockets - use standard tcp/ip sockets\n");
+ printf("\t a|async - asynchronous operation (use poll)\n");
+ printf("\t b|blocking - use blocking calls\n");
+ printf("\t n|nonblocking - use nonblocking calls\n");
+ printf("\t e|echo - server echoes all messages\n");
+ exit(1);
+ }
+ }
+
+ if (flags)
+ poll_timeout = -1;
+
+ ret = dst_addr ? client_run() : svr_run();
+ return ret;
+}