diff --git a/kernel/ibv.c b/kernel/ibv.c index c033e7a00..1aac856a0 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -38,6 +38,7 @@ #include +#include extern uint8_t * host_logical_addr; @@ -45,6 +46,12 @@ size_t guest_to_host(size_t address) { return address ? virt_to_phys(address) + (size_t) host_logical_addr : address; } +/* inline static unsigned long long rdtsc(void) { */ + /* unsigned long lo, hi; */ + /* asm volatile ("rdtsc" : "=a"(lo), "=d"(hi) :: "memory"); */ + + /* return ((unsigned long long) hi << 32ULL | (unsigned long long) lo); */ +/* } */ /* * ibv_wc_status_str @@ -1928,9 +1935,15 @@ typedef struct { } __attribute__((packed)) uhyve_ibv_post_send_t; int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) { // !!! + /* LOG_INFO("ibv_post_send\n"); */ + + unsigned long long tick_start = rdtsc(); + uhyve_ibv_post_send_t uhyve_args; uhyve_args.qp = qp; + unsigned long long tick_gh = rdtsc(); uhyve_args.wr = (struct ibv_send_wr *) guest_to_host((size_t) wr); + tick_gh = rdtsc() - tick_gh; uhyve_args.bad_wr = (struct ibv_send_wr **) guest_to_host((size_t) bad_wr); struct ibv_send_wr * curr_wr; @@ -1961,6 +1974,7 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w curr_wr = wr; for (int i = 0; i < num_wrs; i++) { + /* LOG_INFO("ibv_post_send for wrs, i = %d\n", i); */ is_bind_mw = curr_wr->opcode == IBV_WR_BIND_MW; is_tso = curr_wr->opcode == IBV_WR_TSO; @@ -1978,6 +1992,7 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w curr_wr->next = (struct ibv_send_wr *) guest_to_host((size_t) curr_wr->next); for (int s = 0; s < curr_wr->num_sge; s++) { + /* LOG_INFO("ibv_post_send for sges, s = %d\n", s); */ wr__sg_list__addr[i][s] = curr_wr->sg_list[s].addr; curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr); } @@ -1985,14 +2000,18 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w wr__sg_list[i] = curr_wr->sg_list; curr_wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) curr_wr->sg_list); - curr_wr = curr_wr->next; + curr_wr = wr__next[i]; /* LOG_INFO("\tsg_list - guest backup address: %p", wr__sg_list[i]); */ /* LOG_INFO("\tsg_list - host address: %p", curr_wr->sg_list); */ } + unsigned long long tick_pre_work = rdtsc() - tick_start; + uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args)); + unsigned long long tick_call = rdtsc() - tick_start - tick_pre_work; + if (*bad_wr && *bad_wr == uhyve_args.wr) { *bad_wr = wr; } @@ -2024,6 +2043,12 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w curr_wr = curr_wr->next; } + unsigned long long tick_post_work = rdtsc() - tick_start - tick_pre_work - tick_call; + + LOG_INFO("tick_gh: %llu\n", tick_gh); + LOG_INFO("tick_pre_work: %llu\n", tick_pre_work); + LOG_INFO("tick_call: %llu\n", tick_call); + LOG_INFO("tick_post_work: %llu\n", tick_post_work); return uhyve_args.ret; } diff --git a/tools/uhyve-ibv.c b/tools/uhyve-ibv.c index a97263fd7..4cc75b96b 100644 --- a/tools/uhyve-ibv.c +++ b/tools/uhyve-ibv.c @@ -31,6 +31,17 @@ #include "uhyve-ibv.h" #include // Linux include +#include + + +static inline unsigned long long rdtsc() { + unsigned low, high; + unsigned long long val; + __asm__ volatile ("rdtsc" : "=a" (low), "=d" (high)); + val = high; + val = (val << 32) | low; + return val; +} /* * ibv_wc_status_str @@ -1240,7 +1251,10 @@ void call_ibv_post_send(struct kvm_run * run, uint8_t * guest_mem) { /* printf("\t ->sg_list[0].addr deref 1: %u\n", *((uint8_t *) args->wr->sg_list[0].addr)); */ use_ib_mem_pool = true; + /* unsigned long long ticks_native_call = rdtsc(); */ args->ret = ibv_post_send(args->qp, args->wr, args->bad_wr); + /* ticks_native_call = rdtsc() - ticks_native_call; */ + /* printf(" %llu ", ticks_native_call); */ use_ib_mem_pool = false; } diff --git a/tools/uhyve.c b/tools/uhyve.c index be9402a83..f7e71e2bf 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -222,6 +222,15 @@ typedef struct { char **envp; } __attribute__ ((packed)) uhyve_cmdval_t; +static inline unsigned long long rdtsc() { + unsigned low, high; + unsigned long long val; + __asm__ volatile ("rdtsc" : "=a" (low), "=d" (high)); + val = high; + val = (val << 32) | low; + return val; +} + // DLSYM @@ -1052,7 +1061,8 @@ static inline void check_network(void) static int vcpu_loop(void) { int ret; - + unsigned long long ticks_case_exec; + /* unsigned long long ticks_ioctl; */ if (restart) { pthread_barrier_wait(&barrier); if (cpuid == 0) @@ -1060,7 +1070,12 @@ static int vcpu_loop(void) } while (1) { + /* ticks_ioctl = rdtsc(); */ ret = ioctl(vcpufd, KVM_RUN, NULL); + /* if (run->exit_reason == KVM_EXIT_IO && run->io.port == UHYVE_PORT_IBV_POST_SEND) { */ + /* ticks_ioctl = rdtsc() - ticks_ioctl; */ + /* printf(" io: %llu ", ticks_ioctl); */ + /* } */ if(ret == -1) { switch(errno) { @@ -1246,6 +1261,25 @@ static int vcpu_loop(void) break; } + case UHYVE_PORT_IBV_POST_SEND: + /* ticks_case_exec = rdtsc(); */ + call_ibv_post_send(run, guest_mem); + /* ticks_case_exec = rdtsc() - ticks_case_exec; */ + /* printf(" %llu -- ", ticks_case_exec); */ + break; + case UHYVE_PORT_IBV_POST_RECV: + call_ibv_post_recv(run, guest_mem); + break; + case UHYVE_PORT_IBV_POLL_CQ: + call_ibv_poll_cq(run, guest_mem); + break; + case UHYVE_PORT_IBV_POST_WQ_RECV: + call_ibv_post_wq_recv(run, guest_mem); + break; + case UHYVE_PORT_IBV_POST_SRQ_RECV: + call_ibv_post_srq_recv(run, guest_mem); + break; + case UHYVE_PORT_IBV_WC_STATUS_STR: call_ibv_wc_status_str(run, guest_mem); break; @@ -1315,9 +1349,6 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_WC_READ_FLOW_TAG: call_ibv_wc_read_flow_tag(run, guest_mem); break; - case UHYVE_PORT_IBV_POST_WQ_RECV: - call_ibv_post_wq_recv(run, guest_mem); - break; case UHYVE_PORT_IBV_GET_DEVICE_LIST: call_ibv_get_device_list(run, guest_mem); break; @@ -1417,9 +1448,6 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_ACK_CQ_EVENTS: call_ibv_ack_cq_events(run, guest_mem); break; - case UHYVE_PORT_IBV_POLL_CQ: - call_ibv_poll_cq(run, guest_mem); - break; case UHYVE_PORT_IBV_REQ_NOTIFY_CQ: call_ibv_req_notify_cq(run, guest_mem); break; @@ -1441,9 +1469,6 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_DESTROY_SRQ: call_ibv_destroy_srq(run, guest_mem); break; - case UHYVE_PORT_IBV_POST_SRQ_RECV: - call_ibv_post_srq_recv(run, guest_mem); - break; case UHYVE_PORT_IBV_CREATE_QP: call_ibv_create_qp(run, guest_mem); break; @@ -1483,12 +1508,6 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_DESTROY_RWQ_IND_TABLE: call_ibv_destroy_rwq_ind_table(run, guest_mem); break; - case UHYVE_PORT_IBV_POST_SEND: - call_ibv_post_send(run, guest_mem); - break; - case UHYVE_PORT_IBV_POST_RECV: - call_ibv_post_recv(run, guest_mem); - break; case UHYVE_PORT_IBV_CREATE_AH: call_ibv_create_ah(run, guest_mem); break; diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-avg.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-avg.pdf new file mode 100644 index 000000000..fe4d16c69 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-msg_rate.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-msg_rate.pdf new file mode 100644 index 000000000..750e92e28 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-msg_rate.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-peak.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-peak.pdf new file mode 100644 index 000000000..f0b95b5ad Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw-peak.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw.log b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw.log new file mode 100644 index 000000000..c7c08e150 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_bw.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Read BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x65ef PSN 0xb6086 OUT 0x10 RKey 0x300101b5 VAddr 0x007fbf04745000 + remote address: LID 0x08 QPN 0x65fb PSN 0xdce35e OUT 0x10 RKey 0x680101b7 VAddr 0x007f6714187000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 8.22 6.85 3.593312 + 4 1000 14.90 14.08 3.692198 + 8 1000 32.13 31.96 4.188831 + 16 1000 60.56 58.27 3.818628 + 32 1000 127.97 127.70 4.184625 + 64 1000 240.79 233.24 3.821459 + 128 1000 510.80 508.80 4.168126 + 256 1000 946.36 928.71 3.804016 + 512 1000 2047.47 1532.12 3.137773 + 1024 1000 4077.85 3731.01 3.820554 + 2048 1000 5174.27 5091.78 2.606991 + 4096 1000 5653.51 5649.78 1.446344 + 8192 1000 5867.93 5858.31 0.749863 + 16384 1000 5985.94 5985.76 0.383089 + 32768 1000 5866.28 5860.68 0.187542 + 65536 1000 6006.36 5997.45 0.095959 + 131072 1000 4933.42 4917.02 0.039336 + 262144 1000 4636.63 4636.59 0.018546 + 524288 1000 4584.32 4584.31 0.009169 + 1048576 1000 4421.35 4421.35 0.004421 + 2097152 1000 4337.62 4337.62 0.002169 + 4194304 1000 4254.30 4254.30 0.001064 + 8388608 1000 4281.43 4281.43 0.000535 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99.pdf new file mode 100644 index 000000000..691252429 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99_9.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99_9.pdf new file mode 100644 index 000000000..d7c92bd6e Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-percentile_99_9.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-s_stddev.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-s_stddev.pdf new file mode 100644 index 000000000..2e9317616 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-s_stddev.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_avg.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_avg.pdf new file mode 100644 index 000000000..195fa7d31 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_max.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_max.pdf new file mode 100644 index 000000000..8b81ca82b Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_max.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_min.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_min.pdf new file mode 100644 index 000000000..a18908036 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_min.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_typical.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_typical.pdf new file mode 100644 index 000000000..621fbda72 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat-t_typical.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat.log b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat.log new file mode 100644 index 000000000..410ab86c4 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_read_lat.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x65f0 PSN 0x508ee4 OUT 0x10 RKey 0x380101b5 VAddr 0x007f883fe59000 + remote address: LID 0x08 QPN 0x65fc PSN 0x605b2c OUT 0x10 RKey 0x700101b7 VAddr 0x007f333d5ce000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 1.98 9.33 2.44 2.41 0.38 2.67 9.33 + 4 1000 2.10 14.81 2.55 2.50 0.34 2.75 14.81 + 8 1000 2.09 8.95 2.63 2.54 0.34 2.74 8.95 + 16 1000 2.09 10.01 2.63 2.56 0.50 2.76 10.01 + 32 1000 2.11 8.72 2.63 2.50 0.22 2.75 8.72 + 64 1000 2.13 13.79 2.66 2.52 0.32 2.77 13.79 + 128 1000 2.16 15.79 2.61 2.56 0.20 2.76 15.79 + 256 1000 2.23 14.15 2.69 2.63 0.23 2.87 14.15 + 512 1000 2.37 14.75 2.85 2.77 0.23 3.02 14.75 + 1024 1000 2.61 9.75 3.14 3.03 0.23 3.26 9.75 + 2048 1000 3.05 6.78 3.53 3.47 0.19 3.72 6.78 + 4096 1000 3.38 7.51 3.96 3.87 0.27 4.14 7.51 + 8192 1000 4.03 10.10 4.54 4.52 0.26 5.15 10.10 + 16384 1000 5.31 15.07 5.88 5.87 0.30 6.46 15.07 + 32768 1000 7.86 12.48 8.41 8.46 0.26 8.93 12.48 + 65536 1000 13.24 16.51 13.90 13.86 0.33 15.35 16.51 + 131072 1000 27.89 38.37 29.82 29.91 0.90 35.66 38.37 + 262144 1000 57.77 66.32 58.85 58.86 0.58 61.15 66.32 + 524288 1000 113.93 139.86 115.25 115.39 0.92 116.93 139.86 + 1048576 1000 230.41 248.63 231.73 231.90 1.44 240.43 248.63 + 2097152 1000 470.19 491.24 472.01 472.66 2.55 487.71 491.24 + 4194304 1000 959.65 979.50 961.96 962.50 2.01 974.33 979.50 + 8388608 1000 1840.32 1874.63 1843.18 1843.65 2.97 1858.57 1874.63 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-avg.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-avg.pdf new file mode 100644 index 000000000..4225cb962 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-msg_rate.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-msg_rate.pdf new file mode 100644 index 000000000..94b29361a Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-msg_rate.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-peak.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-peak.pdf new file mode 100644 index 000000000..2c0f305c9 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw-peak.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw.log b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw.log new file mode 100644 index 000000000..fe8325bcd --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_bw.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x65ed PSN 0x403909 RKey 0x200101b5 VAddr 0x007f2d84c6b000 + remote address: LID 0x08 QPN 0x65f9 PSN 0x898fe2 RKey 0x580101b7 VAddr 0x007f934be1a000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 8.91 8.60 4.507910 + 4 5000 17.95 17.76 4.655761 + 8 5000 35.91 35.59 4.664534 + 16 5000 71.48 71.22 4.667198 + 32 5000 143.62 142.60 4.672746 + 64 5000 287.25 263.03 4.309494 + 128 5000 574.50 561.76 4.601927 + 256 5000 1140.94 1121.63 4.594207 + 512 5000 2287.22 2276.22 4.661707 + 1024 5000 4595.97 4557.16 4.666527 + 2048 5000 5728.12 5718.44 2.927844 + 4096 5000 5874.55 5874.07 1.503761 + 8192 5000 5923.54 5920.26 0.757793 + 16384 5000 5946.08 5945.43 0.380508 + 32768 5000 5970.51 5960.54 0.190737 + 65536 5000 5971.93 5971.09 0.095537 + 131072 5000 5061.05 5010.62 0.040085 + 262144 5000 4726.47 4718.31 0.018873 + 524288 5000 4633.23 4625.37 0.009251 + 1048576 5000 4622.75 4598.05 0.004598 + 2097152 5000 4588.50 4588.40 0.002294 + 4194304 5000 4318.04 4314.69 0.001079 + 8388608 5000 4323.17 4313.18 0.000539 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99.pdf new file mode 100644 index 000000000..90df3c939 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99_9.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99_9.pdf new file mode 100644 index 000000000..6ffdd2d84 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-percentile_99_9.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-s_stddev.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-s_stddev.pdf new file mode 100644 index 000000000..cfd633992 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-s_stddev.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_avg.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_avg.pdf new file mode 100644 index 000000000..35576a67d Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_max.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_max.pdf new file mode 100644 index 000000000..72764a97b Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_max.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_min.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_min.pdf new file mode 100644 index 000000000..25ea2c229 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_min.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_typical.pdf b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_typical.pdf new file mode 100644 index 000000000..a2bddfcfb Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat-t_typical.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat.log b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat.log new file mode 100644 index 000000000..314257fb5 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-10-18-05/ib_write_lat.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x65ee PSN 0xc8fd64 RKey 0x280101b5 VAddr 0x007f52d7b02000 + remote address: LID 0x08 QPN 0x65fa PSN 0x2dff0d RKey 0x600101b7 VAddr 0x007fb3c7fb9000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 1.71 6.09 1.75 1.78 0.22 1.98 6.09 + 4 1000 1.64 5.76 1.75 1.77 0.16 1.93 5.76 + 8 1000 1.25 5.99 1.76 1.78 0.21 2.45 5.99 + 16 1000 1.38 5.44 1.76 1.78 0.13 1.87 5.44 + 32 1000 1.47 6.08 1.78 1.81 0.22 1.90 6.08 + 64 1000 1.47 3.86 1.70 1.71 0.10 1.81 3.86 + 128 1000 1.78 6.02 1.89 1.88 0.15 2.51 6.02 + 256 1000 2.52 10.20 2.77 2.78 0.34 3.62 10.20 + 512 1000 2.65 7.07 2.91 2.91 0.21 3.72 7.07 + 1024 1000 2.72 7.59 3.15 3.15 0.23 3.80 7.59 + 2048 1000 3.31 7.69 3.54 3.56 0.26 4.29 7.69 + 4096 1000 3.68 8.85 3.87 3.93 0.30 4.84 8.85 + 8192 1000 4.16 12.80 4.54 4.62 0.38 6.63 12.80 + 16384 1000 5.69 11.72 6.01 6.05 0.42 8.32 11.72 + 32768 1000 8.57 13.07 9.04 9.14 0.52 12.44 13.07 + 65536 1000 16.33 20.29 16.74 16.78 0.33 17.73 20.29 + 131072 1000 30.59 35.24 31.13 31.13 0.28 31.62 35.24 + 262144 1000 58.16 62.33 58.59 58.62 0.28 59.65 62.33 + 524288 1000 112.91 120.12 113.38 113.42 0.30 114.33 120.12 + 1048576 1000 217.35 230.05 222.90 222.79 1.02 225.84 230.05 + 2097152 1000 457.74 473.16 462.88 463.00 2.41 467.34 473.16 + 4194304 1000 940.79 976.00 967.94 968.26 2.77 975.17 976.00 + 8388608 1000 1825.63 1952.97 1904.44 1902.06 20.88 1946.35 1952.97 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_bw.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_bw.log new file mode 100644 index 000000000..788197206 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_bw.log @@ -0,0 +1,63 @@ + +mydestvaddr: 7ff6dcac1000 + +--------------------------------------------------------------------------------------- + RDMA_Read BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x720d PSN 0xddb16e OUT 0x10 RKey 0x30010176 VAddr 0x00007ff6dcac1000 + remote address: LID 0x08 QPN 0x7650 PSN 0xca97db OUT 0x10 RKey 0x600101b7 VAddr 0x00007fe061eee000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 0.51 0.50 0.264148 + 4 1000 1.06 1.05 0.274610 + 8 1000 2.13 2.12 0.278223 + 16 1000 4.22 4.19 0.274397 + 32 1000 8.58 8.52 0.279103 + 64 1000 17.05 16.82 0.275548 + 128 1000 33.81 33.61 0.275336 + 256 1000 68.11 67.84 0.277884 + 512 1000 135.12 134.57 0.275606 + 1024 1000 271.33 270.32 0.276810 + 2048 1000 538.91 538.53 0.275727 + 4096 1000 1105.27 1096.79 0.280777 + 8192 1000 2200.89 2194.98 0.280958 + 16384 1000 4363.67 4360.90 0.279098 + 32768 1000 6059.64 6054.21 0.193735 + 65536 1000 6079.09 6078.81 0.097261 + 131072 1000 5163.16 5162.63 0.041301 + 262144 1000 4898.19 4898.18 0.019593 + 524288 1000 4829.64 4829.63 0.009659 + 1048576 1000 4796.98 4796.12 0.004796 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_lat.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_lat.log new file mode 100644 index 000000000..f939057e4 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_read_lat.log @@ -0,0 +1,57 @@ + +mydestvaddr: 7fbfaf6e2000 + +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x720e PSN 0xddb16e OUT 0x10 RKey 0x38010176 VAddr 0x00007fbfaf6e2000 + remote address: LID 0x08 QPN 0x7651 PSN 0xca97db OUT 0x10 RKey 0x680101b7 VAddr 0x00007f34b0214000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 6.66 18.69 6.77 6.81 0.70 7.43 18.69 + 4 1000 6.61 14.79 6.69 6.72 0.37 6.96 14.79 + 8 1000 6.66 17.09 6.74 6.78 0.40 7.34 17.09 + 16 1000 6.61 18.85 6.70 6.73 0.40 6.91 18.85 + 32 1000 6.60 19.02 6.68 6.73 0.65 7.08 19.02 + 64 1000 6.59 18.43 6.68 6.74 0.65 9.73 18.43 + 128 1000 6.60 18.13 6.68 6.74 0.72 6.97 18.13 + 256 1000 6.60 17.99 6.69 6.74 0.60 7.25 17.99 + 512 1000 6.61 19.59 6.71 6.75 0.45 7.04 19.59 + 1024 1000 6.57 14.51 6.70 6.71 0.34 6.88 14.51 + 2048 1000 6.58 14.60 6.68 6.71 0.33 7.13 14.60 + 4096 1000 6.71 22.30 6.80 6.84 0.40 7.70 22.30 + 8192 1000 9.66 15.68 9.77 9.81 0.34 10.68 15.68 + 16384 1000 9.64 16.56 9.81 9.83 0.34 11.10 16.56 + 32768 1000 11.01 23.84 12.89 12.94 0.57 16.52 23.84 + 65536 1000 16.39 29.84 18.91 18.90 0.31 19.10 29.84 + 131072 1000 30.62 45.47 31.05 31.56 1.31 34.53 45.47 + 262144 1000 57.45 71.74 58.39 58.61 0.90 62.78 71.74 + 524288 1000 109.60 123.45 111.90 111.96 0.99 114.21 123.45 + 1048576 1000 214.77 246.18 216.09 216.70 1.79 225.24 246.18 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_bw.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_bw.log new file mode 100644 index 000000000..ee56ee658 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_bw.log @@ -0,0 +1,62 @@ + +mydestvaddr: 7f2c653e0000 + +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x720b PSN 0x5424a4 RKey 0x20010176 VAddr 0x00007f2c653e0000 + remote address: LID 0x08 QPN 0x764e PSN 0x410b11 RKey 0x500101b7 VAddr 0x00007fd9d5b24000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 0.52 0.51 0.269177 + 4 5000 1.07 1.07 0.280715 + 8 5000 2.17 2.16 0.282645 + 16 5000 4.34 4.32 0.282920 + 32 5000 8.72 8.64 0.283134 + 64 5000 17.41 17.35 0.284255 + 128 5000 34.59 34.47 0.282402 + 256 5000 68.83 68.81 0.281834 + 512 5000 138.71 137.88 0.282377 + 1024 5000 277.02 274.56 0.281147 + 2048 5000 553.10 548.22 0.280691 + 4096 5000 1127.76 1114.12 0.285215 + 8192 5000 2251.95 2251.00 0.288128 + 16384 5000 4509.11 4480.95 0.286781 + 32768 5000 6044.99 6044.28 0.193417 + 65536 5000 6050.84 6050.41 0.096807 + 131072 5000 5101.43 5096.53 0.040772 + 262144 5000 4885.80 4865.53 0.019462 + 524288 5000 4813.62 4813.61 0.009627 + 1048576 5000 4792.20 4784.88 0.004785 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_lat.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_lat.log new file mode 100644 index 000000000..da6171deb --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/hermit-ib_write_lat.log @@ -0,0 +1,56 @@ + +mydestvaddr: 7f88504ff000 + +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x720c PSN 0xddb16e RKey 0x28010176 VAddr 0x00007f88504ff000 + remote address: LID 0x08 QPN 0x764f PSN 0xca97db RKey 0x580101b7 VAddr 0x00007f42aff14000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 3.39 9.10 3.51 3.59 0.46 6.81 9.10 + 4 1000 3.41 10.43 3.47 3.56 0.56 7.59 10.43 + 8 1000 3.40 9.50 3.49 3.56 0.52 6.90 9.50 + 16 1000 3.40 9.81 3.47 3.54 0.49 7.61 9.81 + 32 1000 3.40 9.95 3.49 3.56 0.47 6.22 9.95 + 64 1000 3.39 8.76 3.47 3.53 0.39 6.13 8.76 + 128 1000 3.41 10.12 3.48 3.55 0.51 6.02 10.12 + 256 1000 3.75 9.71 3.86 3.90 0.33 5.66 9.71 + 512 1000 3.88 8.21 3.95 3.99 0.29 5.36 8.21 + 1024 1000 4.11 9.72 4.20 4.25 0.32 5.88 9.72 + 2048 1000 4.64 7.88 4.76 4.79 0.25 6.56 7.88 + 4096 1000 4.94 8.42 5.04 5.10 0.23 5.91 8.42 + 8192 1000 5.64 9.40 5.81 5.89 0.28 7.33 9.40 + 16384 1000 6.90 12.28 7.24 7.23 0.32 9.00 12.28 + 32768 1000 9.93 13.75 10.40 10.43 0.29 11.66 13.75 + 65536 1000 17.45 21.36 17.84 17.86 0.27 19.62 21.36 + 131072 1000 30.87 34.96 31.30 31.30 0.23 32.48 34.96 + 262144 1000 57.11 60.67 57.61 57.61 0.24 59.00 60.67 + 524288 1000 109.64 114.33 110.14 110.16 0.24 111.52 114.33 + 1048576 1000 214.72 220.38 215.72 215.73 0.46 216.84 220.38 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-avg.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-avg.pdf new file mode 100644 index 000000000..eafce4bc2 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-msg_rate.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-msg_rate.pdf new file mode 100644 index 000000000..281b6ab21 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-msg_rate.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-peak.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-peak.pdf new file mode 100644 index 000000000..8296416d7 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_bw-peak.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99.pdf new file mode 100644 index 000000000..474bfa7ab Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99_9.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99_9.pdf new file mode 100644 index 000000000..4e0563581 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-percentile_99_9.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-s_stddev.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-s_stddev.pdf new file mode 100644 index 000000000..bdc9e5696 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-s_stddev.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_avg.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_avg.pdf new file mode 100644 index 000000000..8993e5f47 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_max.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_max.pdf new file mode 100644 index 000000000..a88b18e2e Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_max.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_min.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_min.pdf new file mode 100644 index 000000000..16838dca6 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_min.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_typical.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_typical.pdf new file mode 100644 index 000000000..c3f994c7c Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_read_lat-t_typical.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-avg.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-avg.pdf new file mode 100644 index 000000000..faa2e91d8 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-msg_rate.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-msg_rate.pdf new file mode 100644 index 000000000..01ea88591 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-msg_rate.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-peak.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-peak.pdf new file mode 100644 index 000000000..e65f8aacc Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_bw-peak.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99.pdf new file mode 100644 index 000000000..e3e772182 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99_9.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99_9.pdf new file mode 100644 index 000000000..4e9afc2af Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-percentile_99_9.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-s_stddev.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-s_stddev.pdf new file mode 100644 index 000000000..817e50c24 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-s_stddev.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_avg.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_avg.pdf new file mode 100644 index 000000000..ab5ab39e3 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_max.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_max.pdf new file mode 100644 index 000000000..e4d2314f0 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_max.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_min.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_min.pdf new file mode 100644 index 000000000..592b91a20 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_min.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_typical.pdf b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_typical.pdf new file mode 100644 index 000000000..e7d2ec3d3 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-01-30-14-46/ib_write_lat-t_typical.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_bw.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_bw.log new file mode 100644 index 000000000..1ebf2dea0 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_bw.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Read BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7209 PSN 0xf7d77d OUT 0x10 RKey 0x10010176 VAddr 0x007f14fa618000 + remote address: LID 0x08 QPN 0x764c PSN 0x2a3ac5 OUT 0x10 RKey 0x400101b7 VAddr 0x007f492c11d000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 8.17 7.06 3.700176 + 4 1000 16.48 16.41 4.303026 + 8 1000 32.89 32.79 4.298181 + 16 1000 65.92 65.65 4.302656 + 32 1000 131.27 129.59 4.246495 + 64 1000 263.10 261.99 4.292425 + 128 1000 523.95 520.79 4.266277 + 256 1000 1050.15 1035.68 4.242136 + 512 1000 2104.84 1771.93 3.628908 + 1024 1000 4191.61 4029.03 4.125723 + 2048 1000 5286.30 5278.24 2.702461 + 4096 1000 5682.29 5678.46 1.453686 + 8192 1000 5923.54 5922.82 0.758121 + 16384 1000 5997.43 5996.31 0.383764 + 32768 1000 6025.76 6025.42 0.192813 + 65536 1000 6022.86 6022.71 0.096363 + 131072 1000 5214.96 5211.34 0.041691 + 262144 1000 4915.48 4889.58 0.019558 + 524288 1000 4814.53 4813.27 0.009627 + 1048576 1000 4778.75 4778.75 0.004779 + 2097152 1000 4769.61 4769.61 0.002385 + 4194304 1000 4775.22 4775.22 0.001194 + 8388608 1000 4776.15 4776.12 0.000597 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_lat.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_lat.log new file mode 100644 index 000000000..8da911252 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_read_lat.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x720a PSN 0xba7f90 OUT 0x10 RKey 0x18010176 VAddr 0x007efe55f39000 + remote address: LID 0x08 QPN 0x764d PSN 0x65c6d1 OUT 0x10 RKey 0x480101b7 VAddr 0x007f56f4898000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 1.91 10.82 2.30 2.24 0.25 2.68 10.82 + 4 1000 1.91 8.51 2.30 2.25 0.26 2.67 8.51 + 8 1000 1.90 9.85 2.30 2.31 0.60 4.71 9.85 + 16 1000 1.91 8.47 2.30 2.27 0.28 2.68 8.47 + 32 1000 1.92 8.33 2.36 2.33 0.37 2.69 8.33 + 64 1000 1.95 8.43 2.40 2.36 0.38 2.71 8.43 + 128 1000 1.99 9.89 2.37 2.33 0.27 2.75 9.89 + 256 1000 2.06 9.50 2.45 2.41 0.39 2.82 9.50 + 512 1000 2.19 9.55 2.59 2.56 0.31 2.94 9.55 + 1024 1000 2.52 14.95 2.94 2.90 0.31 3.31 14.95 + 2048 1000 2.98 8.40 3.40 3.38 0.33 3.79 8.40 + 4096 1000 3.32 10.74 3.77 3.72 0.33 4.12 10.74 + 8192 1000 3.96 8.79 4.47 4.42 0.34 5.25 8.79 + 16384 1000 5.24 10.96 5.75 5.72 0.30 6.43 10.96 + 32768 1000 7.78 14.53 8.37 8.32 0.27 8.79 14.53 + 65536 1000 13.05 23.60 13.61 13.60 0.35 15.02 23.60 + 131072 1000 27.02 36.37 28.15 28.17 0.57 29.20 36.37 + 262144 1000 53.90 61.21 54.80 54.81 0.42 55.72 61.21 + 524288 1000 108.04 116.80 109.50 109.57 0.61 110.92 116.80 + 1048576 1000 218.76 227.10 220.26 220.32 0.75 221.96 227.10 + 2097152 1000 440.45 453.70 443.19 443.69 2.00 448.70 453.70 + 4194304 1000 905.73 909.38 907.39 907.41 0.60 908.96 909.38 + 8388608 1000 1790.28 1799.48 1793.32 1793.34 0.88 1795.46 1799.48 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_bw.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_bw.log new file mode 100644 index 000000000..f8e260c30 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_bw.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7207 PSN 0xef1fff RKey 0x010176 VAddr 0x007fdee45b7000 + remote address: LID 0x08 QPN 0x764a PSN 0x7a64b2 RKey 0x300101b7 VAddr 0x007fcc60d63000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 9.04 8.80 4.613519 + 4 5000 18.21 18.05 4.730612 + 8 5000 36.42 36.20 4.745012 + 16 5000 72.84 72.41 4.745274 + 32 5000 146.03 144.94 4.749312 + 64 5000 292.76 286.57 4.695104 + 128 5000 584.12 578.97 4.742942 + 256 5000 1168.23 1159.26 4.748320 + 512 5000 2336.47 2319.62 4.750575 + 1024 5000 4695.40 4659.64 4.771473 + 2048 5000 5753.43 5745.55 2.941723 + 4096 5000 5905.63 5903.01 1.511170 + 8192 5000 5946.08 5945.78 0.761060 + 16384 5000 5995.13 5995.10 0.383687 + 32768 5000 5995.13 5994.61 0.191827 + 65536 5000 6004.63 6003.95 0.096063 + 131072 5000 5132.00 5059.31 0.040474 + 262144 5000 4754.29 4748.88 0.018996 + 524288 5000 4651.98 4647.52 0.009295 + 1048576 5000 4467.29 4448.96 0.004449 + 2097152 5000 4264.45 4264.34 0.002132 + 4194304 5000 4055.32 4046.60 0.001012 + 8388608 5000 4026.94 4024.62 0.000503 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_lat.log b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_lat.log new file mode 100644 index 000000000..c79a8ed7c --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-01-30-14-46/native-ib_write_lat.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7208 PSN 0x85640c RKey 0x8010176 VAddr 0x007feb52967000 + remote address: LID 0x08 QPN 0x764b PSN 0x7aeb37 RKey 0x380101b7 VAddr 0x007f1846e08000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 1.71 5.91 1.77 1.80 0.23 2.21 5.91 + 4 1000 1.71 6.64 1.77 1.79 0.27 1.87 6.64 + 8 1000 1.71 7.55 1.77 1.80 0.24 1.92 7.55 + 16 1000 1.72 6.75 1.78 1.81 0.27 1.95 6.75 + 32 1000 1.75 8.75 1.81 1.84 0.24 1.98 8.75 + 64 1000 1.61 5.53 1.69 1.71 0.11 1.81 5.53 + 128 1000 1.78 5.67 1.95 1.94 0.12 2.05 5.67 + 256 1000 2.49 6.33 2.80 2.80 0.15 2.90 6.33 + 512 1000 2.76 4.73 2.94 2.93 0.09 3.04 4.73 + 1024 1000 2.97 6.25 3.18 3.17 0.09 3.30 6.25 + 2048 1000 3.45 6.87 3.67 3.69 0.20 3.79 6.87 + 4096 1000 3.71 7.86 3.93 3.98 0.24 4.80 7.86 + 8192 1000 4.38 8.00 4.56 4.64 0.21 5.01 8.00 + 16384 1000 5.77 10.51 6.07 6.07 0.31 7.47 10.51 + 32768 1000 8.35 13.20 8.87 8.88 0.33 9.93 13.20 + 65536 1000 16.04 21.70 16.52 16.54 0.39 19.02 21.70 + 131072 1000 30.27 31.75 30.87 30.86 0.13 31.13 31.75 + 262144 1000 57.15 58.92 57.63 57.62 0.12 57.88 58.92 + 524288 1000 110.70 118.24 111.17 111.20 0.40 113.01 118.24 + 1048576 1000 212.64 215.01 213.46 213.43 0.28 214.01 215.01 + 2097152 1000 422.18 433.38 423.96 424.25 1.22 431.04 433.38 + 4194304 1000 851.07 860.48 852.36 852.53 1.17 859.62 860.48 + 8388608 1000 1701.09 1721.26 1703.42 1703.89 2.17 1717.23 1721.26 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_1.log new file mode 100644 index 000000000..3d6fe1a8f --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_1.log @@ -0,0 +1,63 @@ + +mydestvaddr: 7f8c0d8b9000 + +--------------------------------------------------------------------------------------- + RDMA_Read BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7340 PSN 0xb981d OUT 0x10 RKey 0x78010187 VAddr 0x00007f8c0d8b9000 + remote address: LID 0x08 QPN 0x7783 PSN 0xf87e8a OUT 0x10 RKey 0xa80101c8 VAddr 0x00007eff5740f000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 0.23 0.22 0.117701 + 4 1000 0.45 0.44 0.116499 + 8 1000 0.90 0.89 0.116855 + 16 1000 1.77 1.77 0.115686 + 32 1000 3.60 3.09 0.101131 + 64 1000 7.08 7.07 0.115785 + 128 1000 14.49 14.23 0.116606 + 256 1000 28.54 28.50 0.116725 + 512 1000 58.10 57.32 0.117397 + 1024 1000 115.39 113.50 0.116219 + 2048 1000 231.58 227.92 0.116696 + 4096 1000 465.90 452.62 0.115872 + 8192 1000 920.06 914.23 0.117021 + 16384 1000 1846.10 1793.29 0.114771 + 32768 1000 3740.14 3668.16 0.117381 + 65536 1000 6008.38 6008.33 0.096133 + 131072 1000 5079.66 5062.61 0.040501 + 262144 1000 4884.99 4884.98 0.019540 + 524288 1000 4826.31 4823.62 0.009647 + 1048576 1000 4795.64 4793.85 0.004794 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_16.log new file mode 100644 index 000000000..fdcbe19d9 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_16.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7f2cb1394000 + +--------------------------------------------------------------------------------------- + RDMA_Read Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7342 PSN 0xddb16e OUT 0x10 RKey 0x88010187 VAddr 0x00007f2cb1394000 + remote address: LID 0x08 QPN 0x7785 PSN 0xca97db OUT 0x10 RKey 0xb80101c8 VAddr 0x00007fea5e51e000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_4.log new file mode 100644 index 000000000..ec9a3d1ef --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_bw-pl_4.log @@ -0,0 +1,66 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7efdcfca1000 + +--------------------------------------------------------------------------------------- + RDMA_Read Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7341 PSN 0xddb16e OUT 0x10 RKey 0x80010187 VAddr 0x00007efdcfca1000 + remote address: LID 0x08 QPN 0x7784 PSN 0xca97db OUT 0x10 RKey 0xb00101c8 VAddr 0x00007fc8f5480000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 inf 0.86 0.452446 + 4 1000 inf 1.73 0.454650 + 8 1000 inf 3.43 0.449076 + 16 1000 inf 7.06 0.462694 + 32 1000 inf 13.90 0.455391 + 64 1000 inf 27.41 0.449028 + 128 1000 inf 55.27 0.452731 + 256 1000 inf 111.13 0.455184 + 512 1000 inf 221.00 0.452610 + 1024 1000 inf 438.84 0.449368 + 2048 1000 inf 880.91 0.451028 + 4096 1000 inf 1797.13 0.460066 + 8192 1000 inf 3569.73 0.456925 + 16384 1000 inf 6057.32 0.387669 + 32768 1000 inf 6062.53 0.194001 + 65536 1000 inf 6064.31 0.097029 + 131072 1000 inf 5054.64 0.040437 + 262144 1000 inf 4869.50 0.019478 + 524288 1000 inf 4819.08 0.009638 + 1048576 1000 inf 4800.00 0.004800 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_1.log new file mode 100644 index 000000000..49f29c0c5 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_1.log @@ -0,0 +1,57 @@ + +mydestvaddr: 7f7c4ea4b000 + +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7343 PSN 0x266539 OUT 0x10 RKey 0x90010187 VAddr 0x00007f7c4ea4b000 + remote address: LID 0x08 QPN 0x7786 PSN 0x266539 OUT 0x10 RKey 0xc00101c8 VAddr 0x00007fa901c15000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 11.78 23.17 11.93 12.23 1.39 20.13 23.17 + 4 1000 11.74 26.83 11.87 12.13 1.27 20.06 26.83 + 8 1000 11.71 26.65 11.87 12.17 1.44 20.09 26.65 + 16 1000 11.77 23.08 11.90 12.14 1.28 20.11 23.08 + 32 1000 11.69 22.80 11.86 12.32 1.82 20.30 22.80 + 64 1000 11.71 23.32 11.88 12.16 1.34 20.02 23.32 + 128 1000 11.73 23.46 11.88 12.15 1.30 20.05 23.46 + 256 1000 11.71 21.12 11.88 12.08 1.11 20.13 21.12 + 512 1000 11.73 22.96 11.88 12.01 0.83 17.14 22.96 + 1024 1000 11.72 20.43 11.87 12.16 1.35 20.07 20.43 + 2048 1000 11.74 20.72 11.86 12.28 1.70 20.25 20.72 + 4096 1000 11.82 20.65 11.94 12.21 1.32 20.13 20.65 + 8192 1000 11.73 21.84 11.87 12.21 1.48 20.06 21.84 + 16384 1000 11.76 20.11 11.87 11.99 0.69 16.03 20.11 + 32768 1000 11.72 25.23 11.86 12.11 1.24 20.12 25.23 + 65536 1000 18.04 31.73 18.49 18.61 0.71 21.91 31.73 + 131072 1000 31.80 44.64 35.05 35.08 0.51 37.75 44.64 + 262144 1000 60.16 70.20 61.49 61.54 0.41 63.22 70.20 + 524288 1000 114.69 130.71 117.67 117.62 0.64 118.71 130.71 + 1048576 1000 223.45 241.76 226.65 226.62 1.26 231.34 241.76 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_16.log new file mode 100644 index 000000000..5772bc54f --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_16.log @@ -0,0 +1,39 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7f00d07dc000 + +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 16 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7345 PSN 0x266539 OUT 0x10 RKey 0xa0010187 VAddr 0x00007f00d07dc000 + remote address: LID 0x08 QPN 0x7788 PSN 0x266539 OUT 0x10 RKey 0xd00101c8 VAddr 0x00007f16f9f77000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_4.log new file mode 100644 index 000000000..2c8cc0198 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_read_lat-pl_4.log @@ -0,0 +1,39 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7fef90436000 + +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 4 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7344 PSN 0xddb16e OUT 0x10 RKey 0x98010187 VAddr 0x00007fef90436000 + remote address: LID 0x08 QPN 0x7787 PSN 0xca97db OUT 0x10 RKey 0xc80101c8 VAddr 0x00007fb940211000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_1.log new file mode 100644 index 000000000..106f73883 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_1.log @@ -0,0 +1,62 @@ + +mydestvaddr: 7f3db344d000 + +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733a PSN 0x5424a4 RKey 0x48010187 VAddr 0x00007f3db344d000 + remote address: LID 0x08 QPN 0x777d PSN 0x410b11 RKey 0x780101c8 VAddr 0x00007fe505b5b000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 0.23 0.22 0.116484 + 4 5000 0.46 0.46 0.119732 + 8 5000 0.93 0.93 0.121553 + 16 5000 1.86 1.85 0.121143 + 32 5000 3.71 3.71 0.121596 + 64 5000 7.45 7.42 0.121489 + 128 5000 14.81 14.69 0.120355 + 256 5000 29.79 29.63 0.121366 + 512 5000 59.19 59.11 0.121065 + 1024 5000 119.04 118.40 0.121245 + 2048 5000 237.02 236.18 0.120925 + 4096 5000 479.66 477.08 0.122132 + 8192 5000 957.79 955.08 0.122250 + 16384 5000 1921.70 1911.34 0.122326 + 32768 5000 3842.69 3825.01 0.122400 + 65536 5000 6074.96 6074.90 0.097198 + 131072 5000 5040.44 5004.11 0.040033 + 262144 5000 4727.68 4711.29 0.018845 + 524288 5000 4635.14 4630.15 0.009260 + 1048576 5000 4596.36 4591.31 0.004591 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_16.log new file mode 100644 index 000000000..cc0ffd238 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_16.log @@ -0,0 +1,65 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7f5c24dbd000 + +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733c PSN 0x5424a4 RKey 0x58010187 VAddr 0x00007f5c24dbd000 + remote address: LID 0x08 QPN 0x777f PSN 0x410b11 RKey 0x880101c8 VAddr 0x00007f283f39a000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 3.08 1.617194 + 4 5000 inf 6.26 1.640151 + 8 5000 inf 12.50 1.638174 + 16 5000 inf 24.44 1.601580 + 32 5000 inf 50.21 1.645278 + 64 5000 inf 100.27 1.642831 + 128 5000 inf 200.67 1.643852 + 256 5000 inf 402.71 1.649484 + 512 5000 inf 792.65 1.623356 + 1024 5000 inf 1591.94 1.630145 + 2048 5000 inf 3211.08 1.644074 + 4096 5000 inf 5819.98 1.489916 + 8192 5000 inf 5884.83 0.753259 + 16384 5000 inf 5919.51 0.378848 + 32768 5000 inf 5936.31 0.189962 + 65536 5000 inf 5947.49 0.095160 + 131072 5000 inf 5103.26 0.040826 + 262144 5000 inf 4866.79 0.019467 + 524288 5000 inf 4649.20 0.009298 + 1048576 5000 inf 4591.13 0.004591 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_4.log new file mode 100644 index 000000000..53c5b6806 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_bw-pl_4.log @@ -0,0 +1,65 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7ff8a7e33000 + +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733b PSN 0x5424a4 RKey 0x50010187 VAddr 0x00007ff8a7e33000 + remote address: LID 0x08 QPN 0x777e PSN 0x410b11 RKey 0x800101c8 VAddr 0x00007fd39a713000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 0.87 0.455223 + 4 5000 inf 1.75 0.458357 + 8 5000 inf 3.53 0.463055 + 16 5000 inf 6.91 0.452679 + 32 5000 inf 14.02 0.459495 + 64 5000 inf 28.22 0.462307 + 128 5000 inf 56.13 0.459809 + 256 5000 inf 112.36 0.460216 + 512 5000 inf 224.38 0.459522 + 1024 5000 inf 449.19 0.459971 + 2048 5000 inf 893.86 0.457656 + 4096 5000 inf 1795.36 0.459612 + 8192 5000 inf 3582.28 0.458532 + 16384 5000 inf 6056.70 0.387629 + 32768 5000 inf 6074.09 0.194371 + 65536 5000 inf 6079.41 0.097271 + 131072 5000 inf 5142.29 0.041138 + 262144 5000 inf 4890.78 0.019563 + 524288 5000 inf 4836.92 0.009674 + 1048576 5000 inf 4604.57 0.004605 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_1.log new file mode 100644 index 000000000..e4b2f11b6 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_1.log @@ -0,0 +1,56 @@ + +mydestvaddr: 7fab0d6aa000 + +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733d PSN 0xddb16e RKey 0x60010187 VAddr 0x00007fab0d6aa000 + remote address: LID 0x08 QPN 0x7780 PSN 0xca97db RKey 0x900101c8 VAddr 0x00007f2599097000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 5.77 14.32 5.85 6.06 0.83 10.03 14.32 + 4 1000 5.77 12.16 5.85 6.05 0.82 10.00 12.16 + 8 1000 5.71 13.29 5.81 5.99 0.77 10.01 13.29 + 16 1000 5.73 10.17 5.81 6.04 0.83 9.90 10.17 + 32 1000 5.74 15.19 5.83 6.05 0.96 10.95 15.19 + 64 1000 5.74 12.13 5.82 5.99 0.73 9.92 12.13 + 128 1000 5.77 14.86 5.84 6.02 0.82 10.84 14.86 + 256 1000 5.69 12.06 5.76 5.92 0.67 9.82 12.06 + 512 1000 5.70 11.18 5.76 5.87 0.55 9.30 11.18 + 1024 1000 5.70 11.36 5.78 5.92 0.64 9.78 11.36 + 2048 1000 5.71 11.95 5.77 5.96 0.74 9.84 11.95 + 4096 1000 5.70 13.44 5.76 5.93 0.72 9.87 13.44 + 8192 1000 5.70 13.14 5.77 5.93 0.69 9.87 13.14 + 16384 1000 6.66 15.52 6.97 7.16 0.87 10.02 15.52 + 32768 1000 9.62 12.50 10.11 10.14 0.30 11.73 12.50 + 65536 1000 17.07 20.75 17.39 17.41 0.19 18.10 20.75 + 131072 1000 30.23 33.27 30.68 30.67 0.18 31.13 33.27 + 262144 1000 56.85 64.22 57.16 57.19 0.26 58.67 64.22 + 524288 1000 109.32 113.26 109.78 109.81 0.24 110.39 113.26 + 1048576 1000 213.83 216.71 214.53 214.55 0.27 215.38 216.71 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_16.log new file mode 100644 index 000000000..7ea09d3d8 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_16.log @@ -0,0 +1,38 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7f1da9999000 + +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733f PSN 0xddb16e RKey 0x70010187 VAddr 0x00007f1da9999000 + remote address: LID 0x08 QPN 0x7782 PSN 0xca97db RKey 0xa00101c8 VAddr 0x00007fc16ff26000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_4.log new file mode 100644 index 000000000..643a045ca --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/hermit-ib_write_lat-pl_4.log @@ -0,0 +1,38 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7fac33e10000 + +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x733e PSN 0xddb16e RKey 0x68010187 VAddr 0x00007fac33e10000 + remote address: LID 0x08 QPN 0x7781 PSN 0xca97db RKey 0x980101c8 VAddr 0x00007fa522440000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_1.log new file mode 100644 index 000000000..c9d85a2c6 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_1.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Read BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7334 PSN 0x795ade OUT 0x10 RKey 0x18010187 VAddr 0x007fca317b6000 + remote address: LID 0x08 QPN 0x7777 PSN 0xadc68c OUT 0x10 RKey 0x480101c8 VAddr 0x007fcdff779000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 7.95 7.36 3.858049 + 4 1000 16.03 15.96 4.182630 + 8 1000 32.13 31.91 4.182245 + 16 1000 64.25 64.09 4.200090 + 32 1000 128.51 128.01 4.194559 + 64 1000 257.55 256.67 4.205353 + 128 1000 511.87 509.91 4.177144 + 256 1000 1032.39 1028.57 4.213007 + 512 1000 2056.09 1551.30 3.177067 + 1024 1000 4094.94 3936.97 4.031457 + 2048 1000 5208.77 5197.71 2.661226 + 4096 1000 5641.27 5615.75 1.437633 + 8192 1000 5959.69 5946.94 0.761209 + 16384 1000 6035.65 6021.60 0.385383 + 32768 1000 6045.58 6043.90 0.193405 + 65536 1000 6042.36 6042.25 0.096676 + 131072 1000 4938.00 4891.52 0.039132 + 262144 1000 4570.97 4570.95 0.018284 + 524288 1000 4502.79 4502.77 0.009006 + 1048576 1000 4480.69 4480.69 0.004481 + 2097152 1000 4354.85 4354.85 0.002177 + 4194304 1000 4327.69 4323.20 0.001081 + 8388608 1000 4380.49 4380.48 0.000548 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_16.log new file mode 100644 index 000000000..4b04a34e3 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_16.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Read Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7336 PSN 0x6b5a0d OUT 0x10 RKey 0x28010187 VAddr 0x007f012e838000 + remote address: LID 0x08 QPN 0x7779 PSN 0xc8650e OUT 0x10 RKey 0x580101c8 VAddr 0x007f6c79519000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 inf 10.83 5.679195 + 4 1000 inf 30.25 7.929367 + 8 1000 inf 57.68 7.560239 + 16 1000 inf 114.36 7.494743 + 32 1000 inf 232.25 7.610406 + 64 1000 inf 431.42 7.068327 + 128 1000 inf 866.51 7.098429 + 256 1000 inf 1657.04 6.787221 + 512 1000 inf 1993.02 4.081707 + 1024 1000 inf 4085.25 4.183295 + 2048 1000 inf 4877.00 2.497025 + 4096 1000 inf 5466.88 1.399521 + 8192 1000 inf 5818.15 0.744723 + 16384 1000 inf 5961.60 0.381542 + 32768 1000 inf 5984.04 0.191489 + 65536 1000 inf 5988.85 0.095822 + 131072 1000 inf 4836.00 0.038688 + 262144 1000 inf 4623.25 0.018493 + 524288 1000 inf 4495.30 0.008991 + 1048576 1000 inf 4382.38 0.004382 + 2097152 1000 inf 4353.48 0.002177 + 4194304 1000 inf 4561.76 0.001140 + 8388608 1000 inf 4624.59 0.000578 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_4.log new file mode 100644 index 000000000..2b9b7aba1 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_bw-pl_4.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Read Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7335 PSN 0xec4e3f OUT 0x10 RKey 0x20010187 VAddr 0x007f4d55425000 + remote address: LID 0x08 QPN 0x7778 PSN 0xef81a7 OUT 0x10 RKey 0x500101c8 VAddr 0x007fcbdabd8000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 1000 inf 10.23 5.363241 + 4 1000 inf 24.69 6.472027 + 8 1000 inf 53.95 7.070826 + 16 1000 inf 108.84 7.133161 + 32 1000 inf 218.29 7.153059 + 64 1000 inf 460.42 7.543473 + 128 1000 inf 796.64 6.526086 + 256 1000 inf 1461.22 5.985155 + 512 1000 inf 1960.90 4.015916 + 1024 1000 inf 2712.03 2.777115 + 2048 1000 inf 3757.25 1.923711 + 4096 1000 inf 4783.09 1.224470 + 8192 1000 inf 5417.88 0.693489 + 16384 1000 inf 5862.75 0.375216 + 32768 1000 inf 6028.71 0.192919 + 65536 1000 inf 6029.11 0.096466 + 131072 1000 inf 4664.65 0.037317 + 262144 1000 inf 4535.07 0.018140 + 524288 1000 inf 4475.06 0.008950 + 1048576 1000 inf 4351.84 0.004352 + 2097152 1000 inf 4266.46 0.002133 + 4194304 1000 inf 4260.81 0.001065 + 8388608 1000 inf 4356.00 0.000545 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_1.log new file mode 100644 index 000000000..38cf9f10f --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_1.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7337 PSN 0x510e24 OUT 0x10 RKey 0x30010187 VAddr 0x007fa529378000 + remote address: LID 0x08 QPN 0x777a PSN 0x39b8c3 OUT 0x10 RKey 0x600101c8 VAddr 0x007fd63e8d1000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 2.10 12.29 2.55 2.50 0.35 2.73 12.29 + 4 1000 2.10 8.53 2.55 2.50 0.29 2.72 8.53 + 8 1000 2.11 8.03 2.54 2.48 0.23 2.71 8.03 + 16 1000 2.06 15.16 2.55 2.55 0.73 7.94 15.16 + 32 1000 2.11 9.24 2.64 2.50 0.23 2.74 9.24 + 64 1000 2.14 15.44 2.58 2.52 0.40 2.76 15.44 + 128 1000 2.17 9.08 2.62 2.57 0.31 2.81 9.08 + 256 1000 2.25 9.67 2.69 2.62 0.28 2.82 9.67 + 512 1000 2.38 16.04 2.82 2.75 0.31 3.01 16.04 + 1024 1000 2.61 10.15 3.06 3.01 0.22 3.24 10.15 + 2048 1000 3.07 9.44 3.52 3.45 0.23 3.70 9.44 + 4096 1000 3.36 11.34 3.86 3.84 0.55 4.06 11.34 + 8192 1000 4.04 11.46 4.51 4.49 0.26 5.01 11.46 + 16384 1000 5.32 7.65 5.79 5.79 0.26 6.27 7.65 + 32768 1000 7.86 9.43 8.43 8.45 0.26 8.87 9.43 + 65536 1000 13.18 20.10 13.76 13.74 0.41 15.76 20.10 + 131072 1000 28.62 37.53 29.99 29.98 0.51 30.83 37.53 + 262144 1000 57.12 63.72 58.15 58.14 0.42 58.91 63.72 + 524288 1000 113.89 129.02 115.00 115.00 0.45 115.98 129.02 + 1048576 1000 228.64 245.60 230.63 230.66 1.02 232.29 245.60 + 2097152 1000 465.65 486.44 468.79 469.11 1.65 472.31 486.44 + 4194304 1000 942.41 960.09 945.74 945.77 2.06 959.18 960.09 + 8388608 1000 1864.30 1882.36 1866.65 1866.96 2.10 1880.92 1882.36 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_16.log new file mode 100644 index 000000000..0c0fd6f67 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_16.log @@ -0,0 +1,19 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 16 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7339 PSN 0x6c224e OUT 0x10 RKey 0x40010187 VAddr 0x007f07c7e62000 + remote address: LID 0x08 QPN 0x777c PSN 0x934061 OUT 0x10 RKey 0x700101c8 VAddr 0x007f9a0461d000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_4.log new file mode 100644 index 000000000..e747c10ee --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_read_lat-pl_4.log @@ -0,0 +1,19 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Read Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 4 + Mtu : 2048[B] + Link type : IB + Outstand reads : 16 + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7338 PSN 0x9a57e1 OUT 0x10 RKey 0x38010187 VAddr 0x007f2282ea3000 + remote address: LID 0x08 QPN 0x777b PSN 0xbd6bc2 OUT 0x10 RKey 0x680101c8 VAddr 0x007f36c69db000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_1.log new file mode 100644 index 000000000..385fab852 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_1.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x732e PSN 0x116adf RKey 0xe8010186 VAddr 0x007f25a920a000 + remote address: LID 0x08 QPN 0x7771 PSN 0x121c0e RKey 0x180101c8 VAddr 0x007f0bb3b0b000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 8.52 8.05 4.220253 + 4 5000 17.18 16.99 4.454580 + 8 5000 34.22 34.08 4.466357 + 16 5000 68.74 68.17 4.467818 + 32 5000 137.17 136.72 4.480107 + 64 5000 275.58 252.83 4.142354 + 128 5000 548.68 539.50 4.419602 + 256 5000 1097.35 1066.11 4.366773 + 512 5000 2189.78 2181.77 4.468257 + 1024 5000 4389.41 4354.58 4.459093 + 2048 5000 5711.37 5683.24 2.909818 + 4096 5000 5848.17 5844.15 1.496103 + 8192 5000 5916.81 5915.60 0.757197 + 16384 5000 5939.30 5938.34 0.380053 + 32768 5000 5961.96 5957.23 0.190631 + 65536 5000 5955.71 5955.44 0.095287 + 131072 5000 5042.47 4984.59 0.039877 + 262144 5000 4714.13 4705.89 0.018824 + 524288 5000 4640.09 4637.11 0.009274 + 1048576 5000 4603.05 4602.84 0.004603 + 2097152 5000 4587.77 4587.02 0.002294 + 4194304 5000 4572.13 4571.19 0.001143 + 8388608 5000 4570.39 4570.39 0.000571 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_16.log new file mode 100644 index 000000000..53e4cfe1a --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_16.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7330 PSN 0xc00976 RKey 0xf8010186 VAddr 0x007fc702fa9000 + remote address: LID 0x08 QPN 0x7773 PSN 0x217708 RKey 0x280101c8 VAddr 0x007fa297f35000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 19.47 10.207975 + 4 5000 inf 42.79 11.217987 + 8 5000 inf 85.06 11.148457 + 16 5000 inf 168.72 11.057049 + 32 5000 inf 346.41 11.351273 + 64 5000 inf 568.60 9.315963 + 128 5000 inf 1328.63 10.884149 + 256 5000 inf 2483.58 10.172755 + 512 5000 inf 4337.20 8.882588 + 1024 5000 inf 5306.21 5.433559 + 2048 5000 inf 5704.10 2.920498 + 4096 5000 inf 5845.50 1.496449 + 8192 5000 inf 5907.87 0.756207 + 16384 5000 inf 5931.91 0.379642 + 32768 5000 inf 5938.59 0.190035 + 65536 5000 inf 5945.67 0.095131 + 131072 5000 inf 4894.28 0.039154 + 262144 5000 inf 4669.61 0.018678 + 524288 5000 inf 4601.61 0.009203 + 1048576 5000 inf 4570.20 0.004570 + 2097152 5000 inf 4544.84 0.002272 + 4194304 5000 inf 4538.73 0.001135 + 8388608 5000 inf 4532.68 0.000567 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_4.log new file mode 100644 index 000000000..a742489a6 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_bw-pl_4.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x732f PSN 0xd327b6 RKey 0xf0010186 VAddr 0x007f6af6ea3000 + remote address: LID 0x08 QPN 0x7772 PSN 0xbdffd1 RKey 0x200101c8 VAddr 0x007f66eeb81000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 17.67 9.265592 + 4 5000 inf 38.91 10.200770 + 8 5000 inf 77.67 10.180127 + 16 5000 inf 155.44 10.186805 + 32 5000 inf 313.72 10.280041 + 64 5000 inf 500.41 8.198696 + 128 5000 inf 1220.08 9.994878 + 256 5000 inf 2353.79 9.641106 + 512 5000 inf 4209.37 8.620785 + 1024 5000 inf 5126.57 5.249606 + 2048 5000 inf 5540.38 2.836675 + 4096 5000 inf 5820.42 1.490028 + 8192 5000 inf 5891.32 0.754089 + 16384 5000 inf 5928.96 0.379453 + 32768 5000 inf 5942.80 0.190170 + 65536 5000 inf 5960.57 0.095369 + 131072 5000 inf 4998.80 0.039990 + 262144 5000 inf 4719.12 0.018876 + 524288 5000 inf 4639.44 0.009279 + 1048576 5000 inf 4601.66 0.004602 + 2097152 5000 inf 4584.14 0.002292 + 4194304 5000 inf 4568.72 0.001142 + 8388608 5000 inf 4516.51 0.000565 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_1.log new file mode 100644 index 000000000..31603263d --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_1.log @@ -0,0 +1,40 @@ +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7331 PSN 0xc47f25 RKey 0x010187 VAddr 0x007f0a43e0f000 + remote address: LID 0x08 QPN 0x7774 PSN 0x68a821 RKey 0x300101c8 VAddr 0x007fcff3a23000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] + 2 1000 1.68 5.87 1.78 1.82 0.26 2.95 5.87 + 4 1000 1.70 5.03 1.78 1.80 0.09 1.93 5.03 + 8 1000 1.69 5.16 1.78 1.81 0.14 1.92 5.16 + 16 1000 1.65 5.62 1.79 1.81 0.13 1.93 5.62 + 32 1000 1.70 7.94 1.82 1.86 0.32 1.98 7.94 + 64 1000 1.70 6.29 1.77 1.82 0.25 1.99 6.29 + 128 1000 1.85 6.12 2.03 2.08 0.18 2.37 6.12 + 256 1000 2.74 7.13 2.94 2.99 0.22 3.73 7.13 + 512 1000 2.73 7.35 3.05 3.10 0.22 3.90 7.35 + 1024 1000 2.98 8.85 3.30 3.35 0.29 4.25 8.85 + 2048 1000 3.60 11.54 3.93 3.91 0.26 4.60 11.54 + 4096 1000 3.90 8.40 4.20 4.26 0.35 6.00 8.40 + 8192 1000 4.38 11.87 4.85 4.93 0.35 6.68 11.87 + 16384 1000 5.84 10.68 6.23 6.26 0.38 9.01 10.68 + 32768 1000 8.73 13.30 9.28 9.32 0.41 11.81 13.30 + 65536 1000 16.60 20.73 17.23 17.25 0.32 17.97 20.73 + 131072 1000 30.99 35.77 31.52 31.54 0.33 32.15 35.77 + 262144 1000 58.30 63.07 58.96 58.97 0.38 61.87 63.07 + 524288 1000 113.33 119.84 113.83 113.86 0.31 114.75 119.84 + 1048576 1000 223.20 231.51 224.14 224.17 0.51 224.74 231.51 + 2097152 1000 452.59 467.77 457.64 457.25 2.07 464.14 467.77 + 4194304 1000 922.14 933.88 924.94 925.06 1.18 931.54 933.88 + 8388608 1000 1845.39 1858.74 1847.35 1847.72 1.72 1854.80 1858.74 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_16.log new file mode 100644 index 000000000..7e57eab0a --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_16.log @@ -0,0 +1,19 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7333 PSN 0x893817 RKey 0x10010187 VAddr 0x007fe45aeb7000 + remote address: LID 0x08 QPN 0x7776 PSN 0x2c3305 RKey 0x400101c8 VAddr 0x007f3f23eac000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_4.log new file mode 100644 index 000000000..074772408 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-21/native-ib_write_lat-pl_4.log @@ -0,0 +1,19 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Latency Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 1 + Post List : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 220[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7332 PSN 0x53f29f RKey 0x8010187 VAddr 0x007fd99c109000 + remote address: LID 0x08 QPN 0x7775 PSN 0x6ed634 RKey 0x380101c8 VAddr 0x007ffa9e753000 +--------------------------------------------------------------------------------------- + #bytes #iterations t_min[usec] t_max[usec] t_typical[usec] t_avg[usec] t_stdev[usec] 99% percentile[usec] 99.9% percentile[usec] diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_1.log new file mode 100644 index 000000000..976a8b91d --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_1.log @@ -0,0 +1,62 @@ + +mydestvaddr: 7f5ee247a000 + +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7349 PSN 0x5424a4 RKey 0xc0010187 VAddr 0x00007f5ee247a000 + remote address: LID 0x08 QPN 0x778c PSN 0x410b11 RKey 0xf00101c8 VAddr 0x00007f2635731000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 0.23 0.23 0.120540 + 4 5000 0.46 0.46 0.120015 + 8 5000 0.92 0.92 0.120822 + 16 5000 1.86 1.84 0.120431 + 32 5000 3.70 3.69 0.120891 + 64 5000 7.39 7.38 0.120933 + 128 5000 14.82 14.75 0.120830 + 256 5000 29.53 29.46 0.120682 + 512 5000 59.13 59.00 0.120826 + 1024 5000 118.30 117.96 0.120786 + 2048 5000 236.10 234.05 0.119835 + 4096 5000 477.49 474.19 0.121392 + 8192 5000 957.44 950.06 0.121608 + 16384 5000 1906.81 1888.40 0.120858 + 32768 5000 3826.93 3804.22 0.121735 + 65536 5000 6021.99 6021.81 0.096349 + 131072 5000 5142.98 5125.33 0.041003 + 262144 5000 4893.16 4892.36 0.019569 + 524288 5000 4632.59 4623.38 0.009247 + 1048576 5000 4588.44 4588.44 0.004588 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_16.log new file mode 100644 index 000000000..53c140dca --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_16.log @@ -0,0 +1,65 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7fad90b43000 + +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x734b PSN 0x5424a4 RKey 0xd0010187 VAddr 0x00007fad90b43000 + remote address: LID 0x08 QPN 0x778e PSN 0x410b11 RKey 0x0101c9 VAddr 0x00007f8f278fa000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 3.07 1.609145 + 4 5000 inf 6.16 1.616106 + 8 5000 inf 12.46 1.633054 + 16 5000 inf 24.13 1.581609 + 32 5000 inf 49.69 1.628381 + 64 5000 inf 99.21 1.625380 + 128 5000 inf 199.71 1.636006 + 256 5000 inf 397.17 1.626828 + 512 5000 inf 787.50 1.612798 + 1024 5000 inf 1578.61 1.616497 + 2048 5000 inf 3163.72 1.619827 + 4096 5000 inf 5948.01 1.522689 + 8192 5000 inf 6012.47 0.769596 + 16384 5000 inf 6048.48 0.387103 + 32768 5000 inf 6064.72 0.194071 + 65536 5000 inf 6068.71 0.097099 + 131072 5000 inf 5155.17 0.041241 + 262144 5000 inf 4894.90 0.019580 + 524288 5000 inf 4660.37 0.009321 + 1048576 5000 inf 4594.51 0.004595 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_4.log new file mode 100644 index 000000000..0b63a76fc --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/hermit-ib_write_bw-pl_4.log @@ -0,0 +1,65 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list + +mydestvaddr: 7f11029cd000 + +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x734a PSN 0x5424a4 RKey 0xc8010187 VAddr 0x00007f11029cd000 + remote address: LID 0x08 QPN 0x778d PSN 0x410b11 RKey 0xf80101c8 VAddr 0x00007f97b0fc2000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 0.87 0.456410 + 4 5000 inf 1.72 0.450780 + 8 5000 inf 3.41 0.447285 + 16 5000 inf 6.89 0.451583 + 32 5000 inf 13.88 0.454721 + 64 5000 inf 27.74 0.454443 + 128 5000 inf 55.42 0.453999 + 256 5000 inf 109.67 0.449213 + 512 5000 inf 219.77 0.450093 + 1024 5000 inf 442.22 0.452838 + 2048 5000 inf 852.69 0.436575 + 4096 5000 inf 1776.22 0.454713 + 8192 5000 inf 3557.98 0.455421 + 16384 5000 inf 6060.63 0.387880 + 32768 5000 inf 6074.08 0.194371 + 65536 5000 inf 6080.26 0.097284 + 131072 5000 inf 5128.94 0.041032 + 262144 5000 inf 4901.50 0.019606 + 524288 5000 inf 4827.76 0.009656 + 1048576 5000 inf 4737.46 0.004737 +--------------------------------------------------------------------------------------- +LOG: UHYVE - call_ibv_get_device_list +LOG: UHYVE - call_ibv_open_device +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_get_device_name +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_query_device +LOG: UHYVE - call_ibv_alloc_pd +LOG: UHYVE - call_ibv_reg_mr +LOG: UHYVE - call_ibv_create_cq +LOG: UHYVE - call_ibv_create_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_query_port +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_modify_qp +LOG: UHYVE - call_ibv_destroy_qp +LOG: UHYVE - call_ibv_destroy_cq +LOG: UHYVE - call_ibv_dereg_mr +LOG: UHYVE - call_ibv_dealloc_pd +LOG: UHYVE - call_ibv_close_device diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-avg.pdf b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-avg.pdf new file mode 100644 index 000000000..60da5785c Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-avg.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-msg_rate.pdf b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-msg_rate.pdf new file mode 100644 index 000000000..713be3a64 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-msg_rate.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-peak.pdf b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-peak.pdf new file mode 100644 index 000000000..ee8ccc3c3 Binary files /dev/null and b/usr/benchmarks/ib/evaluation/18-02-01-18-46/ib_write_bw-peak.pdf differ diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_1.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_1.log new file mode 100644 index 000000000..e12dad7da --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_1.log @@ -0,0 +1,41 @@ +--------------------------------------------------------------------------------------- + RDMA_Write BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + CQ Moderation : 100 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7346 PSN 0xaff1cd RKey 0xa8010187 VAddr 0x007fce3bbf5000 + remote address: LID 0x08 QPN 0x7789 PSN 0x963ecd RKey 0xd80101c8 VAddr 0x007fd72a885000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 8.73 8.44 4.422401 + 4 5000 17.50 17.35 4.547207 + 8 5000 35.08 34.74 4.553343 + 16 5000 69.84 69.44 4.551064 + 32 5000 140.32 139.06 4.556762 + 64 5000 280.00 256.31 4.199385 + 128 5000 557.45 544.01 4.456533 + 256 5000 1117.44 1092.75 4.475888 + 512 5000 2240.01 2217.27 4.540963 + 1024 5000 4469.77 4451.45 4.558282 + 2048 5000 5711.37 5695.19 2.915937 + 4096 5000 5874.55 5859.86 1.500125 + 8192 5000 5925.79 5912.11 0.756750 + 16384 5000 5942.69 5942.13 0.380296 + 32768 5000 5960.83 5955.66 0.190581 + 65536 5000 5922.14 5920.25 0.094724 + 131072 5000 5041.45 4999.29 0.039994 + 262144 5000 4725.94 4724.44 0.018898 + 524288 5000 4648.76 4641.31 0.009283 + 1048576 5000 4608.07 4606.47 0.004606 + 2097152 5000 4588.32 4588.06 0.002294 + 4194304 5000 4576.26 4567.99 0.001142 + 8388608 5000 4488.89 4488.83 0.000561 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_16.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_16.log new file mode 100644 index 000000000..15a024282 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_16.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 16 + CQ Moderation : 16 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7348 PSN 0x18ded2 RKey 0xb8010187 VAddr 0x007fd7f5843000 + remote address: LID 0x08 QPN 0x778b PSN 0x13cbd0 RKey 0xe80101c8 VAddr 0x007f9a194b3000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 20.12 10.549137 + 4 5000 inf 43.87 11.499125 + 8 5000 inf 87.90 11.521752 + 16 5000 inf 175.86 11.525470 + 32 5000 inf 344.15 11.277085 + 64 5000 inf 576.59 9.446906 + 128 5000 inf 1374.89 11.263115 + 256 5000 inf 2570.74 10.529766 + 512 5000 inf 4389.14 8.988965 + 1024 5000 inf 5408.62 5.538432 + 2048 5000 inf 5713.46 2.925292 + 4096 5000 inf 5859.31 1.499984 + 8192 5000 inf 5912.28 0.756771 + 16384 5000 inf 5943.23 0.380367 + 32768 5000 inf 5947.53 0.190321 + 65536 5000 inf 5910.11 0.094562 + 131072 5000 inf 4945.79 0.039566 + 262144 5000 inf 4712.60 0.018850 + 524288 5000 inf 4632.65 0.009265 + 1048576 5000 inf 4598.49 0.004598 + 2097152 5000 inf 4582.56 0.002291 + 4194304 5000 inf 4552.99 0.001138 + 8388608 5000 inf 4406.11 0.000551 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_4.log b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_4.log new file mode 100644 index 000000000..8f593a208 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/18-02-01-18-46/native-ib_write_bw-pl_4.log @@ -0,0 +1,44 @@ +--------------------------------------------------------------------------------------- +Post List requested - CQ moderation will be the size of the post list +--------------------------------------------------------------------------------------- + RDMA_Write Post List BW Test + Dual-port : OFF Device : mlx4_0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + TX depth : 128 + Post List : 4 + CQ Moderation : 4 + Mtu : 2048[B] + Link type : IB + Max inline data : 0[B] + rdma_cm QPs : OFF + Data ex. method : Ethernet +--------------------------------------------------------------------------------------- + local address: LID 0x01 QPN 0x7347 PSN 0x6c295 RKey 0xb0010187 VAddr 0x007f22c8fae000 + remote address: LID 0x08 QPN 0x778a PSN 0xeec5bd RKey 0xe00101c8 VAddr 0x007efda7b53000 +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 2 5000 inf 17.74 9.303345 + 4 5000 inf 37.56 9.846171 + 8 5000 inf 74.84 9.809626 + 16 5000 inf 150.52 9.864741 + 32 5000 inf 302.87 9.924496 + 64 5000 inf 490.93 8.043317 + 128 5000 inf 1200.73 9.836410 + 256 5000 inf 2399.31 9.827594 + 512 5000 inf 4251.82 8.707736 + 1024 5000 inf 5243.55 5.369391 + 2048 5000 inf 5540.70 2.836839 + 4096 5000 inf 5748.47 1.471608 + 8192 5000 inf 5847.63 0.748497 + 16384 5000 inf 5906.08 0.377989 + 32768 5000 inf 5961.55 0.190770 + 65536 5000 inf 5977.06 0.095633 + 131072 5000 inf 4826.14 0.038609 + 262144 5000 inf 4668.97 0.018676 + 524288 5000 inf 4607.72 0.009215 + 1048576 5000 inf 4591.57 0.004592 + 2097152 5000 inf 4579.42 0.002290 + 4194304 5000 inf 4531.71 0.001133 + 8388608 5000 inf 4543.50 0.000568 +--------------------------------------------------------------------------------------- diff --git a/usr/benchmarks/ib/evaluation/README.md b/usr/benchmarks/ib/evaluation/README.md new file mode 100644 index 000000000..91b957d7c --- /dev/null +++ b/usr/benchmarks/ib/evaluation/README.md @@ -0,0 +1 @@ +# hc_ib_benchmarking diff --git a/usr/benchmarks/ib/evaluation/__init__.py b/usr/benchmarks/ib/evaluation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/usr/benchmarks/ib/evaluation/client.py b/usr/benchmarks/ib/evaluation/client.py new file mode 100755 index 000000000..7e9ad927f --- /dev/null +++ b/usr/benchmarks/ib/evaluation/client.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import subprocess +import os +import errno +from time import sleep +from datetime import datetime + +# from parse_results import parse_results +from config import \ +BENCHMARKS, OPTIONS, NATIVE_SERVER_IP, HERMIT_RUN_CLIENT, HERMIT_SERVER_IP + + +# CREATE_PLOTS = True + +timestamp = datetime.now().strftime("%y-%m-%d-%H-%M") + +directory = os.path.join('/home/wierichs/ib_benchmarking', timestamp) +try: + os.makedirs(directory) +except OSError as e: + if e.errno != errno.EEXIST: + raise + +for bm in BENCHMARKS: + for opt in OPTIONS: + post_list = opt.split()[-1] + file_name = 'native-' + bm + '-pl_' + post_list + '.log' + with open(os.path.join(directory, file_name), "w+") as logfile: + subprocess.call([bm] + opt.split() + [NATIVE_SERVER_IP], stdout = logfile) + sleep(2) + +for bm in BENCHMARKS: + for opt in OPTIONS: + post_list = opt.split()[-1] + file_name = 'hermit-' + bm + '-pl_' + post_list + '.log' + with open(os.path.join(directory, file_name), "w+") as logfile: + # subprocess.call((HERMIT_RUN_CLIENT + bm).split() + opt.split() + + # [HERMIT_SERVER_IP], stdout = logfile) + subprocess.call(HERMIT_RUN_CLIENT + bm + ' ' + opt + ' ' + HERMIT_SERVER_IP, + shell=True, stdout=logfile) + sleep(2) diff --git a/usr/benchmarks/ib/evaluation/config.py b/usr/benchmarks/ib/evaluation/config.py new file mode 100644 index 000000000..93c813c43 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/config.py @@ -0,0 +1,49 @@ +import os + + +# BENCHMARKS = ['ib_write_bw', + # 'ib_write_lat', + # 'ib_read_bw', + # 'ib_read_lat'] +BENCHMARKS = ['ib_write_bw'] +OPTIONS = ['-a --post_list 1', + '-a --post_list 4', + '-a --post_list 16'] + +TIMESTAMP = '18-02-01-18-46' # TODO: temporary while testing plots + +NATIVE_SERVER_IP = '137.226.133.151' + +HERMIT_PATH = '/home/wierichs' + +HERMIT_SERVER_IP = '10.0.5.3' +HERMIT_SERVER_GATEWAY = '10.0.5.1' +HERMIT_SERVER_TAPDEV = 'tap100' + +HERMIT_CLIENT_IP = '10.0.6.3' +HERMIT_CLIENT_GATEWAY = '10.0.6.1' +HERMIT_CLIENT_TAPDEV = 'tap200' + +HERMIT_MASK = '255.255.255.0' + +HERMIT_RUN_SERVER = 'HERMIT_ISLE=uhyve \ + HERMIT_VERBOSE=1 \ + HERMIT_KVM=1 \ + HERMIT_CPUS=1 \ + HERMIT_IP="' + HERMIT_SERVER_IP + '" \ + HERMIT_GATEWAY="' + HERMIT_SERVER_GATEWAY + '" \ + HERMIT_MASK="' + HERMIT_MASK + '" \ + HERMIT_NETIF=' + HERMIT_SERVER_TAPDEV + ' ' + \ + os.path.join(HERMIT_PATH, 'hermit/bin/proxy') + ' ' + \ + os.path.join(HERMIT_PATH, 'hermit/x86_64-hermit/extra/benchmarks/') + +HERMIT_RUN_CLIENT = 'HERMIT_ISLE=uhyve \ + HERMIT_VERBOSE=1 \ + HERMIT_KVM=1 \ + HERMIT_CPUS=1 \ + HERMIT_IP="' + HERMIT_CLIENT_IP + '" \ + HERMIT_GATEWAY="' + HERMIT_CLIENT_GATEWAY + '" \ + HERMIT_MASK="' + HERMIT_MASK + '" \ + HERMIT_NETIF=' + HERMIT_CLIENT_TAPDEV + ' ' + \ + os.path.join(HERMIT_PATH, 'hermit/bin/proxy') + ' ' + \ + os.path.join(HERMIT_PATH, 'hermit/x86_64-hermit/extra/benchmarks/') diff --git a/usr/benchmarks/ib/evaluation/parse_results.py b/usr/benchmarks/ib/evaluation/parse_results.py new file mode 100644 index 000000000..7c46e61cc --- /dev/null +++ b/usr/benchmarks/ib/evaluation/parse_results.py @@ -0,0 +1,137 @@ +from __future__ import print_function + +import numpy as np +import os + +import matplotlib +from matplotlib import pyplot as plt + +from config import BENCHMARKS, TIMESTAMP, OPTIONS + +matplotlib.rcParams['font.family'] = 'serif' +matplotlib.rcParams['mathtext.default'] = 'regular' +matplotlib.rcParams['xtick.major.pad'] = '8' # TODO: do this separately + +# matplotlib.rcParams['font.sans-serif'] = ['Tahoma'] +# matplotlib.rcParams['figure.figsize'] = 30, 10 + +MACHINE_TYPES = ['hermit', 'native'] + +bw_metrics = ['num_bytes', + 'iterations', + 'peak', + 'avg', + 'msg_rate'] + +lat_metrics = ['num_bytes', + 'iterations', + 't_min', + 't_max', + 't_typical', + 't_avg', + 's_stddev', + 'percentile_99', + 'percentile_99_9'] + +labels = {metric: '' for metric in bw_metrics[2:] + lat_metrics[2:] } +labels[bw_metrics [2]] = 'Peak Bandwidth [MB/sec]' +labels[bw_metrics [3]] = 'Average Bandwidth [MB/sec]' +labels[bw_metrics [4]] = 'Message Rate [Mpps]' +labels[lat_metrics[2]] = 'Min. Latency [${\mu}sec$]' +labels[lat_metrics[3]] = 'Max. Latency [${\mu}sec$]' +labels[lat_metrics[4]] = 'Typical Latency [${\mu}sec$]' +labels[lat_metrics[5]] = 'Average Latency [${\mu}sec$]' +labels[lat_metrics[6]] = 'Latency Standard Deviation [${\mu}sec$]' +labels[lat_metrics[7]] = '99 % Percentile [${\mu}sec$]' +labels[lat_metrics[8]] = '99.9 % Percentile [${\mu}sec$]' + + +def parse_results(directory): + """ blabla + """ + # Examples: results['native']['ib_write_bw -a' ]['4' ]['peak' ] + # results['hermit']['ib_write_lat -a']['16']['t_avg'] + num_wrs_list = [opt.split()[-1] for opt in OPTIONS] + results = {machine: + {bm_name: + {num_wrs: + {metric: [] + for metric in (bw_metrics if 'bw' in bm_name + else lat_metrics)} + for num_wrs in num_wrs_list} + for bm_name in BENCHMARKS} + for machine in MACHINE_TYPES} + + for machine in MACHINE_TYPES: + for bm in BENCHMARKS: + for num_wrs in num_wrs_list: + + read_line = False + file_name = machine + '-' + bm + '-pl_' + num_wrs + '.log' + file_path = os.path.join(directory, file_name) + + with open(file_path, "r") as logfile: + for line in logfile: + if line.startswith(' #bytes'): + read_line = True + continue + elif read_line and line.startswith('------'): + break + + if read_line: + values = line.split() + for metric, val in zip(bw_metrics if 'bw' in bm else lat_metrics, + values): + final_value = 0 + if 'inf' not in val: + final_value = float(val) if '.' in val else int(val) + + results[machine][bm][num_wrs][metric].append(final_value) + + return results + +def plot_results(results, directory): + """ blabla + """ + num_wrs_list = [opt.split()[-1] for opt in OPTIONS] + + for bm in BENCHMARKS: + for metric in (bw_metrics[2:] if 'bw' in bm else lat_metrics[2:]): + + file_name = bm + '-' + metric + '.pdf' + file_path = os.path.join(directory, file_name) + + fig, ax = plt.subplots(figsize=(10, 5)) + + ax.grid() + + # Axis labels + ax.set_xlabel('Message Size [B]') + ax.set_ylabel(labels[metric]) + + # Axis ticks and scale (x: log2 / y: log2 for latency) + ax.set_xscale('log', basex=2) + ax.set_xticks(np.power(2, range( + 1, len(results['native'][bm][num_wrs_list[0]]['num_bytes']) + 1))) + if 'lat' in bm: + ax.set_yscale('log', basey=2) + ax.get_yaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter()) + + # Plot, save and then clear figure + for num_wrs in num_wrs_list: + label_end = ' (' + num_wrs + ' work request' + \ + (')' if num_wrs is '1' else 's)') + ax.plot(results['native'][bm][num_wrs]['num_bytes'], + results['native'][bm][num_wrs][metric], + 'go-', label = 'Native' + label_end, linewidth = 2, markersize = 6) + label = 'HermitCore (' + num_wrs + ' work requests)' + ax.plot(results['hermit'][bm][num_wrs]['num_bytes'], + results['hermit'][bm][num_wrs][metric], + 'ro-', label = 'HermitCore' + label_end, linewidth = 2, markersize = 6) + + plt.legend(fontsize = 10, ncol = 3, loc = 'lower right', + bbox_to_anchor = [1, 1.05]) + + plt.savefig(file_path, bbox_inches = 'tight') + plt.gcf().clear() + diff --git a/usr/benchmarks/ib/evaluation/plot_test.py b/usr/benchmarks/ib/evaluation/plot_test.py new file mode 100755 index 000000000..ce760e8d1 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/plot_test.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +from __future__ import print_function +import os + +from parse_results import parse_results, plot_results +from config import BENCHMARKS, TIMESTAMP + + +directory = os.path.join('/home/anni/swarm1/ib_benchmarking', TIMESTAMP) +results = parse_results(directory) +plot_results(results, directory) diff --git a/usr/benchmarks/ib/evaluation/server.py b/usr/benchmarks/ib/evaluation/server.py new file mode 100755 index 000000000..b1e399db1 --- /dev/null +++ b/usr/benchmarks/ib/evaluation/server.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import subprocess + +from config import BENCHMARKS, OPTIONS, HERMIT_RUN_SERVER + +print('Run native benchmarks.') +for bm in BENCHMARKS: + print('Benchmark: ' + bm) + for opt in OPTIONS: + subprocess.call([bm] + opt.split()) + +print('\nRun HermitCore benchmarks.') +for bm in BENCHMARKS: + print('Benchmark: ' + bm) + for opt in OPTIONS: + # subprocess.call((HERMIT_RUN_SERVER + bm).split() + opt.split()) + subprocess.call(HERMIT_RUN_SERVER + bm + ' ' + opt, shell=True) diff --git a/usr/benchmarks/ib/get_clock.c b/usr/benchmarks/ib/get_clock.c index 91aaf7234..6dd9a2cd7 100755 --- a/usr/benchmarks/ib/get_clock.c +++ b/usr/benchmarks/ib/get_clock.c @@ -39,7 +39,7 @@ /* #define GET_CPU_MHZ_FROM_PROC 1 */ /* For gettimeofday */ -#define _BSD_SOURCE +/* #define _BSD_SOURCE */ #include #include @@ -113,12 +113,12 @@ static double sample_get_cpu_mhz(void) b = (MEASUREMENTS * sxy - sx * sy) / (MEASUREMENTS * sxx - sx * sx); a = (sy - b * sx) / MEASUREMENTS; - if (DEBUG) + if (DEBUG) { fprintf(stderr, "a = %g\n", a); - if (DEBUG) fprintf(stderr, "b = %g\n", b); - if (DEBUG) fprintf(stderr, "a / b = %g\n", a / b); + } + r_2 = (MEASUREMENTS * sxy - sx * sy) * (MEASUREMENTS * sxy - sx * sy) / (MEASUREMENTS * sxx - sx * sx) / (MEASUREMENTS * syy - sy * sy); @@ -130,6 +130,7 @@ static double sample_get_cpu_mhz(void) return 0; } + /* printf("sample_get_cpu_mhz, MHz: %e\n", b); */ return b; } @@ -142,11 +143,7 @@ static double proc_get_cpu_mhz(int no_cpu_freq_warn) int print_flag = 0; double delta; - #if defined(__FreeBSD__) - f = popen("/sbin/sysctl hw.clockrate","r"); - #else f = fopen("/proc/cpuinfo","r"); - #endif if (!f) return 0.0; @@ -176,12 +173,8 @@ static double proc_get_cpu_mhz(int no_cpu_freq_warn) rc = sscanf(s, "%lf", &m); m /= 1000000; #else - #if defined (__FreeBSD__) - rc = sscanf(buf, "hw.clockrate: %lf", &m); - #else rc = sscanf(buf, "cpu MHz : %lf", &m); #endif - #endif if (rc != 1) continue; @@ -201,11 +194,8 @@ static double proc_get_cpu_mhz(int no_cpu_freq_warn) } } -#if defined(__FreeBSD__) - pclose(f); -#else fclose(f); -#endif + /* printf("proc_get_cpu_mhz, MHz: %e\n", mhz); */ return mhz; } #endif @@ -226,6 +216,9 @@ double get_cpu_mhz(int no_cpu_freq_warn) return 0; delta = proc > sample ? proc - sample : sample - proc; + /* printf("proc: %e\n", proc); */ + /* printf("sample: %e\n", sample); */ + /* printf("delta: %e\n", delta); */ if (delta / proc > 0.02) { return sample; } diff --git a/usr/benchmarks/ib/perftest_resources.c b/usr/benchmarks/ib/perftest_resources.c index 542128396..9d79134d8 100755 --- a/usr/benchmarks/ib/perftest_resources.c +++ b/usr/benchmarks/ib/perftest_resources.c @@ -805,8 +805,8 @@ void alloc_ctx(struct pingpong_context *ctx, struct perftest_parameters *user_pa } if (user_param->machine == CLIENT || user_param->tst == LAT || user_param->duplex) { - ALLOCATE(ctx->sge_list, struct ibv_sge, user_param->num_of_qps * user_param->post_list); - ALLOCATE(ctx->wr, struct ibv_send_wr, user_param->num_of_qps * user_param->post_list); + ALLOCATE(ctx->sge_list, struct ibv_sge, user_param->num_of_qps * user_param->post_list); + ALLOCATE(ctx->wr, struct ibv_send_wr, user_param->num_of_qps * user_param->post_list); if ((user_param->verb == SEND && user_param->connection_type == UD ) || user_param->connection_type == DC) { ALLOCATE(ctx->ah, struct ibv_ah*, user_param->num_of_qps); @@ -1443,8 +1443,7 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p } /* main loop for posting - iterations*/ - while (totscnt < tot_iters || totccnt < tot_iters || - (user_param->test_type == DURATION && user_param->state != END_STATE) ) { + while (totscnt < tot_iters || totccnt < tot_iters) { /* main loop to run over all the qps and post each time n messages */ for (index =0; index < num_of_qps ; index++) { @@ -1460,7 +1459,7 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p } /* Loop to post work requests */ - while ( (ctx->scnt[index] < user_param->iters || user_param->test_type == DURATION) && + while ( ctx->scnt[index] < user_param->iters && ((ctx->scnt[index] - ctx->ccnt[index]) < (user_param->tx_depth)) && !((user_param->rate_limit_type == SW_RATE_LIMIT ) && is_sending_burst == 0)) { @@ -1531,7 +1530,7 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p } } - if (totccnt < tot_iters || (user_param->test_type == DURATION && totccnt < totscnt)) { + if (totccnt < tot_iters) { if (user_param->use_event) { if (ctx_notify_events(ctx->channel)) { fprintf(stderr, "Couldn't request CQ notification\n"); @@ -1569,13 +1568,6 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p else user_param->tcompleted[totccnt-1] = get_cycles(); } - - if (user_param->test_type==DURATION && user_param->state == SAMPLE_STATE) { - if (user_param->report_per_port) { - user_param->iters_per_port[user_param->port_by_qp[wc_id]] += user_param->cq_mod; - } - user_param->iters += user_param->cq_mod; - } } } else if (ne < 0) {