From: Arlin Davis Date: Mon, 7 May 2007 19:49:18 +0000 (-0700) Subject: Fix issues with IB DTO extensions X-Git-Tag: libdapl-2.0~6 X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=7cda2304a6880371ec2d9451a4f83a7a254bc474;p=~ardavis%2Fdapl.git Fix issues with IB DTO extensions - debug print_cqe updated for latest IBV definitions - dapli_evd_cq_to_event modified to handle both post and receive extensions - dtestx example tested with rdma_write_imm, fetch_add, and cmp_swap - modify cookie DTO types to support extensions properly --- diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c index 73e6c58..17a05c6 100755 --- a/dapl/common/dapl_evd_util.c +++ b/dapl/common/dapl_evd_util.c @@ -504,17 +504,16 @@ dapli_evd_eh_print_cqe ( #ifdef DAPL_DBG static char *optable[] = { + "OP_RDMA_WRITE", + "OP_RDMA_WRITE_IMM", "OP_SEND", + "OP_SEND_IMM", "OP_RDMA_READ", - "OP_RDMA_WRITE", "OP_COMP_AND_SWAP", "OP_FETCH_AND_ADD", "OP_RECEIVE", + "OP_RECEIVE_IMM", "OP_BIND_MW", -#ifdef DAT_EXTENSIONS - "OP_COMP_AND_SWAP_RESP", - "OP_FETCH_AND_ADD_RESP", -#endif 0 }; @@ -1009,6 +1008,7 @@ dapli_evd_cqe_to_event ( DAPL_COOKIE *cookie; DAT_DTO_COMPLETION_STATUS dto_status; + /* * All that can be relied on if the status is bad is the status * and WRID. @@ -1034,6 +1034,17 @@ dapli_evd_cqe_to_event ( { case DAPL_COOKIE_TYPE_DTO: { +#ifdef DAT_EXTENSIONS + /* Extended via request post or message receive */ + if ((cookie->val.dto.type == DAPL_DTO_TYPE_EXTENSION) || + (cookie->val.dto.type == DAPL_DTO_TYPE_RECV && + DAPL_GET_CQE_OPTYPE(cqe_ptr) != OP_RECEIVE)) + { + dapls_cqe_to_event_extension(ep_ptr, cookie, + cqe_ptr, event_ptr); + break; + } +#endif DAPL_COOKIE_BUFFER *buffer; if ( DAPL_DTO_TYPE_RECV == cookie->val.dto.type ) @@ -1054,24 +1065,6 @@ dapli_evd_cqe_to_event ( cookie->val.dto.cookie; event_ptr->event_data.dto_completion_event_data.status = dto_status; -#ifdef DAPL_DBG - if (dto_status == DAT_DTO_SUCCESS) - { - uint32_t ibtype; - - ibtype = DAPL_GET_CQE_OPTYPE (cqe_ptr); - - dapl_os_assert ((ibtype == OP_SEND && - cookie->val.dto.type == DAPL_DTO_TYPE_SEND) - || (ibtype == OP_RECEIVE && - cookie->val.dto.type == DAPL_DTO_TYPE_RECV) - || (ibtype == OP_RDMA_WRITE && - cookie->val.dto.type == DAPL_DTO_TYPE_RDMA_WRITE) - || (ibtype == OP_RDMA_READ && - cookie->val.dto.type == DAPL_DTO_TYPE_RDMA_READ)); - } -#endif /* DAPL_DBG */ - if ( cookie->val.dto.type == DAPL_DTO_TYPE_SEND || cookie->val.dto.type == DAPL_DTO_TYPE_RDMA_WRITE ) { @@ -1119,13 +1112,6 @@ dapli_evd_cqe_to_event ( dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie); break; } -#ifdef DAT_EXTENSIONS - case DAPL_COOKIE_TYPE_EXTENSION: - { - dapls_cqe_to_event_extension(ep_ptr, cookie, cqe_ptr, event_ptr); - break; - } -#endif default: { dapl_os_assert (!"Invalid Operation type"); diff --git a/dapl/include/dapl.h b/dapl/include/dapl.h index 41af96f..2e0714d 100755 --- a/dapl/include/dapl.h +++ b/dapl/include/dapl.h @@ -566,6 +566,9 @@ typedef enum dapl_dto_type DAPL_DTO_TYPE_RECV, DAPL_DTO_TYPE_RDMA_WRITE, DAPL_DTO_TYPE_RDMA_READ, +#ifdef DAT_EXTENSIONS + DAPL_DTO_TYPE_EXTENSION, +#endif } DAPL_DTO_TYPE; typedef enum dapl_cookie_type @@ -573,9 +576,6 @@ typedef enum dapl_cookie_type DAPL_COOKIE_TYPE_NULL, DAPL_COOKIE_TYPE_DTO, DAPL_COOKIE_TYPE_RMR, -#ifdef DAT_EXTENSIONS - DAPL_COOKIE_TYPE_EXTENSION, -#endif } DAPL_COOKIE_TYPE; /* DAPL_DTO_COOKIE used as context for DTO WQEs */ diff --git a/dapl/openib_cma/dapl_ib_extensions.c b/dapl/openib_cma/dapl_ib_extensions.c index 73ce400..3132ffb 100755 --- a/dapl/openib_cma/dapl_ib_extensions.c +++ b/dapl/openib_cma/dapl_ib_extensions.c @@ -200,7 +200,7 @@ dapli_post_ext( IN DAT_EP_HANDLE ep_handle, dat_status = dapls_dto_cookie_alloc( &ep_ptr->req_buffer, - DAPL_COOKIE_TYPE_EXTENSION, + DAPL_DTO_TYPE_EXTENSION, user_cookie, &cookie ); @@ -262,8 +262,8 @@ dapls_cqe_to_event_extension(IN DAPL_EP *ep_ptr, dto_status = dapls_ib_get_dto_status(cqe_ptr); dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: dto_ptr %p ext_ptr %p\n", - dto, ext_data); + " cqe_to_event_ext: dto_ptr %p ext_ptr %p status %d\n", + dto, ext_data, dto_status); event_ptr->event_number = DAT_IB_DTO_EVENT; dto->ep_handle = cookie->ep; diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c index b586ca1..153ce76 100755 --- a/test/dtest/dtestx.c +++ b/test/dtest/dtestx.c @@ -56,7 +56,7 @@ #define DTO_TIMEOUT (1000*1000*5) #define CONN_TIMEOUT (1000*1000*10) -#define SERVER_TIMEOUT (1000*1000*20) +#define SERVER_TIMEOUT (1000*1000*120) #define SERVER_CONN_QUAL 31111 #define BUF_SIZE 256 #define BUF_SIZE_ATOMIC 8 @@ -90,7 +90,7 @@ DAT_PSP_HANDLE psp = DAT_HANDLE_NULL; DAT_CR_HANDLE cr = DAT_HANDLE_NULL; int server; -char *usage = "[-s] | hostname\n"; +char *usage = "-s | hostname (default == -s)\n"; void send_msg( @@ -188,7 +188,7 @@ connect_ep(char *hostname) region, BUF_SIZE, pz, - DAT_MEM_PRIV_ALL_FLAG, + DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC, DAT_VA_TYPE_VA, &lmr[ i ], &lmr_context[ i ], @@ -196,8 +196,6 @@ connect_ep(char *hostname) ®_size[ i ], ®_addr[ i ]); _OK(status, "dat_lmr_create"); - printf(" buf[%d]=%p, reg_addr[%d]=%p\n", - i, buf[ i ], i, reg_addr[ i ]); } /* register atomic return buffer for original data */ @@ -208,7 +206,7 @@ connect_ep(char *hostname) region, BUF_SIZE_ATOMIC, pz, - DAT_MEM_PRIV_ALL_FLAG, + DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC, DAT_VA_TYPE_VA, &lmr_atomic, &lmr_atomic_context, @@ -360,16 +358,19 @@ int disconnect_ep() { DAT_RETURN status; - int i; + int i; + DAT_EVENT event; + DAT_COUNT nmore; status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT); _OK(status, "dat_ep_disconnect"); - printf("EP disconnected\n"); - + status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore); + _OK(status, "dat_ep_disconnect"); + if (server) { status = dat_psp_free(psp); - _OK(status, "dat_ep_disconnect"); + _OK(status, "dat_psp_free"); } for (i = 0; i < REG_MEM_COUNT; i++) { @@ -443,7 +444,6 @@ do_immediate() immed_data, DAT_COMPLETION_DEFAULT_FLAG); _OK(status, "dat_ib_post_rdma_write_immed"); - printf("dat_ib_post_rdma_write_immed posted\n"); /* * Collect first event, write completion or the inbound recv with immed @@ -552,13 +552,14 @@ do_immediate() } if (server) - printf("SUCCESS: Server received immed_data=0x%x\n", immed_data_recv); + printf("Server received immed_data=0x%x\n", immed_data_recv); else - printf("SUCCESS: Client received immed_data=0x%x\n", immed_data_recv); + printf("Client received immed_data=0x%x\n", immed_data_recv); - printf("RCV buffer %p contains: %s\n", + printf("rdma buffer %p contains: %s\n", buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]); + printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n"); return (0); } @@ -599,7 +600,7 @@ do_cmp_swap() DAT_COMPLETION_DEFAULT_FLAG); } else { *target = 0x54321; - sleep(1); + sleep(1); /* client does compare and should swap */ status = dat_ib_post_cmp_and_swap( ep, (DAT_UINT64)0x12345, @@ -610,8 +611,6 @@ do_cmp_swap() DAT_COMPLETION_DEFAULT_FLAG); } _OK(status, "dat_ib_post_cmp_and_swap"); - printf("dat_ib_post_cmp_and_swap posted\n"); - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore); _OK(status, "dat_evd_wait for compare and swap"); if (event.event_number != DAT_IB_DTO_EVENT) { @@ -628,15 +627,25 @@ do_cmp_swap() *atomic_buf); exit(1); } - + sleep(1); /* wait for other side to complete swap */ if (server) { printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf); printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target); + + if (*atomic_buf != 0x54321 || *target != 0x98765) { + printf("ERROR: Server CMP_SWAP\n"); + exit(1); + } } else { printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf); printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target); + + if (*atomic_buf != 0x12345 || *target != 0x54321) { + printf("ERROR: Client CMP_SWAP\n"); + exit(1); + } } - sleep(1); + printf("\n CMP_SWAP test - PASSED\n"); return(0); } @@ -665,6 +674,9 @@ do_fetch_add() cookie.as_64 = 0x7777; if (server) { + /* Wait for client to finish cmp_swap */ + while (*target != 0x98765) + sleep(1); *target = 0x10; sleep(1); status = dat_ib_post_fetch_and_add( ep, @@ -674,6 +686,7 @@ do_fetch_add() &r_iov, DAT_COMPLETION_DEFAULT_FLAG); } else { + /* Wait for server, no swap so nothing to check */ *target = 0x100; sleep(1); status = dat_ib_post_fetch_and_add( ep, @@ -684,7 +697,6 @@ do_fetch_add() DAT_COMPLETION_DEFAULT_FLAG); } _OK(status, "dat_ib_post_fetch_and_add"); - printf("dat_ib_post_fetch_and_add posted\n"); status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore); _OK(status, "dat_evd_wait for fetch and add"); if (event.event_number != DAT_IB_DTO_EVENT) { @@ -741,28 +753,40 @@ do_fetch_add() exit(1); } - sleep(1); + sleep(1); /* wait for other side to complete fetch_add */ + if (server) { printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf); printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target); + + if (*atomic_buf != 0x200 || *target != 0x30) { + printf("ERROR: Server FETCH_ADD\n"); + exit(1); + } } else { printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf); printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target); - } + if (*atomic_buf != 0x20 || *target != 0x300) { + printf("ERROR: Server FETCH_ADD\n"); + exit(1); + } + } + printf("\n FETCH_ADD test - PASSED\n"); return(0); } int main(int argc, char **argv) { - char *hostname; + char *hostname; - if (argc != 2) { + if (argc > 2) { printf(usage); exit(1); } - if (strcmp(argv[ 1 ], "-s") == 0) + + if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0) { server = 1; } else { @@ -770,6 +794,7 @@ main(int argc, char **argv) hostname = argv[ 1 ]; } + /* * connect */