From: Sean Hefty Date: Thu, 29 Oct 2009 21:02:22 +0000 (-0800) Subject: ibacm: rework socket interface messages X-Git-Tag: 1.0.0~14 X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=3d12100492cee6e354032df9be2226642da5f8ef;p=~shefty%2Fibacm.git ibacm: rework socket interface messages Rather than indicating the src/dst address type in the acm message header, indicate the type in the acm_ep_addr structure directly. This adds more flexibility and will permit additional acm_ep_addr structures to be carried in a single message request or response. Allow multiple addresses in a request or response message. This is needed to support a wider range of fabric topologies, where the inbound and outbound paths may differ, plus support failover. Add path records to the resolve message, rather than as a separate operation. Return path records directly from the lookup. Signed-off-by: Sean Hefty --- diff --git a/Makefile.am b/Makefile.am index eacece8..7ae23c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,8 +32,6 @@ man_MANS = \ man/ib_acm_resolve_name.3 \ man/ib_acm_resolve_ip.3 \ man/ib_acm_resolve_path.3 \ - man/ib_acm_query_path.3 \ - man/ib_acm_convert_to_path.3 \ man/ib_acme.1 \ man/ib_acm.1 \ man/ib_acm.7 diff --git a/include/infiniband/acm.h b/include/infiniband/acm.h index 56e6553..debcdfc 100644 --- a/include/infiniband/acm.h +++ b/include/infiniband/acm.h @@ -32,106 +32,83 @@ #include -#define ACM_VERSION 1 - -#define ACM_OP_MASK 0x0F -#define ACM_OP_RESOLVE 0x01 -#define ACM_OP_QUERY 0x02 -//#define ACM_OP_CM 0x03 -//#define ACM_OP_ACK_REQ 0x40 /* optional ack is required */ -#define ACM_OP_ACK 0x80 - -#define ACM_STATUS_SUCCESS 0 -#define ACM_STATUS_ENOMEM 1 -#define ACM_STATUS_EINVAL 2 -#define ACM_STATUS_ENODATA 3 -#define ACM_STATUS_ENOTCONN 5 -#define ACM_STATUS_ETIMEDOUT 6 -#define ACM_STATUS_ESRCADDR 7 -#define ACM_STATUS_ESRCTYPE 8 -#define ACM_STATUS_EDESTADDR 9 -#define ACM_STATUS_EDESTTYPE 10 +#define ACM_VERSION 1 + +#define ACM_OP_MASK 0x0F +#define ACM_OP_RESOLVE 0x01 +#define ACM_OP_ACK 0x80 + +#define ACM_STATUS_SUCCESS 0 +#define ACM_STATUS_ENOMEM 1 +#define ACM_STATUS_EINVAL 2 +#define ACM_STATUS_ENODATA 3 +#define ACM_STATUS_ENOTCONN 5 +#define ACM_STATUS_ETIMEDOUT 6 +#define ACM_STATUS_ESRCADDR 7 +#define ACM_STATUS_ESRCTYPE 8 +#define ACM_STATUS_EDESTADDR 9 +#define ACM_STATUS_EDESTTYPE 10 + +#define ACM_FLAGS_QUERY_SA (1<<31) + +#define ACM_MSG_HDR_LENGTH 16 +#define ACM_MAX_ADDRESS 64 +#define ACM_MSG_EP_LENGTH 72 +/* + * Support up to 6 path records (primary and alternate CM paths, + * inbound and outbound primary and alternate data paths), plus CM data. + */ +#define ACM_MSG_DATA_LENGTH (ACM_MSG_EP_LENGTH * 8) struct acm_hdr { - uint8_t version; - uint8_t opcode; - uint8_t status; - uint8_t param; - uint8_t dest_type; - uint8_t src_type; - uint8_t reserved[2]; - uint64_t tid; + uint8_t version; + uint8_t opcode; + uint8_t status; + uint8_t reserved[3]; + uint16_t length; + uint64_t tid; }; -#define ACM_EP_TYPE_NAME 0x01 -#define ACM_EP_TYPE_ADDRESS_IP 0x02 -#define ACM_EP_TYPE_ADDRESS_IP6 0x03 -#define ACM_EP_TYPE_DEVICE 0x10 -#define ACM_EP_TYPE_AV 0x20 - -#define ACM_MAX_ADDRESS 32 +#define ACM_EP_INFO_NAME 0x0001 +#define ACM_EP_INFO_ADDRESS_IP 0x0002 +#define ACM_EP_INFO_ADDRESS_IP6 0x0003 +#define ACM_EP_INFO_PATH 0x0010 +#define ACM_EP_INFO_CM 0x0100 -union acm_ep_addr +struct acm_cm_data { - uint8_t addr[ACM_MAX_ADDRESS]; - uint8_t name[ACM_MAX_ADDRESS]; - struct ib_acm_dev_addr dev; - struct ibv_ah_attr av; + uint8_t init_depth; + uint8_t resp_resources; + uint8_t reserved2[62]; }; -struct acm_resolve_msg +union acm_ep_info { - struct acm_hdr hdr; - union acm_ep_addr src; - union acm_ep_addr dest; - struct ib_acm_resolve_data data; + uint8_t addr[ACM_MAX_ADDRESS]; + uint8_t name[ACM_MAX_ADDRESS]; + struct acm_cm_data cm; + struct ib_path_record path; }; -//struct acm_cm_param -//{ -// uint32_t qpn; -// uint8_t init_depth; -// uint8_t resp_resources; -// uint8_t retry_cnt; -// uint8_t rnr_retry_cnt; -// uint16_t src_port; -// uint16_t dest_port; -// uint8_t reserved[4]; -//}; - -//struct acm_cm_msg -//{ -// struct acm_hdr hdr; -// union acm_ep_addr src; -// union acm_ep_addr dest; -// struct acm_cm_param param; -//}; - -#define ACM_QUERY_PATH_RECORD 0x01 -#define ACM_QUERY_SA 0x80 - -#define ACM_EP_TYPE_LID 0x01 -#define ACM_EP_TYPE_GID 0x02 - -union acm_query_data +struct acm_ep_addr_data { - struct ib_path_record path; + uint32_t flags; + uint16_t type; + uint16_t reserved; + union acm_ep_info info; }; -struct acm_query_msg +struct acm_resolve_msg { - struct acm_hdr hdr; - union acm_query_data data; - uint8_t reserved[16]; + struct acm_hdr hdr; + struct acm_ep_addr_data data[0]; }; -#define ACM_MSG_DATA_SIZE 80 - struct acm_msg { - struct acm_hdr hdr; - uint8_t data[ACM_MSG_DATA_SIZE]; + struct acm_hdr hdr; + uint8_t data[ACM_MSG_DATA_LENGTH]; }; #endif /* ACM_H */ diff --git a/include/infiniband/ib_acm.h b/include/infiniband/ib_acm.h index 6618e84..09b86c2 100644 --- a/include/infiniband/ib_acm.h +++ b/include/infiniband/ib_acm.h @@ -42,22 +42,49 @@ extern "C" { #endif -struct ib_acm_dev_addr +#define IB_PATH_RECORD_REVERSIBLE 0x80 + +struct ib_path_record +{ + uint64_t service_id; + union ibv_gid dgid; + union ibv_gid sgid; + uint16_t dlid; + uint16_t slid; + uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/ + uint8_t tclass; + uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */ + uint16_t pkey; + uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */ + uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */ + uint8_t rate; /* rate selector-7:6 rate-5:0 */ + uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */ + uint8_t preference; + uint8_t reserved[6]; +}; + +#define IB_ACM_FLAGS_CM (1<<0) +#define IB_ACM_FLAGS_PRIMARY (1<<1) +#define IB_ACM_FLAGS_ALTERNATE (1<<2) +#define IB_ACM_FLAGS_OUTBOUND (1<<3) +#define IB_ACM_FLAGS_INBOUND (1<<4) +#define IB_ACM_FLAGS_INBOUND_REVERSE (1<<5) +#define IB_ACM_FLAGS_BIDIRECTIONAL (IB_ACM_FLAGS_OUTBOUND | IB_ACM_FLAGS_INBOUND_REVERSE) + +struct ib_acm_path_data { - uint64_t guid; - uint16_t pkey_index; - uint8_t port_num; - uint8_t reserved[5]; + uint32_t flags; + uint32_t reserved; + struct ib_path_record path; }; -struct ib_acm_resolve_data +struct ib_acm_cm_data { - uint32_t reserved1; uint8_t init_depth; uint8_t resp_resources; - uint8_t packet_lifetime; - uint8_t mtu; - uint8_t reserved2[8]; + uint8_t reserved2; + uint8_t cm_data_length; + uint32_t cm_data[15]; }; /** @@ -73,8 +100,8 @@ struct ib_acm_resolve_data */ LIB_EXPORT int ib_acm_resolve_name(char *src, char *dest, - struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, - struct ib_acm_resolve_data *data); + struct ib_acm_path_data **paths, int *count, + struct ib_acm_cm_data *data); /** * ib_acm_resolve_ip - Resolve path data between the specified addresses. @@ -89,30 +116,12 @@ int ib_acm_resolve_name(char *src, char *dest, */ LIB_EXPORT int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest, - struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, - struct ib_acm_resolve_data *data); + struct ib_acm_path_data **paths, int *count, + struct ib_acm_cm_data *data); +#define ib_acm_free_paths(paths) free(paths) -#define IB_PATH_RECORD_REVERSIBLE 0x80 - -struct ib_path_record -{ - uint64_t service_id; - union ibv_gid dgid; - union ibv_gid sgid; - uint16_t dlid; - uint16_t slid; - uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/ - uint8_t tclass; - uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */ - uint16_t pkey; - uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */ - uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */ - uint8_t rate; /* rate selector-7:6 rate-5:0 */ - uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */ - uint8_t preference; - uint8_t reserved[6]; -}; +#define IB_ACM_FLAGS_QUERY_SA (1<<31) /** * ib_acm_resolve_path - Resolve path data meeting specified restrictions @@ -125,32 +134,7 @@ struct ib_path_record * caller must provide at least the source and destination LIDs as input. */ LIB_EXPORT -int ib_acm_resolve_path(struct ib_path_record *path); - -/** - * ib_acm_query_path - Resolve path data meeting specified restrictions - * Description: - * Queries the IB SA for a path record using the provided path record to - * restrict the query. - * Notes: - * Uses the provided path record as input into an SA query for path - * information. If successful, fills in any missing information. The - * caller must provide at least the source and destination LIDs as input. - * Use of this call always results in sending a query to the IB SA. - */ -LIB_EXPORT -int ib_acm_query_path(struct ib_path_record *path); - -/** - * ib_acm_convert_to_path - Convert resolved path data to a path record - * Description: - * Converts path information returned from resolving a host name or address - * to the format of an IB path record. - */ -LIB_EXPORT -int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr, - struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data, - struct ib_path_record *path); +int ib_acm_resolve_path(struct ib_path_record *path, uint32_t flags); #ifdef __cplusplus } diff --git a/linux/libibacm.map b/linux/libibacm.map index 4aae477..1b0aab0 100644 --- a/linux/libibacm.map +++ b/linux/libibacm.map @@ -1,9 +1,7 @@ IBACM_1.0 { global: - ib_acm_resolve_ip; ib_acm_resolve_name; + ib_acm_resolve_ip; ib_acm_resolve_path; - ib_acm_query_path; - ib_acm_convert_to_path; local: *; }; diff --git a/man/ib_acm.7 b/man/ib_acm.7 index 77df9c8..750c579 100644 --- a/man/ib_acm.7 +++ b/man/ib_acm.7 @@ -30,5 +30,5 @@ success .IP "!= 0" error .SH "SEE ALSO" -ib_acm_resolve_ip(3), ib_acm_resolve_name(3), ib_acm_resolve_path(3), -ib_acm_query_path(3), ib_acm_convert_to_path(3), ib_acme(1), ib_acm(1) +ib_acm_resolve_name(3), ib_acm_resolve_ip(3), ib_acm_resolve_path(3) +ib_acme(1), ib_acm(1) diff --git a/man/ib_acm_convert_to_path.3 b/man/ib_acm_convert_to_path.3 deleted file mode 100644 index da56324..0000000 --- a/man/ib_acm_convert_to_path.3 +++ /dev/null @@ -1,25 +0,0 @@ -.TH "IB_ACM_CONVERT_TO_PATH" 3 "2009-09-09" "libibacm" "Libibacm Programmer's Manual" libibacm -.SH NAME -ib_acm_convert_to_path \- Convert resolved path data to a path record. -.SH SYNOPSIS -.B "#include " -.P -.B "int" ib_acm_convert_to_path -.BI "(struct ib_acm_dev_addr *" dev_addr "," -.BI "struct ibv_ah_attr *" ah "," -.BI "struct ib_acm_resolve_data *" data "," -.BI "struct ib_path_record *" path ");" -.SH ARGUMENTS -.IP "dev_addr" 12 -Local IB device address information. -.IP "ah" 12 -Address handle attributes to send to the destination. -.IP "data" 12 -Additional route and endpoint information. -.SH "DESCRIPTION" -Converts path information obtained from ib_acm_resolve_ip or ib_acm_resolve_name -to the format of an IB path record. -.SH "SEE ALSO" -ib_acm_resolve_name(3), ib_acm_resolve_ip(3), ib_acm_resolve_path(3), -ib_acm_query_path(3), ib_acm(7) - diff --git a/man/ib_acm_query_path.3 b/man/ib_acm_query_path.3 deleted file mode 100644 index 986ca41..0000000 --- a/man/ib_acm_query_path.3 +++ /dev/null @@ -1,24 +0,0 @@ -.TH "IB_ACM_QUERY_PATH" 3 "2009-09-09" "libibacm" "Libibacm Programmer's Manual" libibacm -.SH NAME -ib_acm_query_path \- Resolve path data meeting specified restrictions. -.SH SYNOPSIS -.B "#include " -.P -.B "int" ib_acm_resolve_name -.BI "(struct ib_path_record *" path ");" -.SH ARGUMENTS -.IP "path" 12 -On input, provides path information used to restrict path resolution. On -output, contains the full path record. -.SH "DESCRIPTION" -Queries the IB SA for a path record using the provided path record to -restrict the query. -.SH "NOTES" -Uses the provided path record as input into an SA query for path -information. If successful, fills in any missing information. The -caller must provide at least the source and destination LIDs as input. -Use of this call always results in sending a query to the IB SA. -.SH "SEE ALSO" -ib_acm_resolve_name(3), ib_acm_resolve_ip(3), ib_acm_resolve_path(3), -ib_acm_convert_to_path(3), ib_acm(7) - diff --git a/man/ib_acm_resolve_ip.3 b/man/ib_acm_resolve_ip.3 index 62db005..43e7478 100644 --- a/man/ib_acm_resolve_ip.3 +++ b/man/ib_acm_resolve_ip.3 @@ -5,36 +5,30 @@ ib_acm_resolve_ip \- Resolve path data between the specified addresses. .B "#include " .P .B "int" ib_acm_resolve_ip -.BI "(struct sockaddr *" src "," -.BI "struct sockaddr *" dest "," -.BI "struct ib_acm_dev_addr *" dev_addr "," -.BI "struct ibv_ah_attr *" ah "," -.BI "struct ib_acm_resolve_data *" data ");" +.BI "(struct sockaddr *" src "," +.BI "struct sockaddr *" dest "," +.BI "struct ib_acm_path_data *" paths "," +.BI "int *" count "," +.BI "struct ib_acm_cm_data *" data ");" .SH ARGUMENTS .IP "src" 12 Source address. .IP "dest" 12 Destination address. -.IP "dev_addr" 12 -On success, local IB device address information. This includes the -local device GUID, port number, and pkey index to use for communication. -.IP "ah" 12 -On success, address handle attributes to send to the destination. The -address handle attributes may be used to configure a connected QP or -create an address handle for datagram traffic. +.IP "paths" 12 +On success, returns path record information for the connection. +.IP "count" 12 +On success, indicates the number of paths returned. .IP "data" 12 -On success, additional route and endpoint information. This includes -the MTU, packet lifetime, and maximum initiator depth and responder -resources information usable between the end points. +On success, additional information required for IB CM communication. .SH "DESCRIPTION" -Discover path information, including identifying the local device, +Discover path information, including identifying the local device, between the given the source and destination addresses. .SH "NOTES" -The source and destination addresses should match entries in acm_addr.cfg -configuration files on their respective systems. Typically, the -source and destination addresses will refer to IP addresses assigned -to an IPoIB instance. +The source and destination addresses should match entries in acm_addr.cfg +configuration files on their respective systems. Typically, the +source and destination addresses will refer to IP addresses assigned +to an IPoIB instance. The caller must free the returned paths array. .SH "SEE ALSO" -ib_acm_resolve_name(3), ib_acm_resolve_path(3), ib_acm_query_path(3), -ib_acm_convert_to_path(3), ib_acm(7) +ib_acm_resolve_name(3), ib_acm_resolve_path(3), ib_acm(7) diff --git a/man/ib_acm_resolve_name.3 b/man/ib_acm_resolve_name.3 index d7bf9fb..4b3ccf3 100644 --- a/man/ib_acm_resolve_name.3 +++ b/man/ib_acm_resolve_name.3 @@ -5,36 +5,31 @@ ib_acm_resolve_name \- Resolve path data between the specified names. .B "#include " .P .B "int" ib_acm_resolve_name -.BI "(char *" src "," -.BI "char *" dest "," -.BI "struct ib_acm_dev_addr *" dev_addr "," -.BI "struct ibv_ah_attr *" ah "," -.BI "struct ib_acm_resolve_data *" data ");" +.BI "(char *" src "," +.BI "char *" dest "," +.BI "struct ib_acm_path_data *" paths "," +.BI "int *" count "," +.BI "struct ib_acm_cm_data *" data ");" .SH ARGUMENTS .IP "src" 12 Source system network name or host name. .IP "dest" 12 Destination system network name or host name. -.IP "dev_addr" 12 -On success, local IB device address information. This includes the -local device GUID, port number, and pkey index to use for communication. -.IP "ah" 12 -On success, address handle attributes to send to the destination. The -address handle attributes may be used to configure a connected QP or -create an address handle for datagram traffic. +.IP "paths" 12 +On success, returns path record information for the connection. +.IP "count" 12 +On success, indicates the number of paths returned. .IP "data" 12 -On success, additional route and endpoint information. This includes -the MTU, packet lifetime, and maximum initiator depth and responder -resources information usable between the end points. +On success, additional information required for IB CM communication. .SH "DESCRIPTION" -Discover path information, including identifying the local device, +Discover path information, including identifying the local device, between the given the source and destination names. .SH "NOTES" -The source and destination names should match entries in acm_addr.cfg -configuration files on their respective systems. Typically, the -source and destination names will refer to system host names -assigned to an Infiniband port. +The source and destination names should match entries in acm_addr.cfg +configuration files on their respective systems. Typically, the +source and destination names will refer to system host names +assigned to an Infiniband port. The caller must free the returned +paths array. .SH "SEE ALSO" -ib_acm_resolve_ip(3), ib_acm_resolve_path(3), ib_acm_query_path(3), -ib_acm_convert_to_path(3), ib_acm(7) +ib_acm_resolve_ip(3), ib_acm_resolve_path(3), ib_acm(7) diff --git a/man/ib_acm_resolve_path.3 b/man/ib_acm_resolve_path.3 index 11ada6b..fd13190 100644 --- a/man/ib_acm_resolve_path.3 +++ b/man/ib_acm_resolve_path.3 @@ -4,21 +4,24 @@ ib_acm_resolve_path \- Resolve path data meeting specified restrictions. .SH SYNOPSIS .B "#include " .P -.B "int" ib_acm_resolve_name -.BI "(struct ib_path_record *" path ");" +.B "int" ib_acm_resolve_path +.BI "(struct ib_path_record *" path "," +.BI "uint32_t " flags ");" .SH ARGUMENTS .IP "path" 12 On input, provides path information used to restrict path resolution. On output, contains the full path record. +.IP "flags" 12 +If set to IB_ACM_FLAGS_QUERY_SA, the requested path data will be obtained +by querying the SA directly. Otherwise, cached data may be provided. .SH "DESCRIPTION" -Discover path information using the provided path record to -restrict the discovery. +Discover path information using the provided path record to +restrict the discovery. .SH "NOTES" -Uses the provided path record as input into an query for path -information. If successful, fills in any missing information. The -caller must provide at least the source and destination LIDs as input. -The returned path record may come from cached data. +Uses the provided path record as input into an query for path +information. If successful, fills in any missing information. The +caller must provide at least the source and destination LIDs as input. +The returned path record may come from cached data. .SH "SEE ALSO" -ib_acm_resolve_name(3), ib_acm_resolve_ip(3), ib_acm_query_path(3), -ib_acm_convert_to_path(3), ib_acm(7) +ib_acm_resolve_name(3), ib_acm_resolve_ip(3), ib_acm(7) diff --git a/src/acm.c b/src/acm.c index f4e086e..0311b52 100644 --- a/src/acm.c +++ b/src/acm.c @@ -43,16 +43,15 @@ struct acm_dest { - uint8_t address[ACM_MAX_ADDRESS]; /* keep first */ - struct ibv_ah *ah; - struct ibv_ah_attr av; - union ibv_gid mgid; - DLIST_ENTRY req_queue; - uint32_t remote_qpn; - uint8_t init_depth; - uint8_t resp_resources; - uint8_t mtu; - uint8_t packet_lifetime; + uint8_t address[ACM_MAX_ADDRESS]; /* keep first */ + struct ibv_ah *ah; + struct ibv_ah_attr av; + struct ib_path_record path; + union ibv_gid mgid; + DLIST_ENTRY req_queue; + uint32_t remote_qpn; + uint8_t init_depth; + uint8_t resp_resources; }; struct acm_port @@ -95,7 +94,7 @@ struct acm_ep struct ibv_mr *mr; uint8_t *recv_bufs; DLIST_ENTRY entry; - union acm_ep_addr addr[MAX_EP_ADDR]; + union acm_ep_info addr[MAX_EP_ADDR]; uint8_t addr_type[MAX_EP_ADDR]; void *dest_map[ACM_ADDRESS_RESERVED - 1]; struct acm_dest mc_dest[MAX_EP_MC]; @@ -176,9 +175,9 @@ static void acm_write(int level, const char *format, ...) va_end(args); } -static void acm_log_ep_addr(int level, const char *msg, - union acm_ep_addr *addr, uint8_t ep_type) +static void acm_log_ep_addr(int level, const char *msg, uint16_t type, uint8_t *data) { + struct ib_path_record *path; char ip_addr[ACM_MAX_ADDRESS]; if (level > log_level) @@ -186,27 +185,28 @@ static void acm_log_ep_addr(int level, const char *msg, lock_acquire(&log_lock); fprintf(flog, msg); - switch (ep_type) { - case ACM_EP_TYPE_NAME: - fprintf(flog, "%s\n", addr->name); + switch (type) { + case ACM_EP_INFO_NAME: + fprintf(flog, "%s\n", data); break; - case ACM_EP_TYPE_ADDRESS_IP: - inet_ntop(AF_INET, addr->addr, ip_addr, ACM_MAX_ADDRESS); + case ACM_EP_INFO_ADDRESS_IP: + inet_ntop(AF_INET, data, ip_addr, ACM_MAX_ADDRESS); fprintf(flog, "%s\n", ip_addr); break; - case ACM_EP_TYPE_ADDRESS_IP6: - inet_ntop(AF_INET6, addr->addr, ip_addr, ACM_MAX_ADDRESS); + case ACM_EP_INFO_ADDRESS_IP6: + inet_ntop(AF_INET6, data, ip_addr, ACM_MAX_ADDRESS); fprintf(flog, "%s\n", ip_addr); break; - case ACM_EP_TYPE_DEVICE: - fprintf(flog, "device guid 0x%llx, pkey index %d, port %d\n", - addr->dev.guid, addr->dev.pkey_index, addr->dev.port_num); + case ACM_EP_INFO_CM: + fprintf(flog, "cm data not displayed\n"); break; - case ACM_EP_TYPE_AV: - fprintf(flog, "endpoint specified using address vector\n"); + case ACM_EP_INFO_PATH: + path = (struct ib_path_record *) data; + fprintf(flog, "path record, SLID 0x%x, DLID 0x%x\n", + ntohs(path->slid), ntohs(path->dlid)); break; default: - fprintf(flog, "unknown endpoint address 0x%x\n", ep_type); + fprintf(flog, "unknown endpoint address 0x%x\n", type); } lock_release(&log_lock); } @@ -390,25 +390,38 @@ static int acm_mc_index(struct acm_ep *ep, union ibv_gid *gid) } static void -acm_init_mc_av(struct acm_port *port, struct ib_mc_member_rec *mc_rec, - struct ibv_ah_attr *av) +acm_record_mc_av(struct acm_port *port, struct ib_mc_member_rec *mc_rec, + struct acm_dest *dest) { uint32_t sl_flow_hop; sl_flow_hop = ntohl(mc_rec->sl_flow_hop); - av->dlid = ntohs(mc_rec->mlid); - av->sl = (uint8_t) (sl_flow_hop >> 28); - av->src_path_bits = port->sa_dest.av.src_path_bits; - av->static_rate = mc_rec->rate & 0x3F; - av->port_num = port->port_num; - - av->is_global = 1; - av->grh.dgid = mc_rec->mgid; - av->grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF; - av->grh.sgid_index = acm_gid_index(port, &mc_rec->port_gid); - av->grh.hop_limit = (uint8_t) sl_flow_hop; - av->grh.traffic_class = mc_rec->tclass; + dest->av.dlid = ntohs(mc_rec->mlid); + dest->av.sl = (uint8_t) (sl_flow_hop >> 28); + dest->av.src_path_bits = port->sa_dest.av.src_path_bits; + dest->av.static_rate = mc_rec->rate & 0x3F; + dest->av.port_num = port->port_num; + + dest->av.is_global = 1; + dest->av.grh.dgid = mc_rec->mgid; + dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF; + dest->av.grh.sgid_index = acm_gid_index(port, &mc_rec->port_gid); + dest->av.grh.hop_limit = (uint8_t) sl_flow_hop; + dest->av.grh.traffic_class = mc_rec->tclass; + + dest->path.dgid = mc_rec->mgid; + dest->path.sgid = mc_rec->port_gid; + dest->path.dlid = mc_rec->mlid; + dest->path.slid = htons(port->lid) | port->sa_dest.av.src_path_bits; + dest->path.flowlabel_hoplimit = htonl(sl_flow_hop & 0xFFFFFFF); + dest->path.tclass = mc_rec->tclass; + dest->path.reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1; + dest->path.pkey = mc_rec->pkey; + dest->path.qosclass_sl = htons((uint16_t) (sl_flow_hop >> 28)); + dest->path.mtu = mc_rec->mtu; + dest->path.rate = mc_rec->rate; + dest->path.packetlifetime = mc_rec->packet_lifetime; } static void acm_process_join_resp(struct acm_ep *ep, struct ib_user_mad *umad) @@ -437,9 +450,7 @@ static void acm_process_join_resp(struct acm_ep *ep, struct ib_user_mad *umad) dest = &ep->mc_dest[index]; dest->remote_qpn = IB_MC_QPN; dest->mgid = mc_rec->mgid; - acm_init_mc_av(ep->port, mc_rec, &dest->av); - dest->mtu = mc_rec->mtu & 0x3F; - dest->packet_lifetime = mc_rec->packet_lifetime & 0x3F; + acm_record_mc_av(ep->port, mc_rec, dest); dest->ah = ibv_create_ah(ep->port->dev->pd, &dest->av); ret = ibv_attach_mcast(ep->qp, &mc_rec->mgid, mc_rec->mlid); if (ret) { @@ -493,8 +504,10 @@ acm_record_av(struct acm_dest *dest, struct acm_ep *ep, dest->av.grh.dgid = ((struct ibv_grh *) (uintptr_t) wc->wr_id)->sgid; dest->mgid = ep->mc_dest[index].mgid; - dest->mtu = ep->mc_dest[index].mtu; - dest->packet_lifetime = ep->mc_dest[index].packet_lifetime; + + dest->path = ep->mc_dest[index].path; + dest->path.dgid = dest->av.grh.dgid; + dest->path.dlid = htons(dest->av.dlid); return ACM_STATUS_SUCCESS; } } @@ -651,11 +664,15 @@ acm_process_resolve_req(struct acm_ep *ep, struct ibv_wc *wc, struct acm_mad *ma static int acm_client_resolve_resp(struct acm_ep *ep, struct acm_client *client, - struct acm_resolve_msg *msg, struct acm_dest *dest, uint8_t status) + struct acm_resolve_msg *req_msg, struct acm_dest *dest, uint8_t status) { + struct acm_msg msg; + struct acm_resolve_msg *resp_msg = (struct acm_resolve_msg *) &msg; int ret; acm_log(1, "status 0x%x\n", status); + memset(&msg, 0, sizeof msg); + lock_acquire(&client->lock); if (client->sock == INVALID_SOCKET) { acm_log(0, "ERROR - connection lost\n"); @@ -663,29 +680,27 @@ acm_client_resolve_resp(struct acm_ep *ep, struct acm_client *client, goto release; } - msg->hdr.opcode |= ACM_OP_ACK; - msg->hdr.status = status; - msg->hdr.param = 0; + resp_msg->hdr = req_msg->hdr; + resp_msg->hdr.opcode |= ACM_OP_ACK; + resp_msg->hdr.status = status; + resp_msg->hdr.length = ACM_MSG_HDR_LENGTH; if (!status) { - msg->hdr.src_type = ACM_EP_TYPE_DEVICE; - msg->src.dev.guid = ep->port->dev->guid; - msg->src.dev.pkey_index = ep->pkey_index; - msg->src.dev.port_num = ep->port->port_num; - - if (dest) { - acm_log(2, "destination found\n"); - msg->hdr.dest_type = ACM_EP_TYPE_AV; - msg->dest.av = dest->av; - msg->data.init_depth = min(ep->port->dev->init_depth, dest->resp_resources); - msg->data.resp_resources = min(ep->port->dev->resp_resources, dest->init_depth); - msg->data.packet_lifetime = dest->packet_lifetime; - msg->data.mtu = dest->mtu; - } + resp_msg->hdr.length += ACM_MSG_EP_LENGTH; + resp_msg->data[0].flags = IB_ACM_FLAGS_CM | + IB_ACM_FLAGS_PRIMARY | IB_ACM_FLAGS_BIDIRECTIONAL; + resp_msg->data[0].type = ACM_EP_INFO_PATH; + resp_msg->data[0].info.path = dest->path; + + resp_msg->hdr.length += ACM_MSG_EP_LENGTH; + resp_msg->data[1].flags = IB_ACM_FLAGS_CM | IB_ACM_FLAGS_BIDIRECTIONAL; + resp_msg->data[1].type = ACM_EP_INFO_CM; + resp_msg->data[1].info.cm.init_depth = dest->init_depth; + resp_msg->data[1].info.cm.resp_resources = dest->resp_resources; } - ret = send(client->sock, (char *) msg, sizeof *msg, 0); - if (ret != sizeof(*msg)) + ret = send(client->sock, (char *) resp_msg, resp_msg->hdr.length, 0); + if (ret != resp_msg->hdr.length) acm_log(0, "failed to send response\n"); else ret = 0; @@ -1118,8 +1133,7 @@ static void acm_process_timeouts(void) rec = (struct acm_resolve_rec *) mad->data; ep = msg->ep; - acm_log_ep_addr(0, "acm_process_timeouts: dest ", - (union acm_ep_addr *) &rec->dest, rec->dest_type); + acm_log_ep_addr(0, "acm_process_timeouts: dest ", rec->dest_type, rec->dest); lock_acquire(&ep->lock); tdest = tfind(rec->dest, &ep->dest_map[rec->dest_type - 1], acm_compare_dest); if (!tdest) { @@ -1300,18 +1314,9 @@ static void acm_svr_accept(void) acm_log(2, "assigned client id %d\n", i); } -static uint8_t acm_get_addr_type(uint8_t ep_type) -{ - if (ep_type >= ACM_ADDRESS_RESERVED) { - acm_log(0, "ERROR - invalid ep type %d\n", ep_type); - return ACM_ADDRESS_INVALID; - } - return ep_type; -} - static int acm_client_query_resp(struct acm_ep *ep, struct acm_client *client, - struct acm_query_msg *msg, uint8_t status) + struct acm_resolve_msg *msg, uint8_t status) { int ret; @@ -1326,8 +1331,8 @@ acm_client_query_resp(struct acm_ep *ep, struct acm_client *client, msg->hdr.opcode |= ACM_OP_ACK; msg->hdr.status = status; - ret = send(client->sock, (char *) msg, sizeof *msg, 0); - if (ret != sizeof(*msg)) + ret = send(client->sock, (char *) msg, msg->hdr.length, 0); + if (ret != msg->hdr.length) acm_log(0, "failed to send response\n"); else ret = 0; @@ -1338,48 +1343,13 @@ release: return ret; } -static struct acm_ep * -acm_get_ep_by_path(struct ib_path_record *path) -{ - struct acm_device *dev; - struct acm_port *port; - struct acm_ep *ep; - DLIST_ENTRY *dev_entry, *ep_entry; - int i; - - for (dev_entry = dev_list.Next; dev_entry != &dev_list; - dev_entry = dev_entry->Next) { - - dev = container_of(dev_entry, struct acm_device, entry); - for (i = 0; i < dev->port_cnt; i++) { - port = &dev->port[i]; - - // requires slid - if (port->lid != ntohs(path->slid)) - continue; - - for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list; - ep_entry = ep_entry->Next) { - - // ignores pkey - ep = container_of(ep_entry, struct acm_ep, entry); - return ep; - } - } - } - - acm_log(0, "could not find endpoint\n"); - return NULL; -} - // TODO: process send/recv asynchronously -static uint8_t acm_query_sa(struct acm_ep *ep, uint8_t query, union acm_query_data *data) +static uint8_t acm_query_sa(struct acm_ep *ep, struct ib_path_record *path) { struct acm_port *port; struct ib_sa_mad *mad; struct ib_user_mad *umad; int ret, len; - size_t size; acm_log(2, "\n"); len = sizeof(*umad) + sizeof(*mad); @@ -1398,16 +1368,7 @@ static uint8_t acm_query_sa(struct acm_ep *ep, uint8_t query, union acm_query_da umad->addr.path_bits = port->sa_dest.av.src_path_bits; mad = (struct ib_sa_mad *) umad->data; - switch (query) { - case ACM_QUERY_PATH_RECORD: - acm_init_path_query(mad, &data->path); - size = sizeof(data->path); - break; - default: - acm_log(0, "ERROR - unknown attribute id\n"); - ret = ACM_STATUS_EINVAL; - goto out; - } + acm_init_path_query(mad, path); ret = umad_send(port->mad_portid, port->mad_agentid, (void *) umad, sizeof(*mad), timeout, retries); @@ -1423,7 +1384,7 @@ static uint8_t acm_query_sa(struct acm_ep *ep, uint8_t query, union acm_query_da goto out; } - memcpy(data, mad->data, size); + memcpy(path, mad->data, sizeof *path); ret = umad->status ? umad->status : mad->status; if (ret) { acm_log(0, "SA query response error: 0x%x\n", ret); @@ -1434,14 +1395,65 @@ out: return (uint8_t) ret; } +static struct acm_ep * +acm_get_ep(struct acm_ep_addr_data *data) +{ + struct acm_device *dev; + struct acm_port *port; + struct acm_ep *ep; + DLIST_ENTRY *dev_entry, *ep_entry; + int i; + + acm_log_ep_addr(2, "acm_get_ep: ", data->type, data->info.addr); + for (dev_entry = dev_list.Next; dev_entry != &dev_list; + dev_entry = dev_entry->Next) { + + dev = container_of(dev_entry, struct acm_device, entry); + for (i = 0; i < dev->port_cnt; i++) { + port = &dev->port[i]; + + if (data->type == ACM_EP_INFO_PATH && + port->lid != ntohs(data->info.path.slid)) + continue; + + for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list; + ep_entry = ep_entry->Next) { + + ep = container_of(ep_entry, struct acm_ep, entry); + if (data->type == ACM_EP_INFO_PATH) + return ep; // TODO: check pkey + + if (acm_addr_index(ep, data->info.addr, + (uint8_t) data->type) >= 0) + return ep; + } + } + } + + acm_log_ep_addr(0, "acm_get_ep: could not find ", data->type, data->info.addr); + return NULL; +} + static int -acm_svr_query(struct acm_client *client, struct acm_query_msg *msg) +acm_svr_query(struct acm_client *client, struct acm_resolve_msg *msg) { struct acm_ep *ep; uint8_t status; acm_log(2, "processing client query\n"); - ep = acm_get_ep_by_path(&msg->data.path); + if (msg->hdr.length != ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH) { + acm_log(0, "ERROR - invalid length: 0x%x\n", msg->hdr.length); + status = ACM_STATUS_EINVAL; + goto resp; + } + + if (msg->data[0].type != ACM_EP_INFO_PATH) { + acm_log(0, "ERROR - unsupported type: 0x%x\n", msg->data[0].type); + status = ACM_STATUS_EINVAL; + goto resp; + } + + ep = acm_get_ep(&msg->data[0]); if (!ep) { acm_log(0, "could not find local end point\n"); status = ACM_STATUS_ESRCADDR; @@ -1450,7 +1462,7 @@ acm_svr_query(struct acm_client *client, struct acm_query_msg *msg) (void) atomic_inc(&client->refcnt); lock_acquire(&ep->lock); - status = acm_query_sa(ep, msg->hdr.param & ~ACM_QUERY_SA, &msg->data); + status = acm_query_sa(ep, &msg->data[0].info.path); lock_release(&ep->lock); resp: @@ -1458,8 +1470,8 @@ resp: } static uint8_t -acm_send_resolve(struct acm_ep *ep, union acm_ep_addr *src, uint8_t src_type, - struct acm_dest *dest, uint8_t dest_type) +acm_send_resolve(struct acm_ep *ep, struct acm_ep_addr_data *saddr, + struct acm_ep_addr_data *daddr) { struct acm_send_msg *msg; struct acm_mad *mad; @@ -1488,12 +1500,12 @@ acm_send_resolve(struct acm_ep *ep, union acm_ep_addr *src, uint8_t src_type, mad->tid = (uint64_t) atomic_inc(&tid); rec = (struct acm_resolve_rec *) mad->data; - rec->src_type = src_type; + rec->src_type = (uint8_t) saddr->type; rec->src_length = ACM_MAX_ADDRESS; - memcpy(rec->src, src->addr, ACM_MAX_ADDRESS); - rec->dest_type = dest_type; + memcpy(rec->src, saddr->info.addr, ACM_MAX_ADDRESS); + rec->dest_type = (uint8_t) daddr->type; rec->dest_length = ACM_MAX_ADDRESS; - memcpy(rec->dest, dest->address, ACM_MAX_ADDRESS); + memcpy(rec->dest, daddr->info.addr, ACM_MAX_ADDRESS); rec->resp_resources = ep->port->dev->resp_resources; rec->init_depth = ep->port->dev->init_depth; @@ -1505,35 +1517,27 @@ acm_send_resolve(struct acm_ep *ep, union acm_ep_addr *src, uint8_t src_type, return 0; } -static struct acm_ep * -acm_get_ep_by_addr(union acm_ep_addr *addr, uint8_t src_type) +static uint8_t +acm_svr_verify_resolve(struct acm_resolve_msg *msg) { - struct acm_device *dev; - struct acm_port *port; - struct acm_ep *ep; - DLIST_ENTRY *dev_entry, *ep_entry; - int i; - - acm_log_ep_addr(2, "acm_get_ep_by_addr: ", addr, src_type); - for (dev_entry = dev_list.Next; dev_entry != &dev_list; - dev_entry = dev_entry->Next) { - - dev = container_of(dev_entry, struct acm_device, entry); - for (i = 0; i < dev->port_cnt; i++) { - port = &dev->port[i]; + if (msg->hdr.length != ACM_MSG_HDR_LENGTH + 2 * ACM_MSG_EP_LENGTH) { + acm_log(0, "ERROR - invalid msg hdr length %d\n", msg->hdr.length); + return ACM_STATUS_EINVAL; + } - for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list; - ep_entry = ep_entry->Next) { + if ((msg->data[0].flags != IB_ACM_FLAGS_INBOUND) || + !msg->data[0].type || (msg->data[0].type >= ACM_ADDRESS_RESERVED)) { + acm_log(0, "ERROR - source address required first\n"); + return ACM_STATUS_ESRCTYPE; + } - ep = container_of(ep_entry, struct acm_ep, entry); - if (acm_addr_index(ep, addr->addr, src_type) >= 0) - return ep; - } - } + if ((msg->data[1].flags != IB_ACM_FLAGS_OUTBOUND) || + !msg->data[1].type || (msg->data[1].type >= ACM_ADDRESS_RESERVED)) { + acm_log(0, "ERROR - destination address required second\n"); + return ACM_STATUS_EDESTTYPE; } - acm_log_ep_addr(0, "acm_get_ep_by_addr: could not find ", addr, src_type); - return NULL; + return ACM_STATUS_SUCCESS; } static int @@ -1542,28 +1546,30 @@ acm_svr_resolve(struct acm_client *client, struct acm_resolve_msg *msg) struct acm_ep *ep; struct acm_dest *dest, **tdest; struct acm_request *req; - uint8_t dest_type, src_type; + struct acm_ep_addr_data *saddr, *daddr; uint8_t status; - acm_log_ep_addr(2, "acm_svr_resolve: source ", &msg->src, msg->hdr.src_type); - ep = acm_get_ep_by_addr(&msg->src, msg->hdr.src_type); - if (!ep) { - acm_log(0, "unknown local end point\n"); - status = ACM_STATUS_ESRCADDR; + status = acm_svr_verify_resolve(msg); + if (status) { + acm_log(0, "misformatted or unsupported request\n"); goto resp; } - dest_type = acm_get_addr_type(msg->hdr.dest_type); - if (dest_type == ACM_ADDRESS_INVALID) { - acm_log(0, "ERROR - unknown destination type\n"); - status = ACM_STATUS_EDESTTYPE; + saddr = &msg->data[0]; + daddr = &msg->data[1]; + + acm_log_ep_addr(2, "acm_svr_resolve: source ", saddr->type, saddr->info.addr); + ep = acm_get_ep(saddr); + if (!ep) { + acm_log(0, "unknown local end point\n"); + status = ACM_STATUS_ESRCADDR; goto resp; } - acm_log_ep_addr(2, "acm_svr_resolve: dest ", &msg->dest, msg->hdr.dest_type); + acm_log_ep_addr(2, "acm_svr_resolve: dest ", daddr->type, daddr->info.addr); (void) atomic_inc(&client->refcnt); lock_acquire(&ep->lock); - tdest = tfind(msg->dest.addr, &ep->dest_map[dest_type - 1], acm_compare_dest); + tdest = tfind(&daddr->info.addr, &ep->dest_map[daddr->type - 1], acm_compare_dest); dest = tdest ? *tdest : NULL; if (dest && dest->ah) { acm_log(2, "request satisfied from local cache\n"); @@ -1587,17 +1593,16 @@ acm_svr_resolve(struct acm_client *client, struct acm_resolve_msg *msg) goto free_req; } - memcpy(dest->address, msg->dest.addr, ACM_MAX_ADDRESS); - src_type = acm_get_addr_type(msg->hdr.src_type); + memcpy(dest->address, &daddr->info.addr, ACM_MAX_ADDRESS); acm_log(2, "sending resolve msg to dest\n"); - status = acm_send_resolve(ep, &msg->src, src_type, dest, dest_type); + status = acm_send_resolve(ep, saddr, daddr); if (status) { acm_log(0, "ERROR - failure sending resolve request 0x%x\n", status); goto free_dest; } DListInit(&dest->req_queue); - tsearch(dest, &ep->dest_map[dest_type - 1], acm_compare_dest); + tsearch(dest, &ep->dest_map[daddr->type - 1], acm_compare_dest); } acm_log(2, "queuing client request\n"); @@ -1621,11 +1626,12 @@ resp: static void acm_svr_receive(struct acm_client *client) { struct acm_msg msg; + struct acm_resolve_msg *resolve_msg = (struct acm_resolve_msg *) &msg; int ret; acm_log(2, "\n"); ret = recv(client->sock, (char *) &msg, sizeof msg, 0); - if (ret != sizeof msg) { + if (ret != msg.hdr.length) { acm_log(2, "client disconnected\n"); ret = ACM_STATUS_ENOTCONN; goto out; @@ -1636,17 +1642,15 @@ static void acm_svr_receive(struct acm_client *client) goto out; } - switch (msg.hdr.opcode & ACM_OP_MASK) { - case ACM_OP_RESOLVE: - ret = acm_svr_resolve(client, (struct acm_resolve_msg *) &msg); - break; - case ACM_OP_QUERY: - ret = acm_svr_query(client, (struct acm_query_msg *) &msg); - break; - default: + if ((msg.hdr.opcode & ACM_OP_MASK) != ACM_OP_RESOLVE) { acm_log(0, "ERROR - unknown opcode 0x%x\n", msg.hdr.opcode); - ret = -1; - break; + goto out; + } + + if (resolve_msg->data[0].type == ACM_EP_INFO_PATH) { + ret = acm_svr_query(client, resolve_msg); + } else { + ret = acm_svr_resolve(client, resolve_msg); } out: diff --git a/src/acm_mad.h b/src/acm_mad.h index bd1d980..f18e25c 100644 --- a/src/acm_mad.h +++ b/src/acm_mad.h @@ -48,7 +48,6 @@ #define ACM_CTRL_ACK htons(0x8000) #define ACM_CTRL_RESOLVE htons(0x0001) -#define ACM_CTRL_CM htons(0x0002) struct acm_mad { @@ -67,6 +66,7 @@ struct acm_mad #define ACM_QKEY 0x80010000 +/* Map to ACM_EP_INFO_* */ #define ACM_ADDRESS_INVALID 0x00 #define ACM_ADDRESS_NAME 0x01 #define ACM_ADDRESS_IP 0x02 diff --git a/src/acme.c b/src/acme.c index 68f4b80..5cf0c36 100644 --- a/src/acme.c +++ b/src/acme.c @@ -347,11 +347,10 @@ static void show_path(struct ib_path_record *path) static int resolve_ip(struct ib_path_record *path) { - struct ib_acm_dev_addr dev_addr; - struct ibv_ah_attr ah; - struct ib_acm_resolve_data data; + struct ib_acm_path_data *paths; + struct ib_acm_cm_data data; struct sockaddr_in src, dest; - int ret; + int ret, count; src.sin_family = AF_INET; ret = inet_pton(AF_INET, src_addr, &src.sin_addr); @@ -368,37 +367,32 @@ static int resolve_ip(struct ib_path_record *path) } ret = ib_acm_resolve_ip((struct sockaddr *) &src, (struct sockaddr *) &dest, - &dev_addr, &ah, &data); + &paths, &count, &data); if (ret) { printf("ib_acm_resolve_ip failed: 0x%x\n", ret); return ret; } - ret = ib_acm_convert_to_path(&dev_addr, &ah, &data, path); - if (ret) - printf("ib_acm_convert_to_path failed: 0x%x\n", ret); - - return ret; + *path = paths[0].path; + ib_acm_free_paths(paths); + return 0; } static int resolve_name(struct ib_path_record *path) { - struct ib_acm_dev_addr dev_addr; - struct ibv_ah_attr ah; - struct ib_acm_resolve_data data; - int ret; + struct ib_acm_path_data *paths; + struct ib_acm_cm_data data; + int ret, count; - ret = ib_acm_resolve_name(src_addr, dest_addr, &dev_addr, &ah, &data); + ret = ib_acm_resolve_name(src_addr, dest_addr, &paths, &count, &data); if (ret) { printf("ib_acm_resolve_name failed: 0x%x\n", ret); return ret; } - ret = ib_acm_convert_to_path(&dev_addr, &ah, &data, path); - if (ret) - printf("ib_acm_convert_to_path failed: 0x%x\n", ret); - - return ret; + *path = paths[0].path; + ib_acm_free_paths(paths); + return 0; } static int resolve_lid(struct ib_path_record *path) @@ -409,7 +403,7 @@ static int resolve_lid(struct ib_path_record *path) path->dlid = htons((uint16_t) atoi(dest_addr)); path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1; - ret = ib_acm_resolve_path(path); + ret = ib_acm_resolve_path(path, 0); if (ret) printf("ib_acm_resolve_path failed: 0x%x\n", ret); @@ -420,7 +414,7 @@ static int verify_resolve(struct ib_path_record *path) { int ret; - ret = ib_acm_query_path(path); + ret = ib_acm_resolve_path(path, IB_ACM_FLAGS_QUERY_SA); if (ret) printf("SA verification: failed 0x%x\n", ret); else diff --git a/src/libacm.c b/src/libacm.c index fe853ae..b018e20 100644 --- a/src/libacm.c +++ b/src/libacm.c @@ -54,91 +54,6 @@ static SOCKET sock = INVALID_SOCKET; static short server_port = 6125; static int ready; -static struct acm_device *dev_array; -static int dev_cnt; - - -static int acm_init_port(struct acm_device *dev, int index) -{ - struct acm_port *port; - struct ibv_port_attr attr; - int ret; - - port = &dev->ports[index]; - port->port_num = index + 1; - ret = ibv_query_gid(dev->verbs, port->port_num, 0, &port->gid); - if (ret) - return -1; - - ret = ibv_query_port(dev->verbs, port->port_num, &attr); - if (ret) - return -1; - - port->lid = attr.lid; - for (port->pkey_cnt = 0; !ret && port->pkey_cnt < 4; port->pkey_cnt++) { - ret = ibv_query_pkey(dev->verbs, port->port_num, - port->pkey_cnt, &port->pkey[port->pkey_cnt]); - } - - return port->pkey_cnt ? 0 : ret; -} - -static int acm_open_devices(void) -{ - struct ibv_device **dev_list; - struct acm_device *dev; - struct ibv_device_attr attr; - int i, p, cnt, ret; - - dev_list = ibv_get_device_list(&cnt); - if (!dev_list) - return -1; - - dev_array = (struct acm_device *) zalloc(sizeof(struct acm_device) * cnt); - if (!dev_array) - goto err1; - - for (i = 0; dev_list[i];) { - dev = &dev_array[i]; - - dev->guid = ibv_get_device_guid(dev_list[i]); - dev->verbs = ibv_open_device(dev_list[i]); - if (dev->verbs == NULL) - goto err2; - - ++i; - ret = ibv_query_device(dev->verbs, &attr); - if (ret) - goto err2; - - dev->port_cnt = attr.phys_port_cnt; - dev->ports = zalloc(sizeof(struct acm_port) * dev->port_cnt); - if (!dev->ports) - goto err2; - - for (p = 0; p < dev->port_cnt; p++) { - ret = acm_init_port(dev, p); - if (ret) - goto err2; - } - } - - ibv_free_device_list(dev_list); - dev_cnt = cnt; - return 0; - -err2: - while (i--) { - ibv_close_device(dev_array[i].verbs); - if (dev_array[i].ports) - free(dev_array[i].ports); - } - free(dev_array); -err1: - ibv_free_device_list(dev_list); - return -1; -} - static int acm_init(void) { struct sockaddr_in addr; @@ -162,10 +77,6 @@ static int acm_init(void) if (ret) goto err2; - ret = acm_open_devices(); - if (ret) - goto err2; - ready = 1; return 0; @@ -185,11 +96,51 @@ void LIB_DESTRUCTOR acm_cleanup(void) } } +static int acm_format_resp(struct acm_resolve_msg *msg, + struct ib_acm_path_data **paths, int *count, struct ib_acm_cm_data *data) +{ + struct ib_acm_path_data *path_data; + int i, addr_cnt; + + *count = 0; + addr_cnt = (msg->hdr.length - ACM_MSG_HDR_LENGTH) / + sizeof(struct acm_ep_addr_data); + path_data = (struct ib_acm_path_data *) + zalloc(addr_cnt * sizeof(struct ib_acm_path_data)); + if (!path_data) + return -1; + + memset(data, 0, sizeof *data); + for (i = 0; i < addr_cnt; i++) { + switch (msg->data[i].type) { + case ACM_EP_INFO_PATH: + path_data[i].flags = msg->data[i].flags; + path_data[i].path = msg->data[i].info.path; + (*count)++; + break; + case ACM_EP_INFO_CM: + data->init_depth = msg->data[i].info.cm.init_depth; + data->resp_resources = msg->data[i].info.cm.resp_resources; + break; + default: + goto err; + } + } + + *paths = path_data; + return 0; +err: + free(path_data); + return -1; +} + static int acm_resolve(uint8_t *src, uint8_t *dest, uint8_t type, - struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, - struct ib_acm_resolve_data *data) + struct ib_acm_path_data **paths, int *count, + struct ib_acm_cm_data *data) { - struct acm_resolve_msg msg; + struct acm_msg msg; + struct acm_resolve_msg *resolve_msg = (struct acm_resolve_msg *) &msg; + struct acm_ep_addr_data *src_data, *dest_data; int ret; lock_acquire(&lock); @@ -199,44 +150,48 @@ static int acm_resolve(uint8_t *src, uint8_t *dest, uint8_t type, memset(&msg, 0, sizeof msg); msg.hdr.version = ACM_VERSION; msg.hdr.opcode = ACM_OP_RESOLVE; - msg.hdr.dest_type = type; - msg.hdr.src_type = type; + msg.hdr.length = ACM_MSG_HDR_LENGTH + (2 * ACM_MSG_EP_LENGTH); + + src_data = &resolve_msg->data[0]; + dest_data = &resolve_msg->data[1]; + + src_data->type = type; + src_data->flags = IB_ACM_FLAGS_INBOUND; + dest_data->type = type; + dest_data->flags = IB_ACM_FLAGS_OUTBOUND; switch (type) { - case ACM_EP_TYPE_NAME: - strncpy((char *) msg.src.name, (char *) src, ACM_MAX_ADDRESS); - strncpy((char *) msg.dest.name, (char *) dest, ACM_MAX_ADDRESS); - break; - case ACM_EP_TYPE_ADDRESS_IP: - memcpy(msg.src.addr, &((struct sockaddr_in *) src)->sin_addr, 4); - memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 4); + case ACM_EP_INFO_NAME: + strncpy((char *) src_data->info.name, (char *) src, ACM_MAX_ADDRESS); + strncpy((char *) dest_data->info.name, (char *) dest, ACM_MAX_ADDRESS); break; - case ACM_EP_TYPE_ADDRESS_IP6: - memcpy(msg.src.addr, &((struct sockaddr_in6 *) src)->sin6_addr, 16); - memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 16); + case ACM_EP_INFO_ADDRESS_IP: + memcpy(src_data->info.addr, &((struct sockaddr_in *) src)->sin_addr, 4); + memcpy(dest_data->info.addr, &((struct sockaddr_in *) dest)->sin_addr, 4); break; - case ACM_EP_TYPE_AV: - memcpy(&msg.src.av, src, sizeof(msg.src.av)); - memcpy(&msg.dest.av, dest, sizeof(msg.dest.av)); + case ACM_EP_INFO_ADDRESS_IP6: + memcpy(src_data->info.addr, &((struct sockaddr_in6 *) src)->sin6_addr, 16); + memcpy(dest_data->info.addr, &((struct sockaddr_in6 *) dest)->sin6_addr, 16); break; default: ret = -1; goto out; } - ret = send(sock, (char *) &msg, sizeof msg, 0); - if (ret != sizeof msg) + ret = send(sock, (char *) &msg, msg.hdr.length, 0); + if (ret != msg.hdr.length) goto out; ret = recv(sock, (char *) &msg, sizeof msg, 0); - if (ret != sizeof msg) + if (ret < ACM_MSG_HDR_LENGTH || ret != msg.hdr.length) goto out; - memcpy(dev_addr, &msg.src.dev, sizeof(*dev_addr)); - *ah = msg.dest.av; - memcpy(data, &msg.data, sizeof(*data)); - ret = 0; + if (msg.hdr.status) { + ret = msg.hdr.status; + goto out; + } + ret = acm_format_resp(resolve_msg, paths, count, data); out: lock_release(&lock); return ret; @@ -244,30 +199,32 @@ out: LIB_EXPORT int ib_acm_resolve_name(char *src, char *dest, - struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, - struct ib_acm_resolve_data *data) + struct ib_acm_path_data **paths, int *count, + struct ib_acm_cm_data *data) { return acm_resolve((uint8_t *) src, (uint8_t *) dest, - ACM_EP_TYPE_NAME, dev_addr, ah, data); + ACM_EP_INFO_NAME, paths, count, data); } LIB_EXPORT int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest, - struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah, - struct ib_acm_resolve_data *data) + struct ib_acm_path_data **paths, int *count, + struct ib_acm_cm_data *data) { if (((struct sockaddr *) dest)->sa_family == AF_INET) { return acm_resolve((uint8_t *) src, (uint8_t *) dest, - ACM_EP_TYPE_ADDRESS_IP, dev_addr, ah, data); + ACM_EP_INFO_ADDRESS_IP, paths, count, data); } else { return acm_resolve((uint8_t *) src, (uint8_t *) dest, - ACM_EP_TYPE_ADDRESS_IP6, dev_addr, ah, data); + ACM_EP_INFO_ADDRESS_IP6, paths, count, data); } } -static int acm_query_path(struct ib_path_record *path, uint8_t query_sa) +LIB_EXPORT +int ib_acm_resolve_path(struct ib_path_record *path, uint32_t flags) { - struct acm_query_msg msg; + struct acm_msg msg; + struct acm_ep_addr_data *data; int ret; lock_acquire(&lock); @@ -276,100 +233,27 @@ static int acm_query_path(struct ib_path_record *path, uint8_t query_sa) memset(&msg, 0, sizeof msg); msg.hdr.version = ACM_VERSION; - msg.hdr.opcode = ACM_OP_QUERY; - msg.hdr.param = ACM_QUERY_PATH_RECORD | query_sa; - - if (path->dgid.global.interface_id || path->dgid.global.subnet_prefix) { - msg.hdr.dest_type = ACM_EP_TYPE_GID; - } else if (path->dlid) { - msg.hdr.dest_type = ACM_EP_TYPE_LID; - } else { - ret = -1; - goto out; - } - - if (path->sgid.global.interface_id || path->sgid.global.subnet_prefix) { - msg.hdr.src_type = ACM_EP_TYPE_GID; - } else if (path->slid) { - msg.hdr.src_type = ACM_EP_TYPE_LID; - } else { - ret = -1; - goto out; - } + msg.hdr.opcode = ACM_OP_RESOLVE; + msg.hdr.length = ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH; - msg.data.path = *path; + data = &((struct acm_resolve_msg *) &msg)->data[0]; + data->flags = flags; + data->type = ACM_EP_INFO_PATH; + data->info.path = *path; - ret = send(sock, (char *) &msg, sizeof msg, 0); - if (ret != sizeof msg) + ret = send(sock, (char *) &msg, msg.hdr.length, 0); + if (ret != msg.hdr.length) goto out; ret = recv(sock, (char *) &msg, sizeof msg, 0); - if (ret != sizeof msg) + if (ret < ACM_MSG_HDR_LENGTH || ret != msg.hdr.length) goto out; - *path = msg.data.path; ret = msg.hdr.status; + if (!ret) + *path = data->info.path; out: lock_release(&lock); return ret; } - -LIB_EXPORT -int ib_acm_query_path(struct ib_path_record *path) -{ - return acm_query_path(path, ACM_QUERY_SA); -} - -LIB_EXPORT -int ib_acm_resolve_path(struct ib_path_record *path) -{ - return acm_query_path(path, 0); -} - -static struct acm_device *acm_get_device(uint64_t guid) -{ - int i; - - for (i = 0; i < dev_cnt; i++) { - if (dev_array[i].guid == guid) - return &dev_array[i]; - } - - return NULL; -} - -LIB_EXPORT -int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr, - struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data, - struct ib_path_record *path) -{ - struct acm_device *dev; - int p = dev_addr->port_num - 1; - - dev = acm_get_device(dev_addr->guid); - if (!dev) - return -1; - - if (ah->grh.sgid_index || dev_addr->pkey_index > 4) - return -1; - - if (ah->is_global) { - path->dgid = ah->grh.dgid; - path->sgid = dev->ports[p].gid; - path->flowlabel_hoplimit = - htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit); - path->tclass = ah->grh.traffic_class; - } - - path->dlid = htons(ah->dlid); - path->slid = htons(dev->ports[p].lid | ah->src_path_bits); - path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1; - path->pkey = htons(dev->ports[p].pkey[dev_addr->pkey_index]); - path->qosclass_sl = htons((uint16_t) ah->sl); - path->mtu = (2 << 6) | data->mtu; - path->rate = (2 << 6) | ah->static_rate; - path->packetlifetime = (2 << 6) | data->packet_lifetime; - - return 0; -} diff --git a/windows/library/libacm_exports.src b/windows/library/libacm_exports.src index 0b6b1df..edf3934 100644 --- a/windows/library/libacm_exports.src +++ b/windows/library/libacm_exports.src @@ -8,4 +8,5 @@ LIBRARY libibacm.dll EXPORTS ib_acm_resolve_name ib_acm_resolve_ip +ib_acm_resolve_path #endif