diff options
| -rw-r--r-- | ctdb/ib/ibwrapper.c | 217 | ||||
| -rw-r--r-- | ctdb/ib/ibwrapper.h | 4 | ||||
| -rw-r--r-- | ctdb/ib/ibwrapper_internal.h | 22 |
3 files changed, 214 insertions, 29 deletions
diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c index 5d2d35449b..9778b4e041 100644 --- a/ctdb/ib/ibwrapper.c +++ b/ctdb/ib/ibwrapper.c @@ -46,22 +46,192 @@ static ibw_mr *ibw_alloc_mr(ibw_ctx_priv *pctx) { } -static int ibwctx_destruct(void *ptr) +static int ibw_ctx_priv_destruct(void *ptr) { - ibw_ctx *pctx = talloc_get_type(ptr, ibw_ctx); + ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv); assert(pctx!=NULL); + if (pctx->cm_id) { + rdma_destroy_id(pctx->cm_id); + pctx->cm_id = NULL; + } + if (pctx->cm_channel) { + rdma_destroy_event_channel(pctx->cm_channel); + pctx->cm_channel = NULL; + } + + /* free memory regions */ +} + +static int ibw_ctx_destruct(void *ptr) +{ + ibw_ctx *ctx = talloc_get_type(ptr, ibw_ctx); + assert(ctx!=NULL); + + if (pconn->cm_id) { + rdma_destroy_id(pconn->cm_id); + pconn->cm_id = NULL; + } + /* free memory regions */ return 0; } -int ibw_process_event(ibw_ctx *ctx, int fd_index); +static int ibw_conn_priv_destruct(void *ptr) +{ + ibw_conn *pconn = talloc_get_type(ptr, ibw_conn_priv); + assert(pconn!=NULL); +} + +static int ibw_conn_destruct(void *ptr) +{ + ibw_conn *conn = talloc_get_type(ptr, ibw_conn); + ibw_ctx *ctx; + + assert(conn!=NULL); + ctx = ibw_conn->ctx; + assert(ctx!=NULL); + + /* unhook conn from ctx's linked list */ + assert(ctx->first_conn!=NULL); + assert(ctx->last_conn!=NULL); + + if (conn->prev==NULL) { + assert(ctx->first_conn==conn); + ctx->first_conn = conn->next; + } else { + conn->prev->next = conn->next; + } + + if (conn->next==NULL) { + assert(ctx->last_conn==conn); + ctx->last_conn = conn->prev; + } else { + conn->next->prev = conn->prev; + } + return 0; +} + +static ibw_conn *ibw_new_conn(ibw_ctx *ctx) +{ + ibw_conn *conn; + ibw_conn_priv *pconn; + + conn = talloc_zero(ctx, ibw_conn); + assert(conn!=NULL); + talloc_set_destructor(conn, ibw_conn_destruct); + + pconn = talloc_zero(ctx, ibw_conn_priv); + assert(pconn!=NULL); + talloc_set_destructor(pconn, ibw_conn_priv_destruct); + + conn->ctx = ctx; + + /* append conn to the end of ctx's linked list */ + conn->prev = ctx->last_conn; + conn->next = NULL; + if (ctx->first_conn) { + assert(ctx->last_conn!=NULL); + conn->prev->next = conn; + } else { + ctx->first_conn = ctx->last_conn = conn; + } + + return conn; +} static void ibw_process_cm_event(struct event_context *ev, struct fd_event *fde, uint16_t flags, void *private_data) { - if (fde-> + int rc; + ibw_ctx *ctx = talloc_get_type(private_data, ibw_ctx); + ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv); + struct rdma_cm_id *id; + struct rdma_cm_event event; + + assert(ctx!=NULL); + + rc = rdma_get_cm_event(cb->cm_channel, &event); + if (rc) { + ctx->state = IBWS_ERROR; + sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc); + DEBUG(0, ibw_lasterr); + return; + } + id = &event.id; + + /* find whose cma_id we have */ + +// DEBUG(10, "cma_event type %d cma_id %p (%s)\n", event->event, cma_id, +// (cma_id == cb->cm_id) ? "parent" : "child"); + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + assert(pctx->state==IWINT_INIT); + pctx->state = IWINT_ADDR_RESOLVED; + ret = rdma_resolve_route(cma_id, 2000); + if (ret) { + cb->state = ERROR; + fprintf(stderr, "rdma_resolve_route error %d\n", ret); + sem_post(&cb->sem); + } + break; + + case RDMA_CM_EVENT_ROUTE_RESOLVED: + assert(pctx->state==IWINT_ADDR_RESOLVED); + pctx->state = IWINT_ROUTE_RESOLVED; + sem_post(&cb->sem); + break; + + case RDMA_CM_EVENT_CONNECT_REQUEST: + cb->state = CONNECT_REQUEST; + cb->child_cm_id = cma_id; + DEBUG_LOG("child cma %p\n", cb->child_cm_id); + sem_post(&cb->sem); + break; + + case RDMA_CM_EVENT_ESTABLISHED: + DEBUG_LOG("ESTABLISHED\n"); + + /* + * Server will wake up when first RECV completes. + */ + if (!cb->server) { + cb->state = CONNECTED; + } + sem_post(&cb->sem); + break; + + case RDMA_CM_EVENT_ADDR_ERROR: + case RDMA_CM_EVENT_ROUTE_ERROR: + case RDMA_CM_EVENT_CONNECT_ERROR: + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_REJECTED: + fprintf(stderr, "cma event %d, error %d\n", event->event, + event->status); + sem_post(&cb->sem); + ret = -1; + break; + + case RDMA_CM_EVENT_DISCONNECTED: + fprintf(stderr, "%s DISCONNECT EVENT...\n", cb->server ? "server" : "client"); + sem_post(&cb->sem); + break; + + case RDMA_CM_EVENT_DEVICE_REMOVAL: + fprintf(stderr, "cma detected device removal!!!!\n"); + ret = -1; + break; + + default: + fprintf(stderr, "oof bad type!\n"); + sem_post(&cb->sem); + ret = -1; + break; + } + + rdma_ack_cm_event(event); } static int ibw_process_init_attrs(ibw_initattr *attr, int nattr, ibw_opts *opts) @@ -96,16 +266,17 @@ ibw_ctx *ibw_init(ibw_initattr *attr, int nattr, ibw_ctx *ctx = talloc_zero(NULL, ibw_ctx); ibw_ctx_priv *pctx; int rc; - ibw_event_ud *event_priv; + /* initialize basic data structures */ memset(ibw_lasterr, 0, IBW_LASTERR_BUFSIZE); assert(ctx!=NULL); ibw_lasterr[0] = '\0'; - talloc_set_destructor(ctx, ibwctx_destruct); + talloc_set_destructor(ctx, ibw_ctx_destruct); ctx->userdata = userdata; pctx = talloc_zero(ctx, ibw_ctx_priv); + talloc_set_destructor(pctx, ibw_ctx_priv_destruct); ctx->internal = (void *)pctx; assert(pctx!=NULL); @@ -126,12 +297,8 @@ ibw_ctx *ibw_init(ibw_initattr *attr, int nattr, goto cleanup; } - event_priv = talloc_zero(ctx, ibw_event_ud); - event_priv->ctx = ctx; - event_priv->id = IBWET_CM; - pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx, - pctx->cm_channel->fd, EVENT_FD_READ, ibw_process_cm_event, event_priv); + pctx->cm_channel->fd, EVENT_FD_READ, ibw_process_cm_event, ctx); rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, cb, RDMA_PS_TCP); if (rc) { @@ -161,40 +328,56 @@ int ibw_stop(ibw_ctx *ctx) int ibw_bind(ibw_ctx *ctx, struct sockaddr_in *my_addr) { ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; + int rc; + + rc = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &my_addr); + if (rc) { + sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc); + return rc; + } + + return 0; } int ibw_listen(ibw_ctx *ctx, int backlog) { ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; + + return 0; } int ibw_accept(ibw_ctx *ctx, void *conn_userdata) { ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; + + return 0; } int ibw_connect(ibw_ctx *ctx, struct sockaddr_in *serv_addr, void *conn_userdata) { ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; + + return 0; } void ibw_disconnect(ibw_conn *conn) { ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; -} - -int ibw_process_event(ibw_ctx *ctx, ...) -{ - ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal; + + return 0; } int ibw_alloc_send_buf(ibw_conn *conn, void **buf, void **key, int n) { + ibw_conn_priv *pconn = (ibw_ctx_priv *)ctx->internal; + + return 0; } int ibw_send(ibw_conn *conn, void *buf, void *key, int n) { - + ibw_conn_priv *pconn = (ibw_ctx_priv *)ctx->internal; + return 0; } const char *ibw_getLastError() diff --git a/ctdb/ib/ibwrapper.h b/ctdb/ib/ibwrapper.h index 8183919020..c5ca0272cc 100644 --- a/ctdb/ib/ibwrapper.h +++ b/ctdb/ib/ibwrapper.h @@ -37,6 +37,8 @@ typedef struct _ibw_ctx { ibw_state_ctx state; void *internal; + + ibw_conn *first_conn, *last_conn; } ibw_ctx; typedef enum { @@ -52,6 +54,8 @@ typedef struct _ibw_conn { void *conn_userdata; /* see ibw_connect and ibw_accept */ void *internal; + + ibw_conn *prev, next; } ibw_conn; /* diff --git a/ctdb/ib/ibwrapper_internal.h b/ctdb/ib/ibwrapper_internal.h index dbf11f6273..df2f14a2c1 100644 --- a/ctdb/ib/ibwrapper_internal.h +++ b/ctdb/ib/ibwrapper_internal.h @@ -33,6 +33,13 @@ typedef struct _ibw_opts { int ib_port; } ibw_opts; +typedef enum { + IWINT_INIT = 0, + IWINT_ADDR_RESOLVED, + IWINT_ROUTE_RESOLVED, + IWINT_ERROR +} ibw_state_ctx; + typedef struct _ibw_ctx_priv { ibw_mr *avail_first; ibw_mr *avail_last; @@ -45,10 +52,10 @@ typedef struct _ibw_ctx_priv { struct ibv_context *context; struct ibv_pd *pd; + struct rdma_cm_id *cm_id; /* server cm id */ struct rdma_event_channel *cm_channel; struct fd_event *cm_channel_event; - struct rdma_cm_id *cm_id; /* connection on client side,*/ ibw_connstate_fn_t connstate_func; ibw_receive_fn_t receive_func; @@ -57,18 +64,9 @@ typedef struct _ibw_ctx_priv { typedef struct _ibw_conn_priv { struct ibv_cq *cq; struct ibv_qp *qp; - struct ib_cm_id *cm_id; -} ibw_conn_priv; -typedef enum { - IBWET_CM, - IBWET_VERBS -} ibw_event_type; - -typedef struct _ibw_event_ud { - ibw_ctx *ctx; - ibw_event_type id; -} ibw_event_ud; + struct rdma_cm_id *cm_id; /* client's cm id */ +} ibw_conn_priv; /* * Must be called in all cases after selecting/polling |