[PATCH 03/12] libibverbs: support provider completion channels and Slingshot transport
From: Doug Ledford
Date: Sat Oct 31 2026 - 15:27:17 EST
Two independent additions the cxiverbs provider needs from libibverbs.
They are kept together because both are small and neither is useful
without the provider.
Slingshot transport: add IBV_NODE_SLINGSHOT, IBV_TRANSPORT_SLINGSHOT
and the IBV_GID_TYPE_SLINGSHOT_UFH GID type, together with the sysfs
name "SLINGSHOT UFH" that the kernel reports for it. The kernel
header mirrors that define the matching RDMA_NODE_SLINGSHOT and
IB_UVERBS_GID_TYPE_SLINGSHOT_UFH values (rdma_netlink.h and
ib_user_ioctl_verbs.h) are updated here, byte for byte from the kernel.
Provider completion channels: add create_comp_channel and
destroy_comp_channel to the provider ops. The default implementation
returns EOPNOTSUPP and libibverbs then falls back to the kernel
uverbs implementation, so existing providers are unaffected. The
CXI hardware delivers completions through its own event queues and
needs to supply the channel itself. As with read_counters_ex, the new
ops join the private ABI that the unreleased UET series already bumped
to 66, so IBVERBS_PABI_VERSION is not bumped again.
The sysfs GID type attribute is read into a 16 byte buffer; the
previous 11 bytes truncated "SLINGSHOT UFH" and made the type
unrecognisable.
Co-developed-by: Iraimani P <iraimani.pavadai@xxxxxxx>
Signed-off-by: Iraimani P <iraimani.pavadai@xxxxxxx>
Signed-off-by: Doug Ledford <doug.ledford@xxxxxxx>
---
kernel-headers/rdma/ib_user_ioctl_verbs.h | 2 ++
kernel-headers/rdma/rdma_netlink.h | 1 +
libibverbs/cmd_device.c | 7 ++++++-
libibverbs/driver.h | 3 +++
libibverbs/dummy_ops.c | 15 +++++++++++++++
libibverbs/enum_strs.c | 3 ++-
libibverbs/init.c | 5 +++++
libibverbs/man/ibv_query_gid_ex.3.md | 3 ++-
libibverbs/verbs.c | 13 +++++++++++++
libibverbs/verbs.h | 5 +++++
10 files changed, 54 insertions(+), 3 deletions(-)
diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h
index 75cdfdbdb2f7..e8e84aa034ef 100644
--- a/kernel-headers/rdma/ib_user_ioctl_verbs.h
+++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h
@@ -258,6 +258,7 @@ enum rdma_driver_id {
RDMA_DRIVER_ERDMA,
RDMA_DRIVER_MANA,
RDMA_DRIVER_IONIC,
+ RDMA_DRIVER_CXI,
};
enum ib_uverbs_gid_type {
@@ -266,6 +267,7 @@ enum ib_uverbs_gid_type {
IB_UVERBS_GID_TYPE_ROCE_V2,
IB_UVERBS_GID_TYPE_UET_UDP,
IB_UVERBS_GID_TYPE_UET_IP,
+ IB_UVERBS_GID_TYPE_SLINGSHOT_UFH,
};
struct ib_uverbs_gid_entry {
diff --git a/kernel-headers/rdma/rdma_netlink.h b/kernel-headers/rdma/rdma_netlink.h
index ee11c3bbbae2..69ff1dee2db6 100644
--- a/kernel-headers/rdma/rdma_netlink.h
+++ b/kernel-headers/rdma/rdma_netlink.h
@@ -158,6 +158,7 @@ enum {
RDMA_NODE_USNIC,
RDMA_NODE_USNIC_UDP,
RDMA_NODE_UNSPECIFIED,
+ RDMA_NODE_SLINGSHOT,
};
/*
diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c
index da7783565d79..ecc58fff66ba 100644
--- a/libibverbs/cmd_device.c
+++ b/libibverbs/cmd_device.c
@@ -283,11 +283,12 @@ static int query_sysfs_gid(struct ibv_context *context, uint8_t port_num, int in
#define V2_TYPE "RoCE v2"
#define UET_UDP_TYPE "UET UDP"
#define UET_IP_TYPE "UET IP"
+#define SLINGSHOT_UFH_TYPE "SLINGSHOT UFH"
static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num,
unsigned int index, enum ibv_gid_type_sysfs *type)
{
struct verbs_device *verbs_device = verbs_get_device(context->device);
- char buff[11];
+ char buff[16];
/* Reset errno so that we can rely on its value upon any error flow in
* ibv_read_sysfs_file.
@@ -335,6 +336,8 @@ static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num,
*type = IBV_GID_TYPE_SYSFS_UET_UDP;
} else if (!strcmp(buff, UET_IP_TYPE)) {
*type = IBV_GID_TYPE_SYSFS_UET_IP;
+ } else if (!strcmp(buff, SLINGSHOT_UFH_TYPE)) {
+ *type = IBV_GID_TYPE_SYSFS_SLINGSHOT_UFH;
} else {
errno = ENOTSUP;
return -1;
@@ -389,6 +392,8 @@ static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num,
entry->gid_type = IBV_GID_TYPE_UET_UDP;
} else if (gid_type == IBV_GID_TYPE_SYSFS_UET_IP) {
entry->gid_type = IBV_GID_TYPE_UET_IP;
+ } else if (gid_type == IBV_GID_TYPE_SYSFS_SLINGSHOT_UFH) {
+ entry->gid_type = IBV_GID_TYPE_SLINGSHOT_UFH;
} else {
entry->gid_type = IBV_GID_TYPE_ROCE_V2;
}
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index 8327550231f4..b6f75faf7997 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -173,6 +173,7 @@ enum ibv_gid_type_sysfs {
IBV_GID_TYPE_SYSFS_ROCE_V2,
IBV_GID_TYPE_SYSFS_UET_UDP,
IBV_GID_TYPE_SYSFS_UET_IP,
+ IBV_GID_TYPE_SYSFS_SLINGSHOT_UFH,
};
enum verbs_query_gid_attr_mask {
@@ -409,6 +410,7 @@ struct verbs_context_ops {
struct ibv_cq_ex *(*create_cq_ex)(
struct ibv_context *context,
struct ibv_cq_init_attr_ex *init_attr);
+ struct ibv_comp_channel *(*create_comp_channel)(struct ibv_context *context);
struct ibv_flow *(*create_flow)(struct ibv_qp *qp,
struct ibv_flow_attr *flow_attr);
struct ibv_flow_action *(*create_flow_action_esp)(struct ibv_context *context,
@@ -437,6 +439,7 @@ struct verbs_context_ops {
int (*dealloc_td)(struct ibv_td *td);
int (*dereg_mr)(struct verbs_mr *vmr);
int (*destroy_ah)(struct ibv_ah *ah);
+ int (*destroy_comp_channel)(struct ibv_comp_channel *channel);
int (*destroy_comp_cntr)(struct ibv_comp_cntr *comp_cntr);
int (*destroy_counters)(struct ibv_counters *counters);
int (*destroy_cq)(struct ibv_cq *cq);
diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
index fee27c1f6687..278a5e768348 100644
--- a/libibverbs/dummy_ops.c
+++ b/libibverbs/dummy_ops.c
@@ -210,6 +210,12 @@ static struct ibv_cq_ex *create_cq_ex(struct ibv_context *context,
return NULL;
}
+static struct ibv_comp_channel *create_comp_channel(struct ibv_context *context)
+{
+ errno = EOPNOTSUPP;
+ return NULL;
+}
+
static struct ibv_flow *create_flow(struct ibv_qp *qp,
struct ibv_flow_attr *flow_attr)
{
@@ -310,6 +316,11 @@ static int destroy_ah(struct ibv_ah *ah)
return EOPNOTSUPP;
}
+static int destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+ return EOPNOTSUPP;
+}
+
static int destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr)
{
return EOPNOTSUPP;
@@ -784,6 +795,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
create_counters,
create_cq,
create_cq_ex,
+ create_comp_channel,
create_flow,
create_flow_action_esp,
create_jkey,
@@ -800,6 +812,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
dealloc_td,
dereg_mr,
destroy_ah,
+ destroy_comp_channel,
destroy_comp_cntr,
destroy_counters,
destroy_cq,
@@ -939,6 +952,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_PRIV_OP(ctx, cq_event);
SET_PRIV_OP(ctx, create_ah);
SET_OP(vctx, create_ah_ex);
+ SET_OP(vctx, create_comp_channel);
SET_PRIV_OP(ctx, create_cq);
SET_PRIV_OP_IC(vctx, create_cq_ex);
SET_PRIV_OP_IC(vctx, create_comp_cntr);
@@ -959,6 +973,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(vctx, destroy_counters);
SET_PRIV_OP(ctx, dereg_mr);
SET_PRIV_OP(ctx, destroy_ah);
+ SET_OP(vctx, destroy_comp_channel);
SET_PRIV_OP_IC(vctx, destroy_comp_cntr);
SET_PRIV_OP(ctx, destroy_cq);
SET_OP2(vctx, ibv_destroy_flow, destroy_flow);
diff --git a/libibverbs/enum_strs.c b/libibverbs/enum_strs.c
index 54ce7362ab09..735048633366 100644
--- a/libibverbs/enum_strs.c
+++ b/libibverbs/enum_strs.c
@@ -42,9 +42,10 @@ const char *ibv_node_type_str(enum ibv_node_type node_type)
[IBV_NODE_USNIC] = "usNIC",
[IBV_NODE_USNIC_UDP] = "usNIC UDP",
[IBV_NODE_UNSPECIFIED] = "unspecified",
+ [IBV_NODE_SLINGSHOT] = "Slingshot",
};
- if (node_type < IBV_NODE_CA || node_type > IBV_NODE_UNSPECIFIED)
+ if (node_type < IBV_NODE_CA || node_type > IBV_NODE_SLINGSHOT)
return "unknown";
return node_type_str[node_type];
diff --git a/libibverbs/init.c b/libibverbs/init.c
index 5f3d11e2d2d7..99bff6561406 100644
--- a/libibverbs/init.c
+++ b/libibverbs/init.c
@@ -116,6 +116,8 @@ enum ibv_node_type decode_knode_type(unsigned int knode_type)
return IBV_NODE_USNIC_UDP;
case RDMA_NODE_UNSPECIFIED:
return IBV_NODE_UNSPECIFIED;
+ case RDMA_NODE_SLINGSHOT:
+ return IBV_NODE_SLINGSHOT;
}
return IBV_NODE_UNKNOWN;
}
@@ -415,6 +417,9 @@ static struct verbs_device *try_driver(const struct verbs_device_ops *ops,
case IBV_NODE_UNSPECIFIED:
dev->transport_type = IBV_TRANSPORT_UNSPECIFIED;
break;
+ case IBV_NODE_SLINGSHOT:
+ dev->transport_type = IBV_TRANSPORT_SLINGSHOT;
+ break;
default:
dev->transport_type = IBV_TRANSPORT_UNKNOWN;
break;
diff --git a/libibverbs/man/ibv_query_gid_ex.3.md b/libibverbs/man/ibv_query_gid_ex.3.md
index 261249bc4c92..4c7a5e6f76a5 100644
--- a/libibverbs/man/ibv_query_gid_ex.3.md
+++ b/libibverbs/man/ibv_query_gid_ex.3.md
@@ -63,7 +63,8 @@ struct ibv_gid_entry {
*gid_type*
: enum ibv_gid_type, can be one of IBV_GID_TYPE_IB, IBV_GID_TYPE_ROCE_V1,
- IBV_GID_TYPE_ROCE_V2, IBV_GID_TYPE_UET_UDP or IBV_GID_TYPE_UET_IP.
+ IBV_GID_TYPE_ROCE_V2, IBV_GID_TYPE_UET_UDP, IBV_GID_TYPE_UET_IP,
+ or IBV_GID_TYPE_SLINGSHOT_UFH.
*ndev_ifindex*
: The interface index of the net device associated with this GID.
diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
index b2f9f4e3f8c0..b79e35ce9a0e 100644
--- a/libibverbs/verbs.c
+++ b/libibverbs/verbs.c
@@ -873,6 +873,11 @@ struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
struct ib_uverbs_create_comp_channel_resp resp = {};
struct ibv_comp_channel *channel;
+ /* if the provider doesn't support this, fall back to kernel */
+ channel = get_ops(context)->create_comp_channel(context);
+ if (channel || errno != EOPNOTSUPP)
+ return channel;
+
channel = malloc(sizeof *channel);
if (!channel)
return NULL;
@@ -897,6 +902,12 @@ int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
int ret;
context = channel->context;
+
+ /* if the provider doesn't support this, fall back to kernel */
+ ret = get_ops(context)->destroy_comp_channel(channel);
+ if (ret != EOPNOTSUPP)
+ return ret;
+
pthread_mutex_lock(&context->mutex);
if (channel->refcnt) {
@@ -1236,6 +1247,8 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num,
*type = IBV_GID_TYPE_SYSFS_UET_UDP;
else if (entry.gid_type == IBV_GID_TYPE_UET_IP)
*type = IBV_GID_TYPE_SYSFS_UET_IP;
+ else if (entry.gid_type == IBV_GID_TYPE_SLINGSHOT_UFH)
+ *type = IBV_GID_TYPE_SYSFS_SLINGSHOT_UFH;
else
*type = IBV_GID_TYPE_SYSFS_ROCE_V2;
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 1bb680b7be39..560643b77e35 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -76,6 +76,7 @@ enum ibv_gid_type {
IBV_GID_TYPE_ROCE_V2,
IBV_GID_TYPE_UET_UDP,
IBV_GID_TYPE_UET_IP,
+ IBV_GID_TYPE_SLINGSHOT_UFH,
};
struct ibv_gid_entry {
@@ -103,6 +104,7 @@ enum ibv_node_type {
IBV_NODE_USNIC,
IBV_NODE_USNIC_UDP,
IBV_NODE_UNSPECIFIED,
+ IBV_NODE_SLINGSHOT,
};
enum ibv_transport_type {
@@ -112,6 +114,7 @@ enum ibv_transport_type {
IBV_TRANSPORT_USNIC,
IBV_TRANSPORT_USNIC_UDP,
IBV_TRANSPORT_UNSPECIFIED,
+ IBV_TRANSPORT_SLINGSHOT,
};
enum ibv_qp_msg_order {
@@ -2546,6 +2549,8 @@ struct verbs_context {
int (*remove_addr)(struct ibv_job *job, unsigned int addr_idx);
int (*query_addr)(struct ibv_job *job, unsigned int addr_idx,
struct ibv_ah_attr_ex *ah_attr);
+ struct ibv_comp_channel *(*create_comp_channel)(struct ibv_context *context);
+ int (*destroy_comp_channel)(struct ibv_comp_channel *channel);
void (*free_buf)(struct ibv_buf *buf);
void *(*alloc_buf)(struct ibv_pd *pd, size_t size,
struct ibv_buf **buf);
--
2.56.0