Skip to content

Commit 698a8dd

Browse files
mark-blochgregkh
mark-bloch
authored and
gregkh
committed
IB/cm: Mark stale CM id's whenever the mad agent was unregistered
commit 9db0ff53cb9b43ed75bacd42a89c1a0ab048b2b0 upstream. When there is a CM id object that has port assigned to it, it means that the cm-id asked for the specific port that it should go by it, but if that port was removed (hot-unplug event) the cm-id was not updated. In order to fix that the port keeps a list of all the cm-id's that are planning to go by it, whenever the port is removed it marks all of them as invalid. This commit fixes a kernel panic which happens when running traffic between guests and we force reboot a guest mid traffic, it triggers a kernel panic: Call Trace: [<ffffffff815271fa>] ? panic+0xa7/0x16f [<ffffffff8152b534>] ? oops_end+0xe4/0x100 [<ffffffff8104a00b>] ? no_context+0xfb/0x260 [<ffffffff81084db2>] ? del_timer_sync+0x22/0x30 [<ffffffff8104a295>] ? __bad_area_nosemaphore+0x125/0x1e0 [<ffffffff81084240>] ? process_timeout+0x0/0x10 [<ffffffff8104a363>] ? bad_area_nosemaphore+0x13/0x20 [<ffffffff8104aabf>] ? __do_page_fault+0x31f/0x480 [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffffa0752675>] ? free_msg+0x55/0x70 [mlx5_core] [<ffffffffa0753434>] ? cmd_exec+0x124/0x840 [mlx5_core] [<ffffffff8105a924>] ? find_busiest_group+0x244/0x9f0 [<ffffffff8152d45e>] ? do_page_fault+0x3e/0xa0 [<ffffffff8152a815>] ? page_fault+0x25/0x30 [<ffffffffa024da25>] ? cm_alloc_msg+0x35/0xc0 [ib_cm] [<ffffffffa024e821>] ? ib_send_cm_dreq+0xb1/0x1e0 [ib_cm] [<ffffffffa024f836>] ? cm_destroy_id+0x176/0x320 [ib_cm] [<ffffffffa024fb00>] ? ib_destroy_cm_id+0x10/0x20 [ib_cm] [<ffffffffa034f527>] ? ipoib_cm_free_rx_reap_list+0xa7/0x110 [ib_ipoib] [<ffffffffa034f590>] ? ipoib_cm_rx_reap+0x0/0x20 [ib_ipoib] [<ffffffffa034f5a5>] ? ipoib_cm_rx_reap+0x15/0x20 [ib_ipoib] [<ffffffff81094d20>] ? worker_thread+0x170/0x2a0 [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff81094bb0>] ? worker_thread+0x0/0x2a0 [<ffffffff8109aef6>] ? kthread+0x96/0xa0 [<ffffffff8100c20a>] ? child_rip+0xa/0x20 [<ffffffff8109ae60>] ? kthread+0x0/0xa0 [<ffffffff8100c200>] ? child_rip+0x0/0x20 Fixes: a977049 ("[PATCH] IB: Add the kernel CM implementation") Signed-off-by: Mark Bloch <markb@mellanox.com> Signed-off-by: Erez Shitrit <erezsh@mellanox.com> Reviewed-by: Maor Gottlieb <maorg@mellanox.com> Signed-off-by: Leon Romanovsky <leon@kernel.org> Signed-off-by: Doug Ledford <dledford@redhat.com> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
1 parent 2b026a2 commit 698a8dd

1 file changed

Lines changed: 110 additions & 16 deletions

File tree

  • drivers/infiniband/core

‎drivers/infiniband/core/cm.c‎

Lines changed: 110 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ static struct ib_cm {
8080
__be32 random_id_operand;
8181
struct list_head timewait_list;
8282
struct workqueue_struct *wq;
83+
/* Sync on cm change port state */
84+
spinlock_t state_lock;
8385
} cm;
8486

8587
/* Counter indexes ordered by attribute ID */
@@ -161,6 +163,8 @@ struct cm_port {
161163
struct ib_mad_agent *mad_agent;
162164
struct kobject port_obj;
163165
u8 port_num;
166+
struct list_head cm_priv_prim_list;
167+
struct list_head cm_priv_altr_list;
164168
struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
165169
};
166170

@@ -241,6 +245,12 @@ struct cm_id_private {
241245
u8 service_timeout;
242246
u8 target_ack_delay;
243247

248+
struct list_head prim_list;
249+
struct list_head altr_list;
250+
/* Indicates that the send port mad is registered and av is set */
251+
int prim_send_port_not_ready;
252+
int altr_send_port_not_ready;
253+
244254
struct list_head work_list;
245255
atomic_t work_count;
246256
};
@@ -259,20 +269,47 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
259269
struct ib_mad_agent *mad_agent;
260270
struct ib_mad_send_buf *m;
261271
struct ib_ah *ah;
272+
struct cm_av *av;
273+
unsigned long flags, flags2;
274+
int ret = 0;
262275

276+
/* don't let the port to be released till the agent is down */
277+
spin_lock_irqsave(&cm.state_lock, flags2);
278+
spin_lock_irqsave(&cm.lock, flags);
279+
if (!cm_id_priv->prim_send_port_not_ready)
280+
av = &cm_id_priv->av;
281+
else if (!cm_id_priv->altr_send_port_not_ready &&
282+
(cm_id_priv->alt_av.port))
283+
av = &cm_id_priv->alt_av;
284+
else {
285+
pr_info("%s: not valid CM id\n", __func__);
286+
ret = -ENODEV;
287+
spin_unlock_irqrestore(&cm.lock, flags);
288+
goto out;
289+
}
290+
spin_unlock_irqrestore(&cm.lock, flags);
291+
/* Make sure the port haven't released the mad yet */
263292
mad_agent = cm_id_priv->av.port->mad_agent;
264-
ah = ib_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr);
265-
if (IS_ERR(ah))
266-
return PTR_ERR(ah);
293+
if (!mad_agent) {
294+
pr_info("%s: not a valid MAD agent\n", __func__);
295+
ret = -ENODEV;
296+
goto out;
297+
}
298+
ah = ib_create_ah(mad_agent->qp->pd, &av->ah_attr);
299+
if (IS_ERR(ah)) {
300+
ret = PTR_ERR(ah);
301+
goto out;
302+
}
267303

268304
m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
269-
cm_id_priv->av.pkey_index,
305+
av->pkey_index,
270306
0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
271307
GFP_ATOMIC,
272308
IB_MGMT_BASE_VERSION);
273309
if (IS_ERR(m)) {
274310
ib_destroy_ah(ah);
275-
return PTR_ERR(m);
311+
ret = PTR_ERR(m);
312+
goto out;
276313
}
277314

278315
/* Timeout set by caller if response is expected. */
@@ -282,7 +319,10 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
282319
atomic_inc(&cm_id_priv->refcount);
283320
m->context[0] = cm_id_priv;
284321
*msg = m;
285-
return 0;
322+
323+
out:
324+
spin_unlock_irqrestore(&cm.state_lock, flags2);
325+
return ret;
286326
}
287327

288328
static int cm_alloc_response_msg(struct cm_port *port,
@@ -352,7 +392,8 @@ static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
352392
grh, &av->ah_attr);
353393
}
354394

355-
static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
395+
static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av,
396+
struct cm_id_private *cm_id_priv)
356397
{
357398
struct cm_device *cm_dev;
358399
struct cm_port *port = NULL;
@@ -387,7 +428,17 @@ static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
387428
&av->ah_attr);
388429
av->timeout = path->packet_life_time + 1;
389430

390-
return 0;
431+
spin_lock_irqsave(&cm.lock, flags);
432+
if (&cm_id_priv->av == av)
433+
list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
434+
else if (&cm_id_priv->alt_av == av)
435+
list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
436+
else
437+
ret = -EINVAL;
438+
439+
spin_unlock_irqrestore(&cm.lock, flags);
440+
441+
return ret;
391442
}
392443

393444
static int cm_alloc_id(struct cm_id_private *cm_id_priv)
@@ -677,6 +728,8 @@ struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
677728
spin_lock_init(&cm_id_priv->lock);
678729
init_completion(&cm_id_priv->comp);
679730
INIT_LIST_HEAD(&cm_id_priv->work_list);
731+
INIT_LIST_HEAD(&cm_id_priv->prim_list);
732+
INIT_LIST_HEAD(&cm_id_priv->altr_list);
680733
atomic_set(&cm_id_priv->work_count, -1);
681734
atomic_set(&cm_id_priv->refcount, 1);
682735
return &cm_id_priv->id;
@@ -892,6 +945,15 @@ static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
892945
break;
893946
}
894947

948+
spin_lock_irq(&cm.lock);
949+
if (!list_empty(&cm_id_priv->altr_list) &&
950+
(!cm_id_priv->altr_send_port_not_ready))
951+
list_del(&cm_id_priv->altr_list);
952+
if (!list_empty(&cm_id_priv->prim_list) &&
953+
(!cm_id_priv->prim_send_port_not_ready))
954+
list_del(&cm_id_priv->prim_list);
955+
spin_unlock_irq(&cm.lock);
956+
895957
cm_free_id(cm_id->local_id);
896958
cm_deref_id(cm_id_priv);
897959
wait_for_completion(&cm_id_priv->comp);
@@ -1192,12 +1254,13 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,
11921254
goto out;
11931255
}
11941256

1195-
ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av);
1257+
ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av,
1258+
cm_id_priv);
11961259
if (ret)
11971260
goto error1;
11981261
if (param->alternate_path) {
11991262
ret = cm_init_av_by_path(param->alternate_path,
1200-
&cm_id_priv->alt_av);
1263+
&cm_id_priv->alt_av, cm_id_priv);
12011264
if (ret)
12021265
goto error1;
12031266
}
@@ -1639,7 +1702,8 @@ static int cm_req_handler(struct cm_work *work)
16391702
cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
16401703

16411704
memcpy(work->path[0].dmac, cm_id_priv->av.ah_attr.dmac, ETH_ALEN);
1642-
ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
1705+
ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av,
1706+
cm_id_priv);
16431707
if (ret) {
16441708
ib_get_cached_gid(work->port->cm_dev->ib_device,
16451709
work->port->port_num, 0, &work->path[0].sgid,
@@ -1650,7 +1714,8 @@ static int cm_req_handler(struct cm_work *work)
16501714
goto rejected;
16511715
}
16521716
if (req_msg->alt_local_lid) {
1653-
ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av);
1717+
ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av,
1718+
cm_id_priv);
16541719
if (ret) {
16551720
ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
16561721
&work->path[0].sgid,
@@ -2705,7 +2770,8 @@ int ib_send_cm_lap(struct ib_cm_id *cm_id,
27052770
goto out;
27062771
}
27072772

2708-
ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av);
2773+
ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av,
2774+
cm_id_priv);
27092775
if (ret)
27102776
goto out;
27112777
cm_id_priv->alt_av.timeout =
@@ -2817,7 +2883,8 @@ static int cm_lap_handler(struct cm_work *work)
28172883
cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
28182884
work->mad_recv_wc->recv_buf.grh,
28192885
&cm_id_priv->av);
2820-
cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av);
2886+
cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av,
2887+
cm_id_priv);
28212888
ret = atomic_inc_and_test(&cm_id_priv->work_count);
28222889
if (!ret)
28232890
list_add_tail(&work->list, &cm_id_priv->work_list);
@@ -3009,7 +3076,7 @@ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
30093076
return -EINVAL;
30103077

30113078
cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3012-
ret = cm_init_av_by_path(param->path, &cm_id_priv->av);
3079+
ret = cm_init_av_by_path(param->path, &cm_id_priv->av, cm_id_priv);
30133080
if (ret)
30143081
goto out;
30153082

@@ -3446,7 +3513,9 @@ static int cm_establish(struct ib_cm_id *cm_id)
34463513
static int cm_migrate(struct ib_cm_id *cm_id)
34473514
{
34483515
struct cm_id_private *cm_id_priv;
3516+
struct cm_av tmp_av;
34493517
unsigned long flags;
3518+
int tmp_send_port_not_ready;
34503519
int ret = 0;
34513520

34523521
cm_id_priv = container_of(cm_id, struct cm_id_private, id);
@@ -3455,7 +3524,14 @@ static int cm_migrate(struct ib_cm_id *cm_id)
34553524
(cm_id->lap_state == IB_CM_LAP_UNINIT ||
34563525
cm_id->lap_state == IB_CM_LAP_IDLE)) {
34573526
cm_id->lap_state = IB_CM_LAP_IDLE;
3527+
/* Swap address vector */
3528+
tmp_av = cm_id_priv->av;
34583529
cm_id_priv->av = cm_id_priv->alt_av;
3530+
cm_id_priv->alt_av = tmp_av;
3531+
/* Swap port send ready state */
3532+
tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3533+
cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3534+
cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
34593535
} else
34603536
ret = -EINVAL;
34613537
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -3875,6 +3951,9 @@ static void cm_add_one(struct ib_device *ib_device)
38753951
port->cm_dev = cm_dev;
38763952
port->port_num = i;
38773953

3954+
INIT_LIST_HEAD(&port->cm_priv_prim_list);
3955+
INIT_LIST_HEAD(&port->cm_priv_altr_list);
3956+
38783957
ret = cm_create_port_fs(port);
38793958
if (ret)
38803959
goto error1;
@@ -3932,6 +4011,8 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
39324011
{
39334012
struct cm_device *cm_dev = client_data;
39344013
struct cm_port *port;
4014+
struct cm_id_private *cm_id_priv;
4015+
struct ib_mad_agent *cur_mad_agent;
39354016
struct ib_port_modify port_modify = {
39364017
.clr_port_cap_mask = IB_PORT_CM_SUP
39374018
};
@@ -3955,15 +4036,27 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
39554036

39564037
port = cm_dev->port[i-1];
39574038
ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4039+
/* Mark all the cm_id's as not valid */
4040+
spin_lock_irq(&cm.lock);
4041+
list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4042+
cm_id_priv->altr_send_port_not_ready = 1;
4043+
list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4044+
cm_id_priv->prim_send_port_not_ready = 1;
4045+
spin_unlock_irq(&cm.lock);
39584046
/*
39594047
* We flush the queue here after the going_down set, this
39604048
* verify that no new works will be queued in the recv handler,
39614049
* after that we can call the unregister_mad_agent
39624050
*/
39634051
flush_workqueue(cm.wq);
3964-
ib_unregister_mad_agent(port->mad_agent);
4052+
spin_lock_irq(&cm.state_lock);
4053+
cur_mad_agent = port->mad_agent;
4054+
port->mad_agent = NULL;
4055+
spin_unlock_irq(&cm.state_lock);
4056+
ib_unregister_mad_agent(cur_mad_agent);
39654057
cm_remove_port_fs(port);
39664058
}
4059+
39674060
device_unregister(cm_dev->device);
39684061
kfree(cm_dev);
39694062
}
@@ -3976,6 +4069,7 @@ static int __init ib_cm_init(void)
39764069
INIT_LIST_HEAD(&cm.device_list);
39774070
rwlock_init(&cm.device_lock);
39784071
spin_lock_init(&cm.lock);
4072+
spin_lock_init(&cm.state_lock);
39794073
cm.listen_service_table = RB_ROOT;
39804074
cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
39814075
cm.remote_id_table = RB_ROOT;

0 commit comments

Comments
 (0)