In the Linux kernel, the following vulnerability has been resolved:
IB/ipoib: Fix mcast list locking
Releasing the priv->lock
while iterating the priv->multicast_list
in
ipoib_mcast_join_task()
opens a window for ipoib_mcast_dev_flush()
to
remove the items while in the middle of iteration. If the mcast is removed
while the lock was dropped, the for loop spins forever resulting in a hard
lockup (as was reported on RHEL 4.18.0-372.75.1.el8_6 kernel):
Task A (kworker/u72:2 below) | Task B (kworker/u72:0 below)
-----------------------------------+-----------------------------------
ipoib_mcast_join_task(work) | ipoib_ib_dev_flush_light(work)
spin_lock_irq(&priv->lock) | __ipoib_ib_dev_flush(priv, ...)
list_for_each_entry(mcast, | ipoib_mcast_dev_flush(dev = priv->dev)
&priv->multicast_list, list) |
ipoib_mcast_join(dev, mcast) |
spin_unlock_irq(&priv->lock) |
| spin_lock_irqsave(&priv->lock, flags)
| list_for_each_entry_safe(mcast, tmcast,
| &priv->multicast_list, list)
| list_del(&mcast->list);
| list_add_tail(&mcast->list, &remove_list)
| spin_unlock_irqrestore(&priv->lock, flags)
spin_lock_irq(&priv->lock) |
| ipoib_mcast_remove_list(&remove_list)
(Here, mcast
is no longer on the | listforeachentrysafe(mcast, tmcast,
priv->multicast_list
and we keep | removelist, list)
spinning on the remove_list
of | >>> waitfor_completion(&mcast->done)
the other thread which is blocked |
and the list is still valid on |
it's stack.)
Fix this by keeping the lock held and changing to GFP_ATOMIC to prevent eventual sleeps. Unfortunately we could not reproduce the lockup and confirm this fix but based on the code review I think this fix should address such lockups.
crash> bc 31
[exception RIP: ipoib_mcast_join_task+0x1b1]
RIP: ffffffffc0944ac1 RSP: ff646f199a8c7e00 RFLAGS: 00000002
RAX: 0000000000000000 RBX: ff1c6a1a04dc82f8 RCX: 0000000000000000
work (&priv->mcast_task{,.work})
RDX: ff1c6a192d60ac68 RSI: 0000000000000286 RDI: ff1c6a1a04dc8000
&mcast->list
RBP: ff646f199a8c7e90 R8: ff1c699980019420 R9: ff1c6a1920c9a000
R10: ff646f199a8c7e00 R11: ff1c6a191a7d9800 R12: ff1c6a192d60ac00
mcast
R13: ff1c6a1d82200000 R14: ff1c6a1a04dc8000 R15: ff1c6a1a04dc82d8
dev priv (&priv->lock) &priv->multicast_list (aka head)
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> --- #5 [ff646f199a8c7e00] ipoibmcastjointask+0x1b1 at ffffffffc0944ac1 [ibipoib] #6 [ff646f199a8c7e98] processonework+0x1a7 at ffffffff9bf10967
crash> rx ff646f199a8c7e68 ff646f199a8c7e68: ff1c6a1a04dc82f8 <<< work = &priv->mcast_task.work
crash> list -hO ipoibdevpriv.multicast_list ff1c6a1a04dc8000 (empty)
crash> ipoibdevpriv.mcasttask.work.func,mcastmutex.owner.counter ff1c6a1a04dc8000 mcasttask.work.func = 0xffffffffc0944910 <ipoib_mcast_join_task>, mcastmutex.owner.counter = 0xff1c69998efec000
crash> b 8
#3 [ff646f1980153d50] waitforcompletion+0x96 at ffffffff9c7d7646 #4 [ff646f1980153d90] ipoibmcastremovelist+0x56 at ffffffffc0944dc6 [ibipoib] #5 [ff646f1980153de8] ipoibmcastdevflush+0x1a7 at ffffffffc09455a7 [ibipoib] #6 [ff646f1980153e58] _ipoibibdevflush+0x1a4 at ffffffffc09431a4 [ib_ipoib] #7 [ff ---truncated---
{ "vanir_signatures": [ { "id": "CVE-2023-52587-0766d5b9", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@ed790bd0903ed3352ebf7f650d910f49b7319b34", "digest": { "function_hash": "201415682156274057564628287501734752899", "length": 2056.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-0b3e1aef", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@4c8922ae8eb8dcc1e4b7d1059d97a8334288d825", "digest": { "function_hash": "63425166528938939438483203789336464125", "length": 2097.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-213812a6", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@ac2630fd3c90ffec34a0bfc4d413668538b0e8f2", "digest": { "threshold": 0.9, "line_hashes": [ "152245487965058250712139116512069311635", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-262bbeae", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@5108a2dc2db5630fb6cd58b8be80a0c134bc310a", "digest": { "function_hash": "201415682156274057564628287501734752899", "length": 2056.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-283c1dc4", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@7c7bd4d561e9dc6f5b7df9e184974915f6701a89", "digest": { "function_hash": "201415682156274057564628287501734752899", "length": 2056.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-42391558", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@4c8922ae8eb8dcc1e4b7d1059d97a8334288d825", "digest": { "threshold": 0.9, "line_hashes": [ "152245487965058250712139116512069311635", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-54af8dad", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@7c7bd4d561e9dc6f5b7df9e184974915f6701a89", "digest": { "threshold": 0.9, "line_hashes": [ "129640355334469884991617847364571738229", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-83b58bb5", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@4f973e211b3b1c6d36f7c6a19239d258856749f9", "digest": { "function_hash": "201415682156274057564628287501734752899", "length": 2056.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-865c5b6b", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@4f973e211b3b1c6d36f7c6a19239d258856749f9", "digest": { "threshold": 0.9, "line_hashes": [ "129640355334469884991617847364571738229", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-88b0fd43", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@ed790bd0903ed3352ebf7f650d910f49b7319b34", "digest": { "threshold": 0.9, "line_hashes": [ "129640355334469884991617847364571738229", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-98fcfc66", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@5108a2dc2db5630fb6cd58b8be80a0c134bc310a", "digest": { "threshold": 0.9, "line_hashes": [ "129640355334469884991617847364571738229", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-9ff8882c", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@615e3adc2042b7be4ad122a043fc9135e6342c90", "digest": { "threshold": 0.9, "line_hashes": [ "152245487965058250712139116512069311635", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-a379bcec", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@342258fb46d66c1b4c7e2c3717ac01e10c03cf18", "digest": { "function_hash": "201415682156274057564628287501734752899", "length": 2056.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-b1c76667", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@ac2630fd3c90ffec34a0bfc4d413668538b0e8f2", "digest": { "function_hash": "63425166528938939438483203789336464125", "length": 2097.0 }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-c5675cef", "signature_type": "Line", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@342258fb46d66c1b4c7e2c3717ac01e10c03cf18", "digest": { "threshold": 0.9, "line_hashes": [ "129640355334469884991617847364571738229", "3967532736514535700981025168245212906", "281362113694911265093565366709197333226", "191096593176429705031928028902878309686", "138066768187320531994682368745762816656", "112929453694000079114177580282377785535", "138374544649259718090809530923415132330", "53456583362759567514597115036725623285", "313765005629161941026230144639663136364", "213208382876286027878466640862484658400", "211497567391943488750616859526718132347", "282085290756441645208025702702868220058", "327644427273325590200135615789132816444", "35381946200517719840603536235103075671", "75645886973244586185797421964632811791", "55012117218605863575472713944120483552" ] }, "deprecated": false, "signature_version": "v1" }, { "id": "CVE-2023-52587-e7e3264e", "signature_type": "Function", "target": { "file": "drivers/infiniband/ulp/ipoib/ipoib_multicast.c", "function": "ipoib_mcast_join" }, "source": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git@615e3adc2042b7be4ad122a043fc9135e6342c90", "digest": { "function_hash": "63425166528938939438483203789336464125", "length": 2097.0 }, "deprecated": false, "signature_version": "v1" } ] }