475 lines
16 KiB
Python
475 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import re
|
|
from collections import defaultdict
|
|
from enum import Enum
|
|
|
|
import gdb
|
|
|
|
|
|
class DiGraph:
|
|
"""
|
|
Adapted from networkx: http://networkx.github.io/
|
|
Represents a directed graph. Edges can store (key, value) attributes.
|
|
"""
|
|
|
|
def __init__(self):
|
|
# Map of node -> set of nodes
|
|
self.adjacency_map = {}
|
|
# Map of (node1, node2) -> map string -> arbitrary attribute
|
|
# This will not be copied in subgraph()
|
|
self.attributes_map = {}
|
|
|
|
def neighbors(self, node):
|
|
return self.adjacency_map.get(node, set())
|
|
|
|
def edges(self):
|
|
edges = []
|
|
for node, neighbors in self.adjacency_map.items():
|
|
for neighbor in neighbors:
|
|
edges.append((node, neighbor))
|
|
return edges
|
|
|
|
def nodes(self):
|
|
return self.adjacency_map.keys()
|
|
|
|
def attributes(self, node1, node2):
|
|
return self.attributes_map[(node1, node2)]
|
|
|
|
def add_edge(self, node1, node2, **kwargs):
|
|
if node1 not in self.adjacency_map:
|
|
self.adjacency_map[node1] = set()
|
|
if node2 not in self.adjacency_map:
|
|
self.adjacency_map[node2] = set()
|
|
self.adjacency_map[node1].add(node2)
|
|
self.attributes_map[(node1, node2)] = kwargs
|
|
|
|
def remove_node(self, node):
|
|
self.adjacency_map.pop(node, None)
|
|
for _, neighbors in self.adjacency_map.items():
|
|
neighbors.discard(node)
|
|
|
|
def subgraph(self, nodes):
|
|
graph = DiGraph()
|
|
for node in nodes:
|
|
for neighbor in self.neighbors(node):
|
|
if neighbor in nodes:
|
|
graph.add_edge(node, neighbor)
|
|
return graph
|
|
|
|
def node_link_data(self):
|
|
"""
|
|
Returns the graph as a dictionary in a format that can be
|
|
serialized.
|
|
"""
|
|
data = {
|
|
"directed": True,
|
|
"multigraph": False,
|
|
"graph": {},
|
|
"links": [],
|
|
"nodes": [],
|
|
}
|
|
|
|
# Do one pass to build a map of node -> position in nodes
|
|
node_to_number = {}
|
|
for node in self.adjacency_map.keys():
|
|
node_to_number[node] = len(data["nodes"])
|
|
data["nodes"].append({"id": node})
|
|
|
|
# Do another pass to build the link information
|
|
for node, neighbors in self.adjacency_map.items():
|
|
for neighbor in neighbors:
|
|
link = self.attributes_map[(node, neighbor)].copy()
|
|
link["source"] = node_to_number[node]
|
|
link["target"] = node_to_number[neighbor]
|
|
data["links"].append(link)
|
|
return data
|
|
|
|
|
|
def strongly_connected_components(G): # noqa: C901
|
|
"""
|
|
Adapted from networkx: http://networkx.github.io/
|
|
Parameters
|
|
----------
|
|
G : DiGraph
|
|
Returns
|
|
-------
|
|
comp : generator of sets
|
|
A generator of sets of nodes, one for each strongly connected
|
|
component of G.
|
|
"""
|
|
preorder = {}
|
|
lowlink = {}
|
|
scc_found = {}
|
|
scc_queue = []
|
|
i = 0 # Preorder counter
|
|
for source in G.nodes():
|
|
if source not in scc_found:
|
|
queue = [source]
|
|
while queue:
|
|
v = queue[-1]
|
|
if v not in preorder:
|
|
i = i + 1
|
|
preorder[v] = i
|
|
done = 1
|
|
v_nbrs = G.neighbors(v)
|
|
for w in v_nbrs:
|
|
if w not in preorder:
|
|
queue.append(w)
|
|
done = 0
|
|
break
|
|
if done == 1:
|
|
lowlink[v] = preorder[v]
|
|
for w in v_nbrs:
|
|
if w not in scc_found:
|
|
if preorder[w] > preorder[v]:
|
|
lowlink[v] = min([lowlink[v], lowlink[w]])
|
|
else:
|
|
lowlink[v] = min([lowlink[v], preorder[w]])
|
|
queue.pop()
|
|
if lowlink[v] == preorder[v]:
|
|
scc_found[v] = True
|
|
scc = {v}
|
|
while scc_queue and preorder[scc_queue[-1]] > preorder[v]:
|
|
k = scc_queue.pop()
|
|
scc_found[k] = True
|
|
scc.add(k)
|
|
yield scc
|
|
else:
|
|
scc_queue.append(v)
|
|
|
|
|
|
def simple_cycles(G): # noqa: C901
|
|
"""
|
|
Adapted from networkx: http://networkx.github.io/
|
|
Parameters
|
|
----------
|
|
G : DiGraph
|
|
Returns
|
|
-------
|
|
cycle_generator: generator
|
|
A generator that produces elementary cycles of the graph.
|
|
Each cycle is represented by a list of nodes along the cycle.
|
|
"""
|
|
|
|
def _unblock(thisnode, blocked, B):
|
|
stack = {thisnode}
|
|
while stack:
|
|
node = stack.pop()
|
|
if node in blocked:
|
|
blocked.remove(node)
|
|
stack.update(B[node])
|
|
B[node].clear()
|
|
|
|
# Johnson's algorithm requires some ordering of the nodes.
|
|
# We assign the arbitrary ordering given by the strongly connected comps
|
|
# There is no need to track the ordering as each node removed as processed.
|
|
# save the actual graph so we can mutate it here
|
|
# We only take the edges because we do not want to
|
|
# copy edge and node attributes here.
|
|
subG = G.subgraph(G.nodes())
|
|
sccs = list(strongly_connected_components(subG))
|
|
while sccs:
|
|
scc = sccs.pop()
|
|
# order of scc determines ordering of nodes
|
|
startnode = scc.pop()
|
|
# Processing node runs 'circuit' routine from recursive version
|
|
path = [startnode]
|
|
blocked = set() # vertex: blocked from search?
|
|
closed = set() # nodes involved in a cycle
|
|
blocked.add(startnode)
|
|
B = defaultdict(set) # graph portions that yield no elementary circuit
|
|
stack = [(startnode, list(subG.neighbors(startnode)))]
|
|
while stack:
|
|
thisnode, nbrs = stack[-1]
|
|
if nbrs:
|
|
nextnode = nbrs.pop()
|
|
if nextnode == startnode:
|
|
yield path[:]
|
|
closed.update(path)
|
|
elif nextnode not in blocked:
|
|
path.append(nextnode)
|
|
stack.append((nextnode, list(subG.neighbors(nextnode))))
|
|
closed.discard(nextnode)
|
|
blocked.add(nextnode)
|
|
continue
|
|
# done with nextnode... look for more neighbors
|
|
if not nbrs: # no more nbrs
|
|
if thisnode in closed:
|
|
_unblock(thisnode, blocked, B)
|
|
else:
|
|
for nbr in subG.neighbors(thisnode):
|
|
if thisnode not in B[nbr]:
|
|
B[nbr].add(thisnode)
|
|
stack.pop()
|
|
path.pop()
|
|
# done processing this node
|
|
subG.remove_node(startnode)
|
|
H = subG.subgraph(scc) # make smaller to avoid work in SCC routine
|
|
sccs.extend(list(strongly_connected_components(H)))
|
|
|
|
|
|
def find_cycle(graph):
|
|
"""
|
|
Looks for a cycle in the graph. If found, returns the first cycle.
|
|
If nodes a1, a2, ..., an are in a cycle, then this returns:
|
|
[(a1,a2), (a2,a3), ... (an-1,an), (an, a1)]
|
|
Otherwise returns an empty list.
|
|
"""
|
|
cycles = list(simple_cycles(graph))
|
|
if cycles:
|
|
nodes = cycles[0]
|
|
nodes.append(nodes[0])
|
|
edges = []
|
|
prev = nodes[0]
|
|
for node in nodes[1:]:
|
|
edges.append((prev, node))
|
|
prev = node
|
|
return edges
|
|
else:
|
|
return []
|
|
|
|
|
|
def get_stacktrace(thread_id):
|
|
"""
|
|
Returns the stack trace for the thread id as a list of strings.
|
|
"""
|
|
gdb.execute("thread %d" % thread_id, from_tty=False, to_string=True)
|
|
output = gdb.execute("bt", from_tty=False, to_string=True)
|
|
stacktrace_lines = output.strip().split("\n")
|
|
return stacktrace_lines
|
|
|
|
|
|
def is_thread_blocked_with_frame(
|
|
thread_id, top_line, expected_top_lines, expected_frame
|
|
):
|
|
"""
|
|
Returns True if we found expected_top_line in top_line, and
|
|
we found the expected_frame in the thread's stack trace.
|
|
"""
|
|
if all(expected not in top_line for expected in expected_top_lines):
|
|
return False
|
|
stacktrace_lines = get_stacktrace(thread_id)
|
|
return any(expected_frame in line for line in stacktrace_lines)
|
|
|
|
|
|
class MutexType(Enum):
|
|
"""Types of mutexes that we can detect deadlocks."""
|
|
|
|
PTHREAD_MUTEX_T = "pthread_mutex_t"
|
|
PTHREAD_RWLOCK_T = "pthread_rwlock_t"
|
|
|
|
@staticmethod
|
|
def get_mutex_type(thread_id, top_line):
|
|
"""
|
|
Returns the probable mutex type, based on the first line
|
|
of the thread's stack. Returns None if not found.
|
|
"""
|
|
|
|
WAITLIST = [
|
|
"__lll_lock_wait",
|
|
"futex_abstimed_wait",
|
|
"futex_abstimed_wait_cancelable",
|
|
"futex_reltimed_wait",
|
|
"futex_reltimed_wait_cancelable",
|
|
"futex_wait",
|
|
"futex_wait_cancelable",
|
|
]
|
|
|
|
if is_thread_blocked_with_frame(thread_id, top_line, WAITLIST, "pthread_mutex"):
|
|
return MutexType.PTHREAD_MUTEX_T
|
|
if is_thread_blocked_with_frame(
|
|
thread_id, top_line, WAITLIST, "pthread_rwlock"
|
|
):
|
|
return MutexType.PTHREAD_RWLOCK_T
|
|
return None
|
|
|
|
@staticmethod
|
|
def get_mutex_owner_and_address_func_for_type(mutex_type):
|
|
"""
|
|
Returns a function to resolve the mutex owner and address for
|
|
the given type. The returned function f has the following
|
|
signature:
|
|
|
|
f: args: (map of thread lwp -> thread id), blocked thread lwp
|
|
returns: (lwp of thread owning mutex, mutex address)
|
|
or (None, None) if not found.
|
|
|
|
Returns None if there is no function for this mutex_type.
|
|
"""
|
|
if mutex_type == MutexType.PTHREAD_MUTEX_T:
|
|
return get_pthread_mutex_t_owner_and_address
|
|
if mutex_type == MutexType.PTHREAD_RWLOCK_T:
|
|
return get_pthread_rwlock_t_owner_and_address
|
|
return None
|
|
|
|
|
|
def print_cycle(graph, lwp_to_thread_id, cycle):
|
|
"""Prints the threads and mutexes involved in the deadlock."""
|
|
for m, n in cycle:
|
|
print(
|
|
"Thread %d (LWP %d) is waiting on %s (0x%016x) held by "
|
|
"Thread %d (LWP %d)"
|
|
% (
|
|
lwp_to_thread_id[m],
|
|
m,
|
|
graph.attributes(m, n)["mutex_type"].value,
|
|
graph.attributes(m, n)["mutex"],
|
|
lwp_to_thread_id[n],
|
|
n,
|
|
)
|
|
)
|
|
|
|
|
|
def get_thread_info():
|
|
"""
|
|
Returns a pair of:
|
|
- map of LWP -> thread ID
|
|
- map of blocked threads LWP -> potential mutex type
|
|
"""
|
|
# LWP -> thread ID
|
|
lwp_to_thread_id = {}
|
|
|
|
# LWP -> potential mutex type it is blocked on
|
|
blocked_threads = {}
|
|
|
|
output = gdb.execute("info threads", from_tty=False, to_string=True)
|
|
lines = output.strip().split("\n")[1:]
|
|
regex = re.compile(r"[\s\*]*(\d+).*Thread.*\(LWP (\d+)\).*")
|
|
for line in lines:
|
|
try:
|
|
thread_id = int(regex.match(line).group(1))
|
|
thread_lwp = int(regex.match(line).group(2))
|
|
lwp_to_thread_id[thread_lwp] = thread_id
|
|
mutex_type = MutexType.get_mutex_type(thread_id, line)
|
|
if mutex_type:
|
|
blocked_threads[thread_lwp] = mutex_type
|
|
except Exception:
|
|
continue
|
|
|
|
return (lwp_to_thread_id, blocked_threads)
|
|
|
|
|
|
def get_pthread_mutex_t_owner_and_address(lwp_to_thread_id, thread_lwp):
|
|
"""
|
|
Finds the thread holding the mutex that this thread is blocked on.
|
|
Returns a pair of (lwp of thread owning mutex, mutex address),
|
|
or (None, None) if not found.
|
|
"""
|
|
# Go up the stack to the pthread_mutex_lock frame
|
|
gdb.execute(
|
|
"thread %d" % lwp_to_thread_id[thread_lwp], from_tty=False, to_string=True
|
|
)
|
|
gdb.execute("frame 1", from_tty=False, to_string=True)
|
|
|
|
# Get the owner of the mutex by inspecting the internal
|
|
# fields of the mutex.
|
|
try:
|
|
mutex_info = gdb.parse_and_eval("mutex").dereference()
|
|
mutex_owner_lwp = int(mutex_info["__data"]["__owner"])
|
|
return (mutex_owner_lwp, int(mutex_info.address))
|
|
except gdb.error:
|
|
return (None, None)
|
|
|
|
|
|
def get_pthread_rwlock_t_owner_and_address(lwp_to_thread_id, thread_lwp):
|
|
"""
|
|
If the thread is waiting on a write-locked pthread_rwlock_t, this will
|
|
return the pair of:
|
|
(lwp of thread that is write-owning the mutex, mutex address)
|
|
or (None, None) if not found, or if the mutex is read-locked.
|
|
"""
|
|
# Go up the stack to the pthread_rwlock_{rd|wr}lock frame
|
|
gdb.execute(
|
|
"thread %d" % lwp_to_thread_id[thread_lwp], from_tty=False, to_string=True
|
|
)
|
|
gdb.execute("frame 2", from_tty=False, to_string=True)
|
|
|
|
# Get the owner of the mutex by inspecting the internal
|
|
# fields of the mutex.
|
|
try:
|
|
rwlock_info = gdb.parse_and_eval("rwlock").dereference()
|
|
rwlock_data = rwlock_info["__data"]
|
|
field_names = ["__cur_writer", "__writer"]
|
|
fields = rwlock_data.type.fields()
|
|
field = [f for f in fields if f.name in field_names][0]
|
|
rwlock_owner_lwp = int(rwlock_data[field])
|
|
# We can only track the owner if it is currently write-locked.
|
|
# If it is not write-locked or if it is currently read-locked,
|
|
# possibly by multiple threads, we cannot find the owner.
|
|
if rwlock_owner_lwp != 0:
|
|
return (rwlock_owner_lwp, int(rwlock_info.address))
|
|
else:
|
|
return (None, None)
|
|
except gdb.error:
|
|
return (None, None)
|
|
|
|
|
|
class Deadlock(gdb.Command):
|
|
"""Detects deadlocks"""
|
|
|
|
def __init__(self):
|
|
super(Deadlock, self).__init__("deadlock", gdb.COMMAND_NONE)
|
|
|
|
def invoke(self, arg, from_tty):
|
|
"""Prints the threads and mutexes in a deadlock, if it exists."""
|
|
lwp_to_thread_id, blocked_threads = get_thread_info()
|
|
|
|
# Nodes represent threads. Edge (A,B) exists if thread A
|
|
# is waiting on a mutex held by thread B.
|
|
graph = DiGraph()
|
|
|
|
# Go through all the blocked threads and see which threads
|
|
# they are blocked on, and build the thread wait graph.
|
|
for thread_lwp, mutex_type in blocked_threads.items():
|
|
get_owner_and_address_func = (
|
|
MutexType.get_mutex_owner_and_address_func_for_type(mutex_type)
|
|
)
|
|
if not get_owner_and_address_func:
|
|
continue
|
|
mutex_owner_lwp, mutex_address = get_owner_and_address_func(
|
|
lwp_to_thread_id, thread_lwp
|
|
)
|
|
if mutex_owner_lwp and mutex_address:
|
|
graph.add_edge(
|
|
thread_lwp,
|
|
mutex_owner_lwp,
|
|
mutex=mutex_address,
|
|
mutex_type=mutex_type,
|
|
)
|
|
|
|
# A deadlock exists if there is a cycle in the graph.
|
|
cycle = find_cycle(graph)
|
|
if cycle:
|
|
print("Found deadlock!")
|
|
print_cycle(graph, lwp_to_thread_id, cycle)
|
|
else:
|
|
print("No deadlock detected. " "Do you have debug symbols installed?")
|
|
|
|
|
|
def load():
|
|
# instantiate the Deadlock command
|
|
Deadlock()
|
|
print('Type "deadlock" to detect deadlocks.')
|
|
|
|
|
|
def info():
|
|
return "Detect deadlocks"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
load()
|