pc2372*CLI> core show locks ======================================================================= === Currently Held Locks ============================================== ======================================================================= === === (): (times locked) === === Thread ID: 0xb759eb90 (tps_processing_function started at [ 451] taskprocessor.c ast_taskprocessor_get()) === ---> Lock #0 (pbx.c): MUTEX 4286 handle_statechange hints 0x8ac6180 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(__ao2_lock+0x4a) [0x8086004] /usr/sbin/asterisk [0x8141a69] /usr/sbin/asterisk [0x8185bd0] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Lock #1 (pbx.c): MUTEX 4287 handle_statechange hint 0xb623d668 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(__ao2_lock+0x4a) [0x8086004] /usr/sbin/asterisk [0x8141a94] /usr/sbin/asterisk [0x8185bd0] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Waiting for Lock #2 (pbx.c): MUTEX 9981 ast_rdlock_contexts &conlock 0x82068a0 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(ast_rdlock_contexts+0x32) [0x8154a9b] /usr/sbin/asterisk [0x8141640] /usr/sbin/asterisk(ast_get_hint+0x20) [0x81427dd] /usr/sbin/asterisk [0x812e2d2] /usr/sbin/asterisk [0x8141b7f] /usr/sbin/asterisk [0x8185bd0] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === --- ---> Locked Here: pbx.c line 9981 (ast_rdlock_contexts) === ------------------------------------------------------------------- === === Thread ID: 0xb6655b90 (do_monitor started at [25035] chan_sip.c restart_monitor()) === ---> Lock #0 (chan_sip.c): MUTEX 24550 handle_request_do &netlock 0xb732e860 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/lib/asterisk/modules/chan_sip.so [0xb72ea724] /usr/lib/asterisk/modules/chan_sip.so [0xb72ea4f8] /usr/sbin/asterisk(ast_io_wait+0x14d) [0x8113e8b] /usr/lib/asterisk/modules/chan_sip.so [0xb72ebfd6] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Lock #1 (chan_sip.c): MUTEX 7736 sip_pvt_lock_full pvt 0xb5efa600 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(__ao2_lock+0x4a) [0x8086004] /usr/lib/asterisk/modules/chan_sip.so [0xb7296fd7] /usr/lib/asterisk/modules/chan_sip.so [0xb72ea815] /usr/lib/asterisk/modules/chan_sip.so [0xb72ea4f8] /usr/sbin/asterisk(ast_io_wait+0x14d) [0x8113e8b] /usr/lib/asterisk/modules/chan_sip.so [0xb72ebfd6] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Waiting for Lock #2 (pbx.c): MUTEX 9981 ast_rdlock_contexts &conlock 0x82068a0 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(ast_rdlock_contexts+0x32) [0x8154a9b] /usr/sbin/asterisk [0x8141640] /usr/sbin/asterisk(ast_get_hint+0x20) [0x81427dd] /usr/sbin/asterisk [0x812e2d2] /usr/sbin/asterisk [0x8141b7f] /usr/sbin/asterisk [0x8185bd0] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === --- ---> Locked Here: pbx.c line 9981 (ast_rdlock_contexts) === ------------------------------------------------------------------- === === Thread ID: 0xb61c3b90 (netconsole started at [ 1350] asterisk.c listener()) === ---> Lock #0 (loader.c): MUTEX 673 ast_module_reload &reloadlock 0x8205260 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_trylock+0xb5) [0x811990c] /usr/sbin/asterisk(ast_module_reload+0x6d) [0x81172a0] /usr/sbin/asterisk [0x80c394a] /usr/sbin/asterisk(ast_cli_command_full+0x275) [0x80cb0c1] /usr/sbin/asterisk(ast_cli_command_multiple_full+0x67) [0x80cb1f7] /usr/sbin/asterisk [0x807d61d] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Lock #1 (loader.c): MUTEX 711 ast_module_reload &(&module_list)->lock 0x82051e8 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(ast_module_reload+0x2a9) [0x81174dc] /usr/sbin/asterisk [0x80c394a] /usr/sbin/asterisk(ast_cli_command_full+0x275) [0x80cb0c1] /usr/sbin/asterisk(ast_cli_command_multiple_full+0x67) [0x80cb1f7] /usr/sbin/asterisk [0x807d61d] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Lock #2 (pbx.c): MUTEX 9981 ast_rdlock_contexts &conlock 0x82068a0 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(ast_rdlock_contexts+0x32) [0x8154a9b] /usr/sbin/asterisk(ast_merge_contexts_and_delete+0x6f) [0x814bb39] /usr/lib/asterisk/modules/pbx_config.so [0xb73e3946] /usr/lib/asterisk/modules/pbx_config.so [0xb73e3a4e] /usr/sbin/asterisk(ast_module_reload+0x3e9) [0x811761c] /usr/sbin/asterisk [0x80c394a] /usr/sbin/asterisk(ast_cli_command_full+0x275) [0x80cb0c1] /usr/sbin/asterisk(ast_cli_command_multiple_full+0x67) [0x80cb1f7] /usr/sbin/asterisk [0x807d61d] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === ---> Waiting for Lock #3 (astobj2.c): MUTEX 493 internal_ao2_link c 0x8ac6180 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x19) [0x811fdf6] /usr/sbin/asterisk(__ast_pthread_mutex_lock+0xb5) [0x8119593] /usr/sbin/asterisk(__ao2_lock+0x4a) [0x8086004] /usr/sbin/asterisk [0x808683c] /usr/sbin/asterisk(__ao2_link+0x30) [0x80869c9] /usr/sbin/asterisk [0x814257b] /usr/sbin/asterisk [0x814f016] /usr/sbin/asterisk(ast_add_extension2+0x60) [0x814e894] /usr/sbin/asterisk [0x814b92d] /usr/sbin/asterisk(ast_merge_contexts_and_delete+0xa1) [0x814bb6b] /usr/lib/asterisk/modules/pbx_config.so [0xb73e3946] /usr/lib/asterisk/modules/pbx_config.so [0xb73e3a4e] /usr/sbin/asterisk(ast_module_reload+0x3e9) [0x811761c] /usr/sbin/asterisk [0x80c394a] /usr/sbin/asterisk(ast_cli_command_full+0x275) [0x80cb0c1] /usr/sbin/asterisk(ast_cli_command_multiple_full+0x67) [0x80cb1f7] /usr/sbin/asterisk [0x807d61d] /usr/sbin/asterisk [0x8197008] /lib/i686/cmov/libpthread.so.0 [0xb7ae74c0] /lib/i686/cmov/libc.so.6(clone+0x5e) [0xb7d1761e] === --- ---> Locked Here: pbx.c line 4286 (handle_statechange) === ------------------------------------------------------------------- === ======================================================================= (gdb) (gdb) bt #0 0xb7f46424 in __kernel_vsyscall () #1 0xb7d0716b in read () from /lib/i686/cmov/libc.so.6 #2 0x081a7711 in read_char (el=0x8acfe50, cp=0xbfc5e40b "�") at read.c:298 #3 0x081a2817 in el_getc (el=0x8acfe50, cp=0xbfc5e40b "�") at read.c:350 #4 0x081a2a41 in el_gets (el=0x8acfe50, nread=0xbfc5f6a8) at read.c:243 #5 0x08085d25 in main (argc=4, argv=0xbfc5f944) at asterisk.c:3867 (gdb) info threads 39 Thread 0xb7a50b90 (LWP 22118) 0xb7f46424 in __kernel_vsyscall () 38 Thread 0xb7a14b90 (LWP 22119) 0xb7f46424 in __kernel_vsyscall () 37 Thread 0xb79d8b90 (LWP 22120) 0xb7f46424 in __kernel_vsyscall () 36 Thread 0xb799cb90 (LWP 22121) 0xb7f46424 in __kernel_vsyscall () 35 Thread 0xb7616b90 (LWP 22147) 0xb7f46424 in __kernel_vsyscall () 34 Thread 0xb75dab90 (LWP 22148) 0xb7f46424 in __kernel_vsyscall () 33 Thread 0xb759eb90 (LWP 22149) 0xb7f46424 in __kernel_vsyscall () 32 Thread 0xb7562b90 (LWP 22150) 0xb7f46424 in __kernel_vsyscall () 31 Thread 0xb7526b90 (LWP 22151) 0xb7f46424 in __kernel_vsyscall () 30 Thread 0xb74eab90 (LWP 22152) 0xb7f46424 in __kernel_vsyscall () 29 Thread 0xb74aeb90 (LWP 22153) 0xb7f46424 in __kernel_vsyscall () 28 Thread 0xb69bfb90 (LWP 22158) 0xb7f46424 in __kernel_vsyscall () 27 Thread 0xb6961b90 (LWP 22159) 0xb7f46424 in __kernel_vsyscall () 26 Thread 0xb6925b90 (LWP 22160) 0xb7f46424 in __kernel_vsyscall () 25 Thread 0xb68e9b90 (LWP 22169) 0xb7f46424 in __kernel_vsyscall () 24 Thread 0xb68adb90 (LWP 22170) 0xb7f46424 in __kernel_vsyscall () 23 Thread 0xb6871b90 (LWP 22171) 0xb7f46424 in __kernel_vsyscall () 22 Thread 0xb6835b90 (LWP 22172) 0xb7f46424 in __kernel_vsyscall () 21 Thread 0xb67f9b90 (LWP 22173) 0xb7f46424 in __kernel_vsyscall () 20 Thread 0xb67bdb90 (LWP 22174) 0xb7f46424 in __kernel_vsyscall () 19 Thread 0xb6781b90 (LWP 22175) 0xb7f46424 in __kernel_vsyscall () 18 Thread 0xb6745b90 (LWP 22176) 0xb7f46424 in __kernel_vsyscall () 17 Thread 0xb6709b90 (LWP 22177) 0xb7f46424 in __kernel_vsyscall () 16 Thread 0xb66cdb90 (LWP 22178) 0xb7f46424 in __kernel_vsyscall () 15 Thread 0xb6691b90 (LWP 22179) 0xb7f46424 in __kernel_vsyscall () 14 Thread 0xb6655b90 (LWP 22190) 0xb7f46424 in __kernel_vsyscall () 13 Thread 0xb6619b90 (LWP 22193) 0xb7f46424 in __kernel_vsyscall () 12 Thread 0xb65ddb90 (LWP 22199) 0xb7f46424 in __kernel_vsyscall () 11 Thread 0xb65a1b90 (LWP 22200) 0xb7f46424 in __kernel_vsyscall () 10 Thread 0xb6565b90 (LWP 22201) 0xb7f46424 in __kernel_vsyscall () 9 Thread 0xb63ffb90 (LWP 22202) 0xb7f46424 in __kernel_vsyscall () 8 Thread 0xb63c3b90 (LWP 22203) 0xb7f46424 in __kernel_vsyscall () 7 Thread 0xb6387b90 (LWP 22204) 0xb7f46424 in __kernel_vsyscall () 6 Thread 0xb634bb90 (LWP 22205) 0xb7f46424 in __kernel_vsyscall () 5 Thread 0xb61ffb90 (LWP 22206) 0xb7f46424 in __kernel_vsyscall () 4 Thread 0xb61c3b90 (LWP 22891) 0xb7f46424 in __kernel_vsyscall () 3 Thread 0xb614bb90 (LWP 23059) 0xb7f46424 in __kernel_vsyscall () 2 Thread 0xb5dc3b90 (LWP 23574) 0xb7f46424 in __kernel_vsyscall () 1 Thread 0xb7a5e9e0 (LWP 24106) 0xb7f46424 in __kernel_vsyscall () (gdb) thread 4 [Switching to thread 4 (Thread 0xb61c3b90 (LWP 22891))]#0 0xb7f46424 in __kernel_vsyscall () (gdb) bt #0 0xb7f46424 in __kernel_vsyscall () #1 0xb7aedc99 in __lll_lock_wait () from /lib/i686/cmov/libpthread.so.0 #2 0xb7ae90d3 in _L_lock_291 () from /lib/i686/cmov/libpthread.so.0 #3 0xb7ae8b36 in pthread_mutex_lock () from /lib/i686/cmov/libpthread.so.0 #4 0xb7d25926 in pthread_mutex_lock () from /lib/i686/cmov/libc.so.6 #5 0x08119603 in __ast_pthread_mutex_lock (filename=0x81c71f4 "astobj2.c", lineno=493, func=0x81c7334 "internal_ao2_link", mutex_name=0x81c7346 "c", t=0x8ac6180) at lock.c:244 #6 0x08086004 in __ao2_lock (user_data=0x8ac61b0, file=0x81c71f4 "astobj2.c", func=0x81c7334 "internal_ao2_link", line=493, var=0x81c7346 "c") at astobj2.c:157 #7 0x0808683c in internal_ao2_link (c=0x8ac61b0, user_data=0xb64512c0, file=0x81c71f4 "astobj2.c", line=517, func=0x81c7359 "__ao2_link") at astobj2.c:493 #8 0x080869c9 in __ao2_link (c=0x8ac61b0, user_data=0xb64512c0) at astobj2.c:517 #9 0x0814257b in ast_add_hint (e=0xb5e74be8) at pbx.c:4498 #10 0x0814f016 in ast_add_extension2_lockopt (con=0xb602d270, replace=0, extension=0xb629e6bc "***6099", priority=-1, label=0x0, callerid=0x0, application=0xb629e6c5 "SIP/6099", data=0x0, datad=0x8063c30 , registrar=0xb746e82f "pbx_ael", lockconts=1, lockhints=1) at pbx.c:8319 #11 0x0814e894 in ast_add_extension2 (con=0xb602d270, replace=0, extension=0xb629e6bc "***6099", priority=-1, label=0x0, callerid=0x0, application=0xb629e6c5 "SIP/6099", data=0x0, datad=0x8063c30 , registrar=0xb746e82f "pbx_ael") at pbx.c:8132 #12 0x0814b92d in context_merge (extcontexts=0xb73e6dac, exttable=0xeff2b98, context=0xf248e98, registrar=0xb73e3b31 "pbx_config") at pbx.c:7053 #13 0x0814bb6b in ast_merge_contexts_and_delete (extcontexts=0xb73e6dac, exttable=0xeff2b98, registrar=0xb73e3b31 "pbx_config") at pbx.c:7111 #14 0xb73e3946 in pbx_load_module () at pbx_config.c:1762 #15 0xb73e3a4e in reload () at pbx_config.c:1792 #16 0x0811761c in ast_module_reload (name=0x0) at loader.c:737 #17 0x080c394a in handle_core_reload (e=0x8202718, cmd=-4, a=0xb61c2cec) at cli.c:324 #18 0x080cb0c1 in ast_cli_command_full (uid=0, gid=0, fd=18, s=0xb61c2e8c "core reload") at cli.c:2488 #19 0x080cb1f7 in ast_cli_command_multiple_full (uid=0, gid=0, fd=18, size=12, s=0xb61c30f4 "core reload") at cli.c:2511 #20 0x0807d61d in netconsole (vconsole=0x822d9e0) at asterisk.c:1272 #21 0x08197008 in dummy_start (data=0xb64e8c68) at utils.c:1004 #22 0xb7ae74c0 in start_thread () from /lib/i686/cmov/libpthread.so.0 #23 0xb7d1761e in clone () from /lib/i686/cmov/libc.so.6 (gdb) thread 14 [Switching to thread 14 (Thread 0xb6655b90 (LWP 22190))]#0 0xb7f46424 in __kernel_vsyscall () (gdb) bt #0 0xb7f46424 in __kernel_vsyscall () #1 0xb7aedc99 in __lll_lock_wait () from /lib/i686/cmov/libpthread.so.0 #2 0xb7ae90d3 in _L_lock_291 () from /lib/i686/cmov/libpthread.so.0 #3 0xb7ae8b36 in pthread_mutex_lock () from /lib/i686/cmov/libpthread.so.0 #4 0xb7d25926 in pthread_mutex_lock () from /lib/i686/cmov/libc.so.6 #5 0x08119603 in __ast_pthread_mutex_lock (filename=0x81e8af4 "pbx.c", lineno=9981, func=0x81ee8cb "ast_rdlock_contexts", mutex_name=0x81ee8c2 "&conlock", t=0x82068a0) at lock.c:244 #6 0x08154a9b in ast_rdlock_contexts () at pbx.c:9981 #7 0x08140cae in pbx_extension_helper (c=0x0, con=0x0, context=0xb6473903 "from-internal-users", exten=0xb6653fd4 "6043", priority=1, label=0x0, callerid=0xb64738a3 "6003", action=E_MATCH, found=0x0, combined_find_spawn=0) at pbx.c:4050 #8 0x08142955 in ast_exists_extension (c=0x0, context=0xb6473903 "from-internal-users", exten=0xb6653fd4 "6043", priority=1, callerid=0xb64738a3 "6003") at pbx.c:4600 #9 0xb72bab97 in get_destination (p=0xb5efa630, oreq=0x0, cc_recall_core_id=0xb6654660) at chan_sip.c:14865 #10 0xb72df9d7 in handle_request_invite (p=0xb5efa630, req=0xb6654d60, debug=0, seqno=2, addr=0xb6654cdc, recount=0xb6654c90, e=0xb5e5eb9b "sip:6043@10.52.17.132:5060", nounlock=0xb6654c8c) at chan_sip.c:21950 #11 0xb72e9f15 in handle_incoming (p=0xb5efa630, req=0xb6654d60, addr=0xb6654cdc, recount=0xb6654c90, nounlock=0xb6654c8c) at chan_sip.c:24392 #12 0xb72ea903 in handle_request_do (req=0xb6654d60, addr=0xb6654cdc) at chan_sip.c:24575 #13 0xb72ea4f8 in sipsock_read (id=0xef19ff8, fd=12, events=1, ignore=0x0) at chan_sip.c:24509 #14 0x08113e8b in ast_io_wait (ioc=0xb78b6920, howlong=109) at io.c:288 #15 0xb72ebfd6 in do_monitor (data=0x0) at chan_sip.c:25004 #16 0x08197008 in dummy_start (data=0xb78cfcc0) at utils.c:1004 #17 0xb7ae74c0 in start_thread () from /lib/i686/cmov/libpthread.so.0 #18 0xb7d1761e in clone () from /lib/i686/cmov/libc.so.6 (gdb) thread 33 [Switching to thread 33 (Thread 0xb759eb90 (LWP 22149))]#0 0xb7f46424 in __kernel_vsyscall () (gdb) bt #0 0xb7f46424 in __kernel_vsyscall () #1 0xb7aedc99 in __lll_lock_wait () from /lib/i686/cmov/libpthread.so.0 #2 0xb7ae90d3 in _L_lock_291 () from /lib/i686/cmov/libpthread.so.0 #3 0xb7ae8b36 in pthread_mutex_lock () from /lib/i686/cmov/libpthread.so.0 #4 0xb7d25926 in pthread_mutex_lock () from /lib/i686/cmov/libc.so.6 #5 0x08119603 in __ast_pthread_mutex_lock (filename=0x81e8af4 "pbx.c", lineno=9981, func=0x81ee8cb "ast_rdlock_contexts", mutex_name=0x81ee8c2 "&conlock", t=0x82068a0) at lock.c:244 #6 0x08154a9b in ast_rdlock_contexts () at pbx.c:9981 #7 0x08141640 in ast_hint_extension (c=0x0, context=0xf248f08 "to-internal-users", exten=0xb60e8fac "6003") at pbx.c:4155 #8 0x081427dd in ast_get_hint (hint=0xb759e058 "", hintsize=512, name=0x0, namesize=0, c=0x0, context=0xf248f08 "to-internal-users", exten=0xb60e8fac "6003") at pbx.c:4562 #9 0x0812e2d2 in manager_state_cb (context=0xf248f08 "to-internal-users", exten=0xb60e8fac "6003", state=1, data=0x0) at manager.c:4996 #10 0x08141b7f in handle_statechange (datap=0xb5baf000) at pbx.c:4299 #11 0x08185bd0 in tps_processing_function (data=0xb78102d8) at taskprocessor.c:310 #12 0x08197008 in dummy_start (data=0xb78012c0) at utils.c:1004 #13 0xb7ae74c0 in start_thread () from /lib/i686/cmov/libpthread.so.0 #14 0xb7d1761e in clone () from /lib/i686/cmov/libc.so.6 (gdb)