gdb -ex "thread apply all bt": [New LWP 10301] [New LWP 10342] [New LWP 10300] [New LWP 10299] [New LWP 10298] [New LWP 10297] [New LWP 10296] [New LWP 10295] [New LWP 10294] [New LWP 10293] [New LWP 10291] [New LWP 10286] [New LWP 10285] [New LWP 10284] [New LWP 10283] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/libthread_db.so.1". 0xb765e7e1 in poll () from /lib/libc.so.6 Thread 16 (Thread 0xb51c2b40 (LWP 10283)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x821256f "taskprocessor.c", lineno=313, func=0x8212bc5 <__PRETTY_FUNCTION__.7782> "tps_processing_function", cond_name=0x82128c1 "&i->poll_cond", mutex_name=0x82128aa "&i->taskprocessor_lock", cond=0x84dd970, t=0x84dd9a4) at lock.c:598 #3 0x081a3cbe in tps_processing_function (data=0x84dd96c) at taskprocessor.c:313 #4 0x081b5da4 in dummy_start (data=0x84dda00) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 15 (Thread 0xb5186b40 (LWP 10284)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0x08081ff8 in listener (unused=0x0) at asterisk.c:1437 #2 0x081b5da4 in dummy_start (data=0x84e1a10) at utils.c:1223 #3 0xb7081679 in start_thread () from /lib/libpthread.so.0 #4 0xb766b58e in clone () from /lib/libc.so.6 Thread 14 (Thread 0xb514ab40 (LWP 10285)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x81fe68b "logger.c", lineno=1213, func=0x81ff5cb <__PRETTY_FUNCTION__.17557> "logger_thread", cond_name=0x81ff10d "&logcond", mutex_name=0x81ff0ff "&logmsgs.lock", cond=0x82688e0 , t=0x82509c8 ) at lock.c:598 #3 0x0813324a in logger_thread (data=0x0) at logger.c:1213 #4 0x081b5da4 in dummy_start (data=0x84e1a10) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 13 (Thread 0xb4dffb40 (LWP 10286)): #0 0xb708a7ed in __lll_lock_wait () from /lib/libpthread.so.0 #1 0xb7083ef6 in pthread_mutex_lock () from /lib/libpthread.so.0 #2 0xb767a4f2 in pthread_mutex_lock () from /lib/libc.so.6 #3 0x0812dbba in __ast_pthread_mutex_lock (filename=0x82105a3 "sched.c", lineno=480, func=0x82109e5 <__PRETTY_FUNCTION__.11280> "ast_sched_del", mutex_name=0x82105ab "&con->lock", t=0x8e87cc0) at lock.c:313 #4 0x08199bb7 in ast_sched_del (con=0x8e87cc0, id=214) at sched.c:480 #5 0xb39cc765 in destroy_peer (peer=0x8e78a68) at pbx_dundi.c:4323 #6 0xb39cc956 in prune_peers () at pbx_dundi.c:4344 #7 0xb39cf91f in unload_module () at pbx_dundi.c:4977 #8 0x0812adb5 in ast_module_shutdown () at loader.c:568 #9 0x08083239 in really_quit (num=0, niceness=SHUTDOWN_REALLY_NICE, restart=1) at asterisk.c:1812 #10 0x08082e91 in quit_handler (num=0, niceness=SHUTDOWN_REALLY_NICE, restart=1) at asterisk.c:1729 #11 0x08084186 in handle_restart_when_convenient (e=0x824c528 , cmd=-4, a=0xb4dfea98) at asterisk.c:2310 #12 0x080d56ef in ast_cli_command_full (uid=0, gid=70, fd=7, s=0xb4dfec34 "core restart when convenient") at cli.c:2693 #13 0x080d57ef in ast_cli_command_multiple_full (uid=0, gid=70, fd=7, size=29, s=0xb4dff088 "core restart when convenient") at cli.c:2716 #14 0x08081d86 in netconsole (vconsole=0x8280e60 ) at asterisk.c:1376 #15 0x081b5da4 in dummy_start (data=0xb4e00df8) at utils.c:1223 #16 0xb7081679 in start_thread () from /lib/libpthread.so.0 #17 0xb766b58e in clone () from /lib/libc.so.6 Thread 12 (Thread 0xb4dc3b40 (LWP 10291)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x81effeb "db.c", lineno=967, func=0x81f100f <__PRETTY_FUNCTION__.13890> "db_sync_thread", cond_name=0x81f0ef0 "&dbcond", mutex_name=0x81f0259 "&dblock", cond=0x8267ca0 , t=0x824de60 ) at lock.c:598 #3 0x080e741a in db_sync_thread (data=0x0) at db.c:967 #4 0x081b5da4 in dummy_start (data=0x85053e0) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 11 (Thread 0xb4d4bb40 (LWP 10293)): #0 0xb76274ac in nanosleep () from /lib/libc.so.6 #1 0xb766262b in usleep () from /lib/libc.so.6 #2 0x080ecb87 in do_refresh (data=0x0) at dnsmgr.c:255 #3 0x081b5da4 in dummy_start (data=0x87e76c0) at utils.c:1223 #4 0xb7081679 in start_thread () from /lib/libpthread.so.0 #5 0xb766b58e in clone () from /lib/libc.so.6 Thread 10 (Thread 0xb49ffb40 (LWP 10294)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0x081b60f5 in ast_wait_for_input (fd=11, ms=5000) at utils.c:1312 #2 0x081a6051 in ast_tcptls_server_root (data=0x8251080 ) at tcptls.c:714 #3 0x081b5da4 in dummy_start (data=0x8803738) at utils.c:1223 #4 0xb7081679 in start_thread () from /lib/libpthread.so.0 #5 0xb766b58e in clone () from /lib/libc.so.6 Thread 9 (Thread 0xb49c3b40 (LWP 10295)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x81f106b "devicestate.c", lineno=558, func=0x81f16cf <__PRETTY_FUNCTION__.13127> "do_devstate_changes", cond_name=0x81f131b "&change_pending", mutex_name=0x81f132b "&state_changes.lock", cond=0x8267d20 , t=0x824e1a8 ) at lock.c:598 #3 0x080e8542 in do_devstate_changes (data=0x0) at devicestate.c:558 #4 0x081b5da4 in dummy_start (data=0x87f1e48) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 8 (Thread 0xb4987b40 (LWP 10296)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x820c24b "presencestate.c", lineno=299, func=0x820c483 <__PRETTY_FUNCTION__.13076> "do_presence_changes", cond_name=0x820c38c "&change_pending", mutex_name=0x820c39c "&state_changes.lock", cond=0x8268a40 , t=0x82528e8 ) at lock.c:598 #3 0x081730a2 in do_presence_changes (data=0x0) at presencestate.c:299 #4 0x081b5da4 in dummy_start (data=0x87ec208) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 7 (Thread 0xb494bb40 (LWP 10297)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x821256f "taskprocessor.c", lineno=313, func=0x8212bc5 <__PRETTY_FUNCTION__.7782> "tps_processing_function", cond_name=0x82128c1 "&i->poll_cond", mutex_name=0x82128aa "&i->taskprocessor_lock", cond=0x8809848, t=0x880987c) at lock.c:598 #3 0x081a3cbe in tps_processing_function (data=0x8809844) at taskprocessor.c:313 #4 0x081b5da4 in dummy_start (data=0x88098c8) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 6 (Thread 0xb47ffb40 (LWP 10298)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0x081065bc in do_parking_thread (ignore=0x0) at features.c:5334 #2 0x081b5da4 in dummy_start (data=0x8814730) at utils.c:1223 #3 0xb7081679 in start_thread () from /lib/libpthread.so.0 #4 0xb766b58e in clone () from /lib/libc.so.6 Thread 5 (Thread 0xb47c3b40 (LWP 10299)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x821256f "taskprocessor.c", lineno=313, func=0x8212bc5 <__PRETTY_FUNCTION__.7782> "tps_processing_function", cond_name=0x82128c1 "&i->poll_cond", mutex_name=0x82128aa "&i->taskprocessor_lock", cond=0x881d6d0, t=0x881d704) at lock.c:598 #3 0x081a3cbe in tps_processing_function (data=0x881d6cc) at taskprocessor.c:313 #4 0x081b5da4 in dummy_start (data=0x881d750) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 4 (Thread 0xb4787b40 (LWP 10300)): #0 0xb7087381 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0xb767a30a in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libc.so.6 #2 0x0812ea35 in __ast_cond_wait (filename=0x82105a3 "sched.c", lineno=120, func=0x82108e1 <__PRETTY_FUNCTION__.11130> "sched_run", cond_name=0x82105b6 "&con->sched_thread->cond", mutex_name=0x82105ab "&con->lock", cond=0x88218d4, t=0x881d860) at lock.c:598 #3 0x08198caa in sched_run (data=0x881d860) at sched.c:120 #4 0x081b5da4 in dummy_start (data=0x881d750) at utils.c:1223 #5 0xb7081679 in start_thread () from /lib/libpthread.so.0 #6 0xb766b58e in clone () from /lib/libc.so.6 Thread 3 (Thread 0xb3510b40 (LWP 10342)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0xb4012666 in scan_thread (unused=0x0) at pbx_spool.c:730 #2 0x081b5da4 in dummy_start (data=0x8d73378) at utils.c:1223 #3 0xb7081679 in start_thread () from /lib/libpthread.so.0 #4 0xb766b58e in clone () from /lib/libc.so.6 Thread 2 (Thread 0xb463bb40 (LWP 10301)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0x081b60f5 in ast_wait_for_input (fd=13, ms=-1) at utils.c:1312 #2 0x08142a79 in get_input (s=0xb463b108, output=0xb463aa3f "") at manager.c:5387 #3 0x08142d84 in do_message (s=0xb463b108) at manager.c:5459 #4 0x08143408 in session_do (data=0x8e503b4) at manager.c:5610 #5 0x081a5fff in handle_tcptls_connection (data=0x8e503b4) at tcptls.c:694 #6 0x081b5da4 in dummy_start (data=0x89c85b8) at utils.c:1223 #7 0xb7081679 in start_thread () from /lib/libpthread.so.0 #8 0xb766b58e in clone () from /lib/libc.so.6 Thread 1 (Thread 0xb520a940 (LWP 10282)): #0 0xb765e7e1 in poll () from /lib/libc.so.6 #1 0x08087e51 in monitor_sig_flags (unused=0x0) at asterisk.c:3521 #2 0x0808a807 in main (argc=1, argv=0xbffe3a14) at asterisk.c:4421 asterisk -x "core show locks": ======================================================================= === 11.19.0 === Currently Held Locks ======================================================================= === === (): (times locked) === === Thread ID: 0xb4dffb40 (netconsole started at [ 1479] asterisk.c listener()) === ---> Lock #0 (loader.c): MUTEX 547 ast_module_shutdown &(&module_list)->lock 0x82505c8 (1) main/logger.c:1701 ast_bt_get_addresses() (0x8134729+19) main/lock.c:258 __ast_pthread_mutex_lock() (0x812da6e+82) main/loader.c:553 ast_module_shutdown() (0x812acad+40) main/asterisk.c:1815 really_quit() main/asterisk.c:1733 quit_handler() main/asterisk.c:2311 handle_restart_when_convenient() main/cli.c:2693 ast_cli_command_full() (0x80d54c8+227) main/cli.c:2717 ast_cli_command_multiple_full() (0x80d5782+6D) main/asterisk.c:1376 netconsole() main/utils.c:1223 dummy_start() :0 start_thread() :0 __clone() (0xb766b530+5E) === ---> Lock #1 (pbx_dundi.c): MUTEX 4340 prune_peers &(&peers)->lock 0xb39da8e8 (1) main/logger.c:1701 ast_bt_get_addresses() (0x8134729+19) main/lock.c:258 __ast_pthread_mutex_lock() (0x812da6e+82) pbx/pbx_dundi.c:4341 prune_peers() pbx/pbx_dundi.c:4979 unload_module() main/loader.c:570 ast_module_shutdown() (0x812acad+108) main/asterisk.c:1815 really_quit() main/asterisk.c:1733 quit_handler() main/asterisk.c:2311 handle_restart_when_convenient() main/cli.c:2693 ast_cli_command_full() (0x80d54c8+227) main/cli.c:2717 ast_cli_command_multiple_full() (0x80d5782+6D) main/asterisk.c:1376 netconsole() main/utils.c:1223 dummy_start() :0 start_thread() :0 __clone() (0xb766b530+5E) === ---> Waiting for Lock #2 (sched.c): MUTEX 480 ast_sched_del &con->lock 0x8e87cc0 (1) main/logger.c:1701 ast_bt_get_addresses() (0x8134729+19) main/lock.c:258 __ast_pthread_mutex_lock() (0x812da6e+82) main/sched.c:481 ast_sched_del() (0x8199b2b+8C) pbx/pbx_dundi.c:4323 destroy_peer() pbx/pbx_dundi.c:4341 prune_peers() pbx/pbx_dundi.c:4979 unload_module() main/loader.c:570 ast_module_shutdown() (0x812acad+108) main/asterisk.c:1815 really_quit() main/asterisk.c:1733 quit_handler() main/asterisk.c:2311 handle_restart_when_convenient() main/cli.c:2693 ast_cli_command_full() (0x80d54c8+227) main/cli.c:2717 ast_cli_command_multiple_full() (0x80d5782+6D) main/asterisk.c:1376 netconsole() main/utils.c:1223 dummy_start() :0 start_thread() :0 __clone() (0xb766b530+5E) === ------------------------------------------------------------------- === ======================================================================= asterisk -x "core show threads": 0xb345cb40 14412 netconsole started at [ 1479] asterisk.c listener() 0xb463bb40 10301 handle_tcptls_connection started at [ 745] tcptls.c ast_tcptls_server_root() 0xb3510b40 10342 scan_thread started at [ 921] pbx_spool.c load_module() 0xb4787b40 10300 sched_run started at [ 182] sched.c ast_sched_start_thread() 0xb47c3b40 10299 tps_processing_function started at [ 471] taskprocessor.c ast_taskprocessor_get() 0xb47ffb40 10298 do_parking_thread started at [ 9095] features.c ast_features_init() 0xb494bb40 10297 tps_processing_function started at [ 471] taskprocessor.c ast_taskprocessor_get() 0xb4987b40 10296 do_presence_changes started at [ 318] presencestate.c ast_presence_state_engine_init() 0xb49c3b40 10295 do_devstate_changes started at [ 759] devicestate.c ast_device_state_engine_init() 0xb49ffb40 10294 desc->accept_fn started at [ 1032] tcptls.c ast_tcptls_server_start() 0xb4d4bb40 10293 do_refresh started at [ 490] dnsmgr.c do_reload() 0xb4dc3b40 10291 db_sync_thread started at [ 1018] db.c astdb_init() 0xb4dffb40 10286 netconsole started at [ 1479] asterisk.c listener() 0xb514ab40 10285 logger_thread started at [ 1309] logger.c init_logger() 0xb5186b40 10284 listener started at [ 1539] asterisk.c ast_makesocket() 0xb51c2b40 10283 tps_processing_function started at [ 471] taskprocessor.c ast_taskprocessor_get() 16 threads listed. asterisk -x "core show taskprocessors": +----- Processor -----+--- Processed ---+- In Queue -+- Max Depth -+ core_event_dispatcher 72 0 12 pbx-core 41 0 3 CCSS core 0 0 0 +---------------------+-----------------+------------+-------------+ 3 taskprocessors