[Thread debugging using libthread_db enabled] [New Thread 0x7fcbe93e56e0 (LWP 14775)] [New Thread 0x408a1950 (LWP 32372)] [New Thread 0x40bf5950 (LWP 32371)] [New Thread 0x40a3b950 (LWP 32370)] [New Thread 0x409bf950 (LWP 32369)] [New Thread 0x4157c950 (LWP 32368)] [New Thread 0x414a4950 (LWP 32367)] [New Thread 0x41428950 (LWP 32366)] [New Thread 0x410f0950 (LWP 32365)] [New Thread 0x41074950 (LWP 32364)] [New Thread 0x40b30950 (LWP 32363)] [New Thread 0x40763950 (LWP 32362)] [New Thread 0x41c65950 (LWP 32361)] [New Thread 0x41f9d950 (LWP 32360)] [New Thread 0x40d64950 (LWP 32359)] [New Thread 0x40e91950 (LWP 32358)] [New Thread 0x40825950 (LWP 32357)] [New Thread 0x41da8950 (LWP 32356)] [New Thread 0x41d2c950 (LWP 32355)] [New Thread 0x413ac950 (LWP 32354)] [New Thread 0x41330950 (LWP 32353)] [New Thread 0x412b4950 (LWP 32352)] [New Thread 0x41238950 (LWP 32351)] [New Thread 0x411bc950 (LWP 32350)] 0x00007fcbe895aa1b in read () from /lib/libc.so.6 Thread 24 (Thread 0x411bc950 (LWP 32350)): #0 0x00007fcbe8937071 in nanosleep () from /lib/libc.so.6 #1 0x00007fcbe8936e97 in sleep () from /lib/libc.so.6 #2 0x0000000000441967 in canary_thread (unused=0x0) at asterisk.c:3087 #3 0x0000000000559f84 in dummy_start (data=0x1c6be40) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 23 (Thread 0x41238950 (LWP 32351)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x0000000000547dfd in __ast_cond_wait (filename=0x5c0ba4 "taskprocessor.c", lineno=293, func=0x5c1150 "tps_processing_function", cond_name=0x5c11c9 "&i->poll_cond", mutex_name=0x5c11b2 "&i->taskprocessor_lock", cond=0x1c7ab80, t=0x1c7abb8) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x000000000054774e in tps_processing_function (data=0x1c7ab78) at taskprocessor.c:293 #3 0x0000000000559f84 in dummy_start (data=0x1c7b7c0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 22 (Thread 0x412b4950 (LWP 32352)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x000000000043c29d in listener (unused=0x0) at asterisk.c:1285 #2 0x0000000000559f84 in dummy_start (data=0x1c7ca60) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 21 (Thread 0x41330950 (LWP 32353)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x00000000004d6a68 in __ast_cond_wait (filename=0x5ab7f4 "logger.c", lineno=981, func=0x5ac639 "logger_thread", cond_name=0x5ac667 "&logcond", mutex_name=0x5ac659 "&logmsgs.lock", cond=0x840860, t=0x811010) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x00000000004d6176 in logger_thread (data=0x0) at logger.c:981 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe00008c0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 20 (Thread 0x413ac950 (LWP 32354)): #0 0x00007fcbe895aa1b in read () from /lib/libc.so.6 #1 0x000000000053ed6f in inotify_daemon (data=0x0) at stdtime/localtime.c:259 #2 0x0000000000559f84 in dummy_start (data=0x1c7ed50) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 19 (Thread 0x41d2c950 (LWP 32355)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x000000000055a302 in ast_wait_for_input (fd=9, ms=5000) at utils.c:1057 #2 0x0000000000549fbd in ast_tcptls_server_root (data=0x8193c0) at tcptls.c:248 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe0033420) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 18 (Thread 0x41da8950 (LWP 32356)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x000000000048e0dc in __ast_cond_wait (filename=0x59e314 "devicestate.c", lineno=543, func=0x59e920 "do_devstate_changes", cond_name=0x59e755 "&change_pending", mutex_name=0x59e934 "&state_changes.lock", cond=0x83a160, t=0x801830) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x000000000048db87 in do_devstate_changes (data=0x0) at devicestate.c:543 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe003d7f0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 17 (Thread 0x40825950 (LWP 32357)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x0000000000547dfd in __ast_cond_wait (filename=0x5c0ba4 "taskprocessor.c", lineno=293, func=0x5c1150 "tps_processing_function", cond_name=0x5c11c9 "&i->poll_cond", mutex_name=0x5c11b2 "&i->taskprocessor_lock", cond=0x1c8b6c0, t=0x1c8b6f8) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x000000000054774e in tps_processing_function (data=0x1c8b6b8) at taskprocessor.c:293 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe003d860) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 16 (Thread 0x40e91950 (LWP 32358)): #0 0x00007fcbe8961d52 in select () from /lib/libc.so.6 #1 0x00000000004b034b in ast_select (nfds=0, rfds=0x40e90f90, wfds=0x0, efds=0x40e90f10, tvp=0x0) at /usr/src/asterisk-1.6.2.10/include/asterisk/channel.h:1746 #2 0x00000000004b030b in do_parking_thread (ignore=0x0) at features.c:3231 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe030c450) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 15 (Thread 0x40d64950 (LWP 32359)): #0 0x00007fcbe8961d52 in select () from /lib/libc.so.6 #1 0x00007fcbe59e5b5d in agent_check_and_process () from /usr/lib/libnetsnmpagent.so.15 #2 0x00007fcbe5f6d9bd in agent_thread (arg=0x0) at snmp/agent.c:202 #3 0x0000000000559f84 in dummy_start (data=0x1c934b0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 14 (Thread 0x41f9d950 (LWP 32360)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x00007fcbcb380d12 in __ast_cond_wait (filename=0x7fcbcb381414 "res_timing_pthread.c", lineno=465, func=0x7fcbcb38182c "do_timing", cond_name=0x7fcbcb381476 "&timing_thread.cond", mutex_name=0x7fcbcb381462 "&timing_thread.lock", cond=0x7fcbcb583210, t=0x7fcbcb582648) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x00007fcbcb38005f in do_timing (arg=0x0) at res_timing_pthread.c:465 #3 0x0000000000559f84 in dummy_start (data=0x7fcbd80a68f0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 13 (Thread 0x41c65950 (LWP 32361)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x0000000000547dfd in __ast_cond_wait (filename=0x5c0ba4 "taskprocessor.c", lineno=293, func=0x5c1150 "tps_processing_function", cond_name=0x5c11c9 "&i->poll_cond", mutex_name=0x5c11b2 "&i->taskprocessor_lock", cond=0x1d1bba0, t=0x1d1bbd8) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x000000000054774e in tps_processing_function (data=0x1d1bb98) at taskprocessor.c:293 #3 0x0000000000559f84 in dummy_start (data=0x1cfd200) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 12 (Thread 0x40763950 (LWP 32362)): #0 0x00007fcbe8937071 in nanosleep () from /lib/libc.so.6 #1 0x00007fcbde9c7e23 in lock_broker (unused=0x0) at func_lock.c:192 #2 0x0000000000559f84 in dummy_start (data=0x7fcbd80c9ee0) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 11 (Thread 0x40b30950 (LWP 32363)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x00000000004cc9a1 in ast_io_wait (ioc=0x7fcbe0485980, howlong=1000) at io.c:275 #2 0x00007fcbd476ecc2 in do_monitor (data=0x0) at chan_mgcp.c:3462 #3 0x0000000000559f84 in dummy_start (data=0x7fcbe0475e00) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 10 (Thread 0x41074950 (LWP 32364)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x00000000004cc9a1 in ast_io_wait (ioc=0x7fcbd80ceb30, howlong=1000) at io.c:275 #2 0x00007fcbce82af08 in network_thread (ignore=0x0) at pbx_dundi.c:2179 #3 0x0000000000559f84 in dummy_start (data=0x7fcbd80cd8c0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 9 (Thread 0x410f0950 (LWP 32365)): #0 0x00007fcbe8937071 in nanosleep () from /lib/libc.so.6 #1 0x00007fcbe8936e97 in sleep () from /lib/libc.so.6 #2 0x00007fcbce82b268 in process_precache (ign=0x0) at pbx_dundi.c:2256 #3 0x0000000000559f84 in dummy_start (data=0x7fcbd80c7530) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 8 (Thread 0x41428950 (LWP 32366)): #0 0x00007fcbe8937071 in nanosleep () from /lib/libc.so.6 #1 0x00007fcbe8936e97 in sleep () from /lib/libc.so.6 #2 0x00007fcbce82b0e4 in process_clearcache (ignore=0x0) at pbx_dundi.c:2219 #3 0x0000000000559f84 in dummy_start (data=0x7fcbd80cd8c0) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 7 (Thread 0x414a4950 (LWP 32367)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x00007fcbd2d1c83d in do_monitor (data=0x0) at chan_dahdi.c:9665 #2 0x0000000000559f84 in dummy_start (data=0x7fcbe04afdb0) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 6 (Thread 0x4157c950 (LWP 32368)): #0 0x00007fcbe8961d52 in select () from /lib/libc.so.6 #1 0x00007fcbdf7ef9b1 in ast_select (nfds=0, rfds=0x4157bfa0, wfds=0x0, efds=0x4157bf20, tvp=0x0) at /usr/src/asterisk-1.6.2.10/include/asterisk/channel.h:1746 #2 0x00007fcbdf7ef742 in do_monitor (data=0x0) at chan_phone.c:1075 #3 0x0000000000559f84 in dummy_start (data=0x1d46230) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 5 (Thread 0x409bf950 (LWP 32369)): #0 0x00007fcbe7628d2b in ?? () from /lib/libgcc_s.so.1 #1 0x00007fcbe7628ea9 in ?? () from /lib/libgcc_s.so.1 #2 0x00007fcbe7629abd in _Unwind_Backtrace () from /lib/libgcc_s.so.1 #3 0x00007fcbe897964e in backtrace () from /lib/libc.so.6 #4 0x00000000004d7c87 in ast_bt_get_addresses (bt=0x7fcbe071ae50) at logger.c:1203 #5 0x0000000000444b39 in __ast_pthread_mutex_trylock (filename=0x7fcbc706ecd4 "chan_sip.c", lineno=15068, func=0x7fcbc70786a0 "dialog_needdestroy", mutex_name=0x7fcbc706ed67 "dialog", t=0x7fcbe071ad30) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:630 #6 0x0000000000444d64 in ?? () #7 0x00007fcbc70352b7 in dialog_needdestroy (dialogobj=0x7fcbe071b918, arg=0x409bf050, flags=7) at chan_sip.c:15068 #8 0x000000000044604d in __ao2_callback (c=0x1fc5f88, flags=7, cb_fn=0x7fcbc7035271, arg=0x409bf050, data=0x0, type=DEFAULT, tag=0x0, file=0x0, line=0, funcname=0x0) at astobj2.c:712 #9 0x000000000044639e in ?? () #10 0x00007fcbc705973e in do_monitor (data=0x0) at chan_sip.c:22569 #11 0x0000000000559f84 in dummy_start (data=0x2820920) at utils.c:968 #12 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #13 0x00007fcbe896864d in clone () from /lib/libc.so.6 #14 0x0000000000000000 in ?? () Thread 4 (Thread 0x40a3b950 (LWP 32370)): #0 0x00007fcbe8127d29 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0 #1 0x0000000000547dfd in __ast_cond_wait (filename=0x5c0ba4 "taskprocessor.c", lineno=293, func=0x5c1150 "tps_processing_function", cond_name=0x5c11c9 "&i->poll_cond", mutex_name=0x5c11b2 "&i->taskprocessor_lock", cond=0x283d880, t=0x283d8b8) at /usr/src/asterisk-1.6.2.10/include/asterisk/lock.h:823 #2 0x000000000054774e in tps_processing_function (data=0x283d878) at taskprocessor.c:293 #3 0x0000000000559f84 in dummy_start (data=0x2821690) at utils.c:968 #4 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #5 0x00007fcbe896864d in clone () from /lib/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x40bf5950 (LWP 32371)): #0 0x00007fcbe8937071 in nanosleep () from /lib/libc.so.6 #1 0x00007fcbc82cbe4d in scan_thread (unused=0x0) at pbx_spool.c:452 #2 0x0000000000559f84 in dummy_start (data=0x2840d30) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 2 (Thread 0x408a1950 (LWP 32372)): #0 0x00007fcbe895fbd6 in poll () from /lib/libc.so.6 #1 0x000000000044183d in monitor_sig_flags (unused=0x0) at asterisk.c:3053 #2 0x0000000000559f84 in dummy_start (data=0x2848a60) at utils.c:968 #3 0x00007fcbe8123fc7 in start_thread () from /lib/libpthread.so.0 #4 0x00007fcbe896864d in clone () from /lib/libc.so.6 #5 0x0000000000000000 in ?? () Thread 1 (Thread 0x7fcbe93e56e0 (LWP 14775)): #0 0x00007fcbe895aa1b in read () from /lib/libc.so.6 #1 0x000000000056a93e in read_char (el=0x1c66f30, cp=0x7ffff13f25ef "") at read.c:298 #2 0x00000000005658f6 in ?? () #3 0x0000000000565b47 in ?? () #4 0x0000000000443c4d in ?? () #5 0x00007fcbe88b71a6 in __libc_start_main () from /lib/libc.so.6 #6 0x000000000041e2f9 in ?? () #7 0x00007ffff13f3c28 in ?? () #8 0x000000000000001c in ?? () #9 0x0000000000000008 in ?? () #10 0x00007ffff13f4a4d in ?? () #11 0x00007ffff13f4a60 in ?? () #12 0x00007ffff13f4a63 in ?? () #13 0x00007ffff13f4a67 in ?? () #14 0x00007ffff13f4a6a in ?? () #15 0x00007ffff13f4a6d in ?? () #16 0x00007ffff13f4a72 in ?? () #17 0x00007ffff13f4a78 in ?? () #18 0x0000000000000000 in ?? () #0 0x00007fcbe895aa1b in read () from /lib/libc.so.6  ======================================================================= === Currently Held Locks ============================================== ======================================================================= === === (): (times locked) === === Thread ID: 1083963728 (do_monitor started at [22618] chan_sip.c restart_monitor()) === ---> Lock #0 (astobj2.c): MUTEX 164 ao2_lock &p->priv_data.lock 0x1fc53a0 (1) /usr/sbin/asterisk(ast_bt_get_addresses+0x1a) [0x4d7c87] /usr/sbin/asterisk [0x44401f] /usr/sbin/asterisk(ao2_lock+0x49) [0x443ebb] /usr/sbin/asterisk [0x445f20] /usr/sbin/asterisk(_ao2_callback+0x59) [0x44639e] /usr/lib/asterisk/modules/chan_sip.so [0x7fcbc705973e] /usr/sbin/asterisk [0x559f84] /lib/libpthread.so.0 [0x7fcbe8123fc7] /lib/libc.so.6(clone+0x6d) [0x7fcbe896864d] === ------------------------------------------------------------------- === =======================================================================