shutdown/restart PBX 9/18/17 @ 6:19 PM ET top - 18:19:19 up 0 min, 2 users, load average: 1.28, 0.34, 0.12 Tasks: 114 total, 1 running, 113 sleeping, 0 stopped, 0 zombie %Cpu(s): 2.0 us, 1.7 sy, 0.0 ni, 96.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.3 st KiB Mem : 1883708 total, 923376 free, 478744 used, 481588 buff/cache KiB Swap: 2097148 total, 2097148 free, 0 used. 1183220 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 957 asterisk 20 0 649224 25108 11708 S 1.7 1.3 0:00.62 httpd 785 mongodb 20 0 477776 49696 20448 S 0.3 2.6 0:00.11 mongod 1043 mysql 20 0 1171144 108728 9160 S 0.3 5.8 0:00.75 mysqld 2751 asterisk 20 0 1613444 72104 16980 S 0.3 3.8 0:04.01 asterisk 2895 asterisk 20 0 921988 28708 11412 S 0.3 1.5 0:00.58 PM2 v2.6.1: God 3366 root 20 0 157700 2252 1584 R 0.3 0.1 0:00.02 top 1 root 20 0 43292 3700 2416 S 0.0 0.2 0:00.87 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd 3 root 20 0 0 0 0 S 0.0 0.0 0:00.01 ksoftirqd/0 4 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0 5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H 6 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/u2:0 7 root rt 0 0 0 0 S 0.0 0.0 0:00.00 migration/0 8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh 9 root 20 0 0 0 0 S 0.0 0.0 0:00.12 rcu_sched 10 root rt 0 0 0 0 S 0.0 0.0 0:00.00 watchdog/0 12 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs 13 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns 14 root 20 0 0 0 0 S 0.0 0.0 0:00.00 khungtaskd 15 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback 16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd 17 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 18 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd 19 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md 20 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/0:1 25 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kswapd0 26 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd 27 root 39 19 0 0 0 S 0.0 0.0 0:00.00 khugepagedpbx*CLI> core show taskprocessors Processor Processed In Queue Max Depth Low water High water app_voicemail 19 0 5 450 500 ast_msg_queue 0 0 0 450 500 CCSS_core 0 0 0 450 500 dns_system_resolver_tp 0 0 0 450 500 hep_queue_tp 0 0 0 450 500 iax2_transmit 0 0 0 450 500 pjsip/default-0000000d 6 0 1 450 500 pjsip/default-0000000e 5 0 1 450 500 pjsip/default-0000000f 5 0 1 450 500 pjsip/default-00000010 6 0 1 450 500 pjsip/default-00000011 5 0 1 450 500 pjsip/default-00000012 5 0 1 450 500 pjsip/default-00000013 7 0 1 450 500 pjsip/default-00000014 5 0 1 450 500 pjsip/distributor-00000022 4 0 2 450 500 pjsip/distributor-00000023 0 0 0 450 500 pjsip/distributor-00000024 0 0 0 450 500 pjsip/distributor-00000025 0 0 0 450 500 pjsip/distributor-00000026 0 0 0 450 500 pjsip/distributor-00000027 3 0 1 450 500 pjsip/distributor-00000028 0 0 0 450 500 pjsip/distributor-00000029 0 0 0 450 500 pjsip/distributor-0000002a 1 0 1 450 500 pjsip/distributor-0000002b 0 0 0 450 500 pjsip/distributor-0000002c 0 0 0 450 500 pjsip/distributor-0000002d 0 0 0 450 500 pjsip/distributor-0000002e 0 0 0 450 500 pjsip/distributor-0000002f 0 0 0 450 500 pjsip/distributor-00000030 0 0 0 450 500 pjsip/distributor-00000031 10 0 2 450 500 pjsip/distributor-00000032 2 0 1 450 500 pjsip/distributor-00000033 0 0 0 450 500 pjsip/distributor-00000034 2 0 1 450 500 pjsip/distributor-00000035 0 0 0 450 500 pjsip/distributor-00000036 4 0 2 450 500 pjsip/distributor-00000037 0 0 0 450 500 pjsip/distributor-00000038 0 0 0 450 500 pjsip/distributor-00000039 0 0 0 450 500 pjsip/distributor-0000003a 6 0 2 450 500 pjsip/distributor-0000003b 4 0 2 450 500 pjsip/distributor-0000003c 0 0 0 450 500 pjsip/distributor-0000003d 0 0 0 450 500 pjsip/distributor-0000003e 0 0 0 450 500 pjsip/distributor-0000003f 2 0 1 450 500 pjsip/distributor-00000040 0 0 0 450 500 pjsip/exten_state 0 0 0 450 500 pjsip/messaging 0 0 0 450 500 pjsip/mwi-00000048 12 0 1 450 500 pjsip/mwi-00000049 12 0 1 450 500 pjsip/mwi-0000004a 8 0 1 450 500 pjsip/mwi-0000004b 4 0 2 450 500 pjsip/mwi-0000004c 0 0 0 450 500 pjsip/mwi-0000004d 0 0 0 450 500 pjsip/mwi-0000004e 0 0 0 450 500 pjsip/mwi-0000004f 0 0 0 450 500 SIP 111 0 5 450 500 SIP-control 223 0 5 450 500 Sorcery 29 0 4 450 500 Sorcery-control 59 0 3 450 500 sorcery/acl-0000005f 0 0 0 450 500 sorcery/aor-0000001a 1 0 1 450 500 sorcery/asterisk-publication-00000054 0 0 0 450 500 sorcery/auth-00000015 0 0 0 450 500 sorcery/bucket-00000000 0 0 0 450 500 sorcery/contact-00000019 13 0 8 1350 1500 sorcery/contact_status-0000001b 29 0 8 1350 1500 sorcery/domain_alias-0000001c 0 0 0 450 500 sorcery/endpoint-00000016 1 0 1 450 500 sorcery/file-00000001 0 0 0 450 500 sorcery/global-0000001d 5 0 1 450 500 sorcery/identify-00000047 0 0 0 450 500 sorcery/inbound-publication-00000044 0 0 0 450 500 sorcery/log_mappings-0000000b 0 0 0 450 500 sorcery/nat_hook-00000017 0 0 0 450 500 sorcery/opus-0000000a 0 0 0 450 500 sorcery/outbound-publish-00000041 0 0 0 450 500 sorcery/registration-00000060 1 0 1 450 500 sorcery/resource_list-00000043 0 0 0 450 500 sorcery/subscription_persistence-00000042 0 0 0 450 500 sorcery/system-0000000c 0 0 0 450 500 sorcery/transport-00000018 0 0 0 450 500 stasis-core 100 0 7 450 500 stasis-core-control 206 0 7 450 500 subm:ast_bridge_topic_all-cached-00000063 1 0 1 450 500 subm:ast_channel_topic_all-00000069 2 0 1 450 500 subm:ast_channel_topic_all-0000006c 1 0 1 450 500 subm:ast_channel_topic_all-cached-00000062 5 0 2 450 500 subm:ast_channel_topic_all-cached-00000064 4 0 2 450 500 subm:ast_channel_topic_all-cached-00000068 3 0 2 450 500 subm:ast_device_state_topic-00000002 53 0 18 450 500 subm:ast_device_state_topic-00000004 52 0 18 450 500 subm:ast_device_state_topic-0000006b 5 0 2 450 500 subm:ast_parking-00000046 1 0 1 450 500 subm:ast_presence_state_topic_all-00000005 1 0 1 450 500 subm:ast_security-00000065 71 0 1 450 500 subm:ast_system-00000009 42 0 36 450 500 subm:ast_system-00000055 41 0 12 450 500 subm:ast_system-00000056 40 0 12 450 500 subm:ast_system-00000061 5 0 2 450 500 subm:cdr_engine-00000003 8 0 2 4500 5000 subm:cel_aggregation_topic-00000006 8 0 2 2700 3000 subm:endpoint_topic_all-cached-00000008 74 0 20 450 500 subm:endpoint_topic_all-cached-00000067 15 0 4 450 500 subm:manager_topic-00000007 178 0 12 2700 3000 subm:stasis_mwi_topic-00000066 16 0 1 450 500 subm:stasis_mwi_topic-0000006a 15 0 1 450 500 subp:701@default-00000058 2 0 1 450 500 subp:701@device-0000006f 1 0 1 450 500 subp:702@device-00000071 5 0 1 450 500 subp:702@device-00000072 4 0 1 450 500 subp:702@device-00000074 1 0 1 450 500 subp:703@default-0000005a 2 0 1 450 500 subp:703@device-00000052 1 0 1 450 500 subp:PJSIP/701-0000001e 11 0 4 450 500 subp:PJSIP/702-0000001f 15 0 4 450 500 subp:PJSIP/703-00000020 3 0 3 450 500 subp:PJSIP/dpma_endpoint-00000021 3 0 3 450 500 subp:SIP/99701-00000057 2 0 2 450 500 subp:SIP/99703-00000059 2 0 2 450 500 subp:SIP/TollFreeProxy-0000005b 4 0 2 450 500 subp:SIP/vitel-inbound-0000005c 4 0 2 450 500 subp:SIP/Vitelity-0000005d 4 0 2 450 500 subp:SIP/VoIPms-0000005e 4 0 2 450 500 123 taskprocessors pbx*CLI> top - 09:31:14 up 15:12, 2 users, load average: 0.41, 0.22, 0.26 Tasks: 112 total, 1 running, 111 sleeping, 0 stopped, 0 zombie %Cpu(s): 2.3 us, 1.7 sy, 0.0 ni, 95.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.3 st KiB Mem : 1883708 total, 190132 free, 930792 used, 762784 buff/cache KiB Swap: 2097148 total, 2095932 free, 1216 used. 639312 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 975 asterisk 20 0 762620 59488 19972 S 2.3 3.2 4:08.45 httpd 2751 asterisk 20 0 1619396 88196 17928 S 0.7 4.7 5:10.11 asterisk 1043 mysql 20 0 1173776 253188 9664 S 0.3 13.4 8:03.23 mysqld 1 root 20 0 43292 3700 2416 S 0.0 0.2 0:13.50 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd 3 root 20 0 0 0 0 S 0.0 0.0 0:01.07 ksoftirqd/0 5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H 7 root rt 0 0 0 0 S 0.0 0.0 0:00.00 migration/0 8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh 9 root 20 0 0 0 0 S 0.0 0.0 0:09.40 rcu_sched 10 root rt 0 0 0 0 S 0.0 0.0 0:00.28 watchdog/0 12 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs 13 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns 14 root 20 0 0 0 0 S 0.0 0.0 0:00.11 khungtaskd 15 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback 16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd 17 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 18 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd 19 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md 25 root 20 0 0 0 0 S 0.0 0.0 0:00.23 kswapd0 26 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd 27 root 39 19 0 0 0 S 0.0 0.0 0:00.57 khugepaged 28 root 20 0 0 0 0 S 0.0 0.0 0:00.00 fsnotify_mark 29 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto 37 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld 38 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/u2:1 39 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd 40 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kpsmoused 41 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 ipv6_addrconf 60 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 deferwq 93 root 20 0 0 0 0 S 0.0 0.0 0:00.38 kauditd 264 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 ata_sff <-------------> --- (11 headers 0 lines) --- Really destroying SIP dialog '0b3801ed58aac0421925cf1d24cbcdab@209.222.10.59:5160' Method: OPTIONS Really destroying SIP dialog '472287de29767a135a80749c2e8ea03b@107.6.67.236:5060' Method: OPTIONS [2017-09-19 09:29:46] ERROR[8527]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[16884]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[16884]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[12203]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[12203]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:46] ERROR[4736]: res_pjsip.c:3770 endpt_send_request: Error 70007 'Not enough memory (PJ_ENOMEM)' sending NOTIFY request to endpoint 701 [2017-09-19 09:29:47] ERROR[2795]: pjproject:0 : ssl0x7f8a7805bcb0 Renegotiation failed: Not enough memory (PJ_ENOMEM) pbx*CLI> pbx*CLI> core show taskprocessors Processor Processed In Queue Max Depth Low water High water app_voicemail 485 0 5 450 500 ast_msg_queue 0 0 0 450 500 CCSS_core 0 0 0 450 500 dns_system_resolver_tp 0 0 0 450 500 hep_queue_tp 0 0 0 450 500 iax2_transmit 0 0 0 450 500 pjsip/default-0000000d 1318 0 1 450 500 pjsip/default-0000000e 1315 0 1 450 500 pjsip/default-0000000f 1303 0 1 450 500 pjsip/default-00000010 1303 0 1 450 500 pjsip/default-00000011 1315 0 1 450 500 pjsip/default-00000012 1311 0 1 450 500 pjsip/default-00000013 1295 0 1 450 500 pjsip/default-00000014 1312 0 1 450 500 pjsip/distributor-00000022 436 0 2 450 500 pjsip/distributor-00000023 350 0 1 450 500 pjsip/distributor-00000024 370 0 1 450 500 pjsip/distributor-00000025 423 0 2 450 500 pjsip/distributor-00000026 372 0 2 450 500 pjsip/distributor-00000027 426 0 1 450 500 pjsip/distributor-00000028 417 0 1 450 500 pjsip/distributor-00000029 365 0 1 450 500 pjsip/distributor-0000002a 360 0 1 450 500 pjsip/distributor-0000002b 340 0 1 450 500 pjsip/distributor-0000002c 336 0 1 450 500 pjsip/distributor-0000002d 434 0 1 450 500 pjsip/distributor-0000002e 371 0 2 450 500 pjsip/distributor-0000002f 368 0 1 450 500 pjsip/distributor-00000030 352 0 1 450 500 pjsip/distributor-00000031 1015 0 2 450 500 pjsip/distributor-00000032 378 0 1 450 500 pjsip/distributor-00000033 328 0 1 450 500 pjsip/distributor-00000034 375 0 2 450 500 pjsip/distributor-00000035 328 0 1 450 500 pjsip/distributor-00000036 461 0 2 450 500 pjsip/distributor-00000037 384 0 1 450 500 pjsip/distributor-00000038 324 0 1 450 500 pjsip/distributor-00000039 346 0 1 450 500 pjsip/distributor-0000003a 389 0 2 450 500 pjsip/distributor-0000003b 368 0 2 450 500 pjsip/distributor-0000003c 357 0 1 450 500 pjsip/distributor-0000003d 338 0 1 450 500 pjsip/distributor-0000003e 354 0 1 450 500 pjsip/distributor-0000003f 372 0 1 450 500 pjsip/distributor-00000040 392 0 1 450 500 pjsip/exten_state 0 0 0 450 500 pjsip/messaging 0 0 0 450 500 pjsip/mwi-00000048 7690 0 21 450 500 pjsip/mwi-00000049 7565 0 20 450 500 pjsip/mwi-0000004a 7445 0 20 450 500 pjsip/mwi-0000004b 7329 0 20 450 500 pjsip/mwi-0000004c 7207 0 20 450 500 pjsip/mwi-0000004d 7083 0 20 450 500 pjsip/mwi-0000004e 6971 0 20 450 500 pjsip/mwi-0000004f 6869 0 20 450 500 SIP 47500 0 9 450 500 SIP-control 95772 0 10 450 500 Sorcery 11795 0 4 450 500 Sorcery-control 23690 0 3 450 500 sorcery/acl-0000005f 0 0 0 450 500 sorcery/aor-0000001a 1 0 1 450 500 sorcery/asterisk-publication-00000054 0 0 0 450 500 sorcery/auth-00000015 0 0 0 450 500 sorcery/bucket-00000000 0 0 0 450 500 sorcery/contact-00000019 266 0 8 1350 1500 sorcery/contact_status-0000001b 11556 0 8 1350 1500 sorcery/domain_alias-0000001c 0 0 0 450 500 sorcery/endpoint-00000016 1 0 1 450 500 sorcery/file-00000001 0 0 0 450 500 sorcery/global-0000001d 5 0 1 450 500 sorcery/identify-00000047 0 0 0 450 500 sorcery/inbound-publication-00000044 0 0 0 450 500 sorcery/log_mappings-0000000b 0 0 0 450 500 sorcery/nat_hook-00000017 0 0 0 450 500 sorcery/opus-0000000a 0 0 0 450 500 sorcery/outbound-publish-00000041 0 0 0 450 500 sorcery/registration-00000060 1 0 1 450 500 sorcery/resource_list-00000043 0 0 0 450 500 sorcery/subscription_persistence-00000042 0 0 0 450 500 sorcery/system-0000000c 0 0 0 450 500 sorcery/transport-00000018 0 0 0 450 500 stasis-core 2437 0 7 450 500 stasis-core-control 6166 0 8 450 500 subm:ast_bridge_topic_all-cached-00000063 67 0 7 450 500 subm:ast_channel_topic_all-00000069 1044 0 149 450 500 subm:ast_channel_topic_all-0000006c 1043 0 149 450 500 subm:ast_channel_topic_all-cached-00000062 1441 0 211 450 500 subm:ast_channel_topic_all-cached-00000064 1440 0 211 450 500 subm:ast_channel_topic_all-cached-00000068 1439 0 211 450 500 subm:ast_device_state_topic-00000002 88 0 18 450 500 subm:ast_device_state_topic-00000004 87 0 18 450 500 subm:ast_device_state_topic-0000006b 40 0 4 450 500 subm:ast_parking-00000046 1 0 1 450 500 subm:ast_presence_state_topic_all-00000005 1 0 1 450 500 subm:ast_security-00000065 92643 0 2 450 500 subm:ast_system-00000009 1318 0 36 450 500 subm:ast_system-00000055 1317 0 12 450 500 subm:ast_system-00000056 1316 0 12 450 500 subm:ast_system-00000061 1281 0 2 450 500 subm:cdr_engine-00000003 1522 0 212 4500 5000 subm:cel_aggregation_topic-00000006 1510 0 211 2700 3000 subm:endpoint_topic_all-cached-00000008 1846 0 211 450 500 subm:endpoint_topic_all-cached-00000067 1787 0 211 450 500 subm:manager_topic-00000007 96363 0 211 2700 3000 subm:stasis_mwi_topic-00000066 482 0 1 450 500 subm:stasis_mwi_topic-0000006a 481 0 1 450 500 subp:701@default-00000058 2 0 1 450 500 subp:701@device-0000006f 154 0 1 450 500 subp:701@device-00000076 151 0 1 450 500 subp:701@device-00000078 148 0 1 450 500 subp:701@device-0000015f 1 0 1 450 500 subp:702@device-00000071 318 0 1 450 500 subp:702@device-00000072 317 0 1 450 500 subp:702@device-00000074 314 0 1 450 500 subp:702@device-0000015c 1 0 1 450 500 subp:703@default-0000005a 2 0 1 450 500 subp:703@device-00000052 1 0 1 450 500 subp:PJSIP/701-0000001e 574 0 207 450 500 subp:PJSIP/702-0000001f 1053 0 211 450 500 subp:PJSIP/703-00000020 3 0 3 450 500 subp:PJSIP/dpma_endpoint-00000021 3 0 3 450 500 subp:SIP/99701-00000057 2 0 2 450 500 subp:SIP/99703-00000059 2 0 2 450 500 subp:SIP/TollFreeProxy-0000005b 4 0 2 450 500 subp:SIP/vitel-inbound-0000005c 8 0 2 450 500 subp:SIP/Vitelity-0000005d 8 0 2 450 500 subp:SIP/VoIPms-0000005e 151 0 19 450 500 127 taskprocessors