Index: main/asterisk.c =================================================================== --- main/asterisk.c (revision 93089) +++ main/asterisk.c (working copy) @@ -231,6 +231,8 @@ static int shuttingdown; static int restartnow; static pthread_t consolethread = AST_PTHREADT_NULL; +static int canary_pid = 0; +static char canary_filename[128]; static char randompool[256]; @@ -2621,6 +2623,35 @@ return NULL; } +static void *canary_thread(void *unused) +{ + struct stat canary_stat; + struct timeval tv; + int dead = 0; + + /* Give the canary time to sing */ + sleep(120); + + for (;;) { + stat(canary_filename, &canary_stat); + tv = ast_tvnow(); + if (!dead && tv.tv_sec > canary_stat.st_mtime + 10) { + ast_log(LOG_WARNING, "Canary is dead!!! Reducing priority\n"); + dead = 1; + ast_set_priority(0); + } + + /* Check the canary once a minute */ + sleep(60); + } +} + +static void canary_exit(void) +{ + if (canary_pid > 0) + kill(canary_pid, SIGKILL); +} + int main(int argc, char *argv[]) { int c; @@ -2804,11 +2835,46 @@ if ((!runuser) && !ast_strlen_zero(ast_config_AST_RUN_USER)) runuser = ast_config_AST_RUN_USER; + /* Must install this signal handler up here to ensure that if the canary + * fails to execute that it doesn't kill the Asterisk process. + */ + signal(SIGCHLD, child_handler); #ifndef __CYGWIN__ - if (isroot) + if (isroot) { ast_set_priority(ast_opt_high_priority); + if (ast_opt_high_priority) { + snprintf(canary_filename, sizeof(canary_filename), "%s/canary", ast_config_AST_RUN_DIR); + canary_pid = fork(); + if (canary_pid == 0) { + char canary_binary[128], *lastslash; + int fd; + + for (fd = 0; fd < 100; fd++) + close(fd); + + execlp("astcanary", "astcanary", canary_filename, NULL); + + /* If not found, try the same path as used to execute asterisk */ + ast_copy_string(canary_binary, argv[0], sizeof(canary_binary)); + if ((lastslash = strrchr(canary_binary, '/'))) { + ast_copy_string(lastslash + 1, "astcanary", sizeof(canary_binary) + canary_binary - (lastslash + 1)); + execl(canary_binary, "astcanary", canary_filename, NULL); + } + + /* Should never happen */ + _exit(1); + } else if (canary_pid > 0) { + pthread_t dont_care; + ast_pthread_create_detached(&dont_care, NULL, canary_thread, NULL); + } + + /* Kill the canary when we exit */ + atexit(canary_exit); + } + } + if (isroot && rungroup) { struct group *gr; gr = getgrnam(rungroup); @@ -2971,7 +3037,6 @@ signal(SIGINT, __quit_handler); signal(SIGTERM, __quit_handler); signal(SIGHUP, hup_handler); - signal(SIGCHLD, child_handler); signal(SIGPIPE, SIG_IGN); /* ensure that the random number generators are seeded with a different value every time Index: utils/astcanary.c =================================================================== --- utils/astcanary.c (revision 0) +++ utils/astcanary.c (revision 0) @@ -0,0 +1,79 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2007, Digium, Inc. + * + * Tilghman Lesher + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +#include +#include +#include +#include +#include +#include +#include + +/*!\brief + * At one time, canaries were carried along with coal miners down + * into a mine. Their purpose was to alert the miners when they + * had drilled into a pocket of methane gas or another noxious + * substance. The canary, being the most sensitive animal would + * immediately fall over. Seeing this, the miners could take + * action to escape the mine, seeing an imminent danger. + * + * This process serves a similar purpose, though with the realtime + * priority being the reason. When a thread starts running away + * with the processor, it is typically difficult to tell what + * thread caused the problem, as the machine acts as if it is + * locked up (in fact, what has happened is that Asterisk runs at + * a higher priority than even the login shell, so the runaway + * thread hogs all available CPU time. + * + * If that happens, this canary process will cease to get any + * process time, which we can monitor with a realtime thread in + * Asterisk. Should that happen, that monitoring thread may take + * immediate action to slow down Asterisk to regular priority, + * thus allowing an administrator to login to the system and + * restart Asterisk or perhaps take another course of action + * (such as retrieving a backtrace to let the developers know + * what precisely went wrong). + * + * Note that according to POSIX.1, all threads inside a single + * process must share the same priority, so when the monitoring + * thread deprioritizes itself, it deprioritizes all threads at + * the same time. This is also why this canary must exist as a + * completely separate process and not simply as a thread within + * Asterisk itself. + */ + +int main(int argc, char *argv[]) +{ + int fd; + /* Run at normal priority */ + setpriority(PRIO_PROCESS, 0, 0); + for (; /* ever */ ;) { + /* Make sure the file exists */ + if ((fd = open(argv[1], O_RDWR | O_TRUNC | O_CREAT)) > 0) + close(fd); + + /* Update the modification times (checked from Asterisk) */ + utime(argv[1], NULL); + + /* Run occasionally */ + sleep(5); + } + /* Never reached */ + return 0; +} + Index: utils/Makefile =================================================================== --- utils/Makefile (revision 93089) +++ utils/Makefile (working copy) @@ -17,7 +17,7 @@ .PHONY: clean all uninstall # to get check_expr, add it to the ALL_UTILS list -ALL_UTILS:=astman smsq stereorize streamplayer aelparse muted check_expr conf2ael hashtest2 hashtest +ALL_UTILS:=astman smsq stereorize streamplayer aelparse muted check_expr conf2ael hashtest2 hashtest astcanary UTILS:=$(ALL_UTILS) LIBS += $(BKTR_LIB) # astobj2 with devmode uses backtrace