nagios4/test/test-downtime.pl

#!/usr/bin/perl -w

# Script to test downtime in Nagios with and without restarts

use English;
use Getopt::Long;
use Algorithm::Permute;

my $testPaused = 0;

my @SLEEPTICKS = ( "-", "\\", "|", "/", "-", "\\", "|", "/");
my $USESLEEPTICKS = 0;

my $maxStatusLevel = 3;

sub statusString {
	my $level = shift;
	my $message = shift;

	return if( $level > $maxStatusLevel);
	return sprintf( "    %s %s", humanTime( time), $message);
}

# Read the contents of a .dat file such as status.dat or retention.dat
sub readDatFile {
	my $file = shift;

	# Read the entire file into a variable
	my $oldrs = $RS;
	$RS = undef;
	open( FILE, "$file") || die "Unable to open $file for reading";
	my $contents = <FILE>;
	close( FILE) || die "Unable to close $file";
	$RS = $oldrs;

	# Strip comment and blank lines
	$contents =~ s/#.*\n//g;
	$contents =~ s/^\s*\n//g;
	$contents =~ s/\n\s*\n/\n/g;

	# Break into sections
	my @sections = split( /}\n/, $contents);

	my %contents;

	# Parse each section
	for( my $x = 0; $x < @sections; $x++) {
		my @lines = split( /\n/, $sections[ $x]);

		# Get the section type
		chomp $lines[ 0];
		$lines[ 0] =~ s/^\s*//g;
		$lines[ 0] =~ s/\s+{\s*$//g;

		# Create a hash of the remaining lines
		my $href = {};
		for( my $y = 1; $y < @lines; $y++) {
			next if( $lines[ $y] =~ /^\s*$/);
			chomp( $lines[ $y]);
			$lines[ $y] =~ s/^\s*//;
			$lines[ $y] =~ s/\s*$//;
			die "Unknown format for line: " . $lines[ $y] unless( $lines[ $y] =~ /^([^=]+)=(.*)$/);
			my( $key, $value) = ( $1, $2);
			$href->{ $key} = $value;
		}

		# Enter stuff into the contents hash
		if( $lines[ 0] =~ /^(info|program|programstatus)$/) {
			$contents{ $lines[ 0]} = $href;
		}
		elsif( $lines[ 0] =~ /^(contact|contactstatus|host|hoststatus|service|servicecomment|servicedowntime|servicestatus)$/) {
			$contents{ $lines[ 0]} = [] unless( exists( $contents{ $lines[ 0]}));
			push( @{ $contents{ $lines[ 0]}}, $href);
		}
		else {
			die "Unanticipated section encountered: " . $lines[ 0];
		}
	}

	return \%contents;
}

# Read the contents of the objects.cache file
sub readObjectCache {
	my $file = shift;

	# Read the entire file into a variable
	my $oldrs = $RS;
	$RS = undef;
	open( FILE, "$file") || die "Unable to open $file for reading";
	my $contents = <FILE>;
	close( FILE) || die "Unable to close $file";
	$RS = $oldrs;

	# Strip comment and blank lines
	$contents =~ s/#.*\n//g;
	$contents =~ s/^\s*\n//g;
	$contents =~ s/\n\s*\n/\n/g;

	# Break into sections
	my @sections = split( /}\n/, $contents);

	my %contents;

	# Parse each definition
	for( my $x = 0; $x < @sections; $x++) {
		my @lines = split( /\n/, $sections[ $x]);

		# Get the definition type
		chomp $lines[ 0];
		my $definitionType = undef;
		if( $lines[ 0] =~ /define\s+(\S+)\s*{/) {
			$definitionType = $1;
		}
		die "Unknown definition line: " . $lines[ 0] if( !defined( $definitionType));

		# Create a hash of the remaining lines
		my $href = {};
		for( my $y = 1; $y < @lines; $y++) {
			chomp( $lines[ $y]);
			next if( $lines[ $y] =~ /^\s*$/);
			die "Unknown format for line: " . $lines[ $y] unless( $lines[ $y] =~ /^\s*(\S+)\s+(.*)\s*$/);
			my( $key, $value) = ( $1, $2);
			$href->{ $key} = $value;
		}

		# Enter stuff into the contents hash
		$contents{ $definitionType} = [] unless( exists( $contents{ $definitionType}));
		push( @{ $contents{ $definitionType}}, $href);
	}

	return \%contents;
}

# Read the contents of the nagios.cfg file.
# NOTE: This function does not recursively read the contents of configuration
# files specified in the cfg_file and cfg_dir directives.
sub readCfgFile {
	my $file = shift;

	my %cfg;

	open( FILE, "$file") || die "Unable to open $file for reading";
	while( <FILE>) {
		chomp;
		next if( /^\s*#/);
		next if( /^\s*$/);
		if( /^([^=]+)\s*=\s*(.*)$/) {
			my( $key, $value) = ( $1, $2);
			if( $key =~ /cfg_file|cfg_dir/) {
				$cfg{ $key} = [] unless( exists( $cfg{ $key}));
				push( @{ $cfg{ $key}}, $value);
			}
			else {
				die "Parameter $key already exists" if( exists( $cfg{ $key}));
				$cfg{ $key} = $value;
			}
		}
		else {
			die "Unknown formatting for line: $_";
		}
	}

	return \%cfg;
}

# Send a command to Nagios to schedule downtime
sub scheduleDowntime {
	my $service = shift;
	my $start = shift;
	my $end = shift;
	my $fixed = shift;
	my $duration = shift;
	my $comment = shift;

	my $cmd = sprintf( "%s;%s;%s;%lu;%lu;%d;%u;%u;%s;%s", 
			"SCHEDULE_SVC_DOWNTIME", $service->{ "host_name"}, 
			$service->{ "service_description"}, $start, $end, $fixed, 
			0, $duration, "Downtime Testing", $comment);

	my $cmdfile = "/usr/local/nagios/var/rw/nagios.cmd";
	open( CMD, ">$cmdfile") || die "Cannot open $cmdfile for writing";
	print CMD sprintf( "[%lu] %s\n", time, $cmd);
	close( CMD) || die "Unable to close $cmdfile";
	sleep 1;
}

# Send a command to Nagios to delete downtime
sub deleteDowntime {
	my $id = shift;
	my $cfg = shift;
	my $service = shift;

	# There is a bug in Nagios that currently causes Nagios to crash when 
	# deleting a downtime. Instead of deleting the downtime, we'll just 
	# have to wait until it ends.
	my $datfile = readDatFile( $cfg->{ "status_file"});
	my $downtimes = findServiceDowntimes( $datfile, $service);
	while( scalar( @$downtimes) > 0) {
		sleep 1;
		$datfile = readDatFile( $cfg->{ "status_file"});
		$downtimes = findServiceDowntimes( $datfile, $service);
	}
	return;
	
	my $cmd = sprintf( "%s;%u", "DEL_SVC_DOWNTIME", $id);
	my $cmdfile = "/usr/local/nagios/var/rw/nagios.cmd";
	open( CMD, ">$cmdfile") || die "Cannot open $cmdfile for writing";
	print CMD sprintf( "[%lu] %s\n", time, $cmd);
	close( CMD) || die "Unable to close $cmdfile";
}

# Send a command to Nagios to process passive service check results
sub sendPassiveResults {
	my $service = shift;
	my $returnCode = shift;
	my $pluginOutput = shift;

	my $cmd = sprintf( "%s;%s;%s;%u;%s", "PROCESS_SERVICE_CHECK_RESULT", 
			$service->{ "host_name"}, $service->{ "service_description"}, 
			$returnCode, $pluginOutput);

	my $cmdfile = "/usr/local/nagios/var/rw/nagios.cmd";
	open( CMD, ">$cmdfile") || die "Cannot open $cmdfile for writing";
	print CMD sprintf( "[%lu] %s\n", time, $cmd);
	close( CMD) || die "Unable to close $cmdfile";
}

# Determine the amount of time to discover a service failure above
# the normal check results reaper frequency
sub maxDiscoveryTime {
	my $cfg = shift;
	my $service = shift;
	my $versionMajor = shift;

	my $maxDiscoveryTime = 0;
	if( $service->{ "active_checks_enabled"}) {
		$maxDiscoveryTime += ( $service->{ "check_interval"} * 
				$cfg->{ "interval_length"});
		$maxDiscoveryTime += ((( $service->{ "retry_interval"} + 
				( $versionMajor < 4 ? 
				$cfg->{ "check_result_reaper_frequency"} : 0)) * 
				$cfg->{ "interval_length"}) * 
				( $service->{ "max_check_attempts"} - 1));
	}
	return $maxDiscoveryTime;
}

# Return information about a particular service recorded in a .dat file
sub findServiceStatus {
	my $datfile = shift;
	my $service = shift;

	my $found;

	if( exists( $datfile->{ "servicestatus"})) {
		for( my $x = 0; $x < @{ $datfile->{ "servicestatus"}}; $x++) {
			if(( $datfile->{ "servicestatus"}->[ $x]->{ "host_name"} eq 
					$service->{ "host_name"}) &&
					( $datfile->{ "servicestatus"}->[ $x]->{ "service_description"} eq 
					$service->{ "service_description"})) {
				return $datfile->{ "servicestatus"}->[ $x];
			}
		}
	}

	return \$found;
}

sub findServiceNextCheck {
	my $cfg = shift;
	my $service = shift;

	my $datfile = readDatFile( $cfg->{ "status_file"});
	my $serviceStatus = findServiceStatus( $datfile, $service);
	return $serviceStatus->{ "next_check"};
}

# Return information about downtimes for a particular service recorded in a 
# .dat file
sub findServiceDowntimes {
	my $datfile = shift;
	my $service = shift;

	my @found;

	if( exists( $datfile->{ "servicedowntime"})) {
		for( my $x = 0; $x < @{ $datfile->{ "servicedowntime"}}; $x++) {
			if(( $datfile->{ "servicedowntime"}->[ $x]->{ "host_name"} eq 
					$service->{ "host_name"}) &&
					( $datfile->{ "servicedowntime"}->[ $x]->{ "service_description"} eq 
					$service->{ "service_description"})) {
				push( @found, $datfile->{ "servicedowntime"}->[ $x]);
			}
		}
	}

	return \@found;
}

# Fine a particular service in a config (.cfg) file or the objects cache.
sub findService {
	my $cfgfile = shift;
	my $host = shift;
	my $service = shift;

	if( exists( $cfgfile->{ "service"})) {
		for( my $x = 0; $x < @{ $cfgfile->{ "service"}}; $x++) {
			if(( $cfgfile->{ "service"}->[ $x]->{ "host_name"} eq $host) &&
					( $cfgfile->{ "service"}->[ $x]->{ "service_description"} eq $service)) {
				return $cfgfile->{ "service"}->[ $x];
			}
		}
	}

	return undef;
}

# Sleep for a given amount of time displaying a message while sleeping
sub noisySleep {
	my $length = shift;
	my $message = shift;

	print $message;
	print "  " if( $USESLEEPTICKS);
	for( my $x = 0; $x < $length; $x++) {
		if( $USESLEEPTICKS) {
			printf( "\b%s", $SLEEPTICKS[ $x % scalar( @SLEEPTICKS)]);
		}
		else {
			print ".";
		}
		sleep 1;
	}
	print "\b " if( $USESLEEPTICKS);
	print "\n";
}

# Sleep until a given time displaying a message while sleeping
sub noisySleepUntilTime {
	my $time = shift;
	my $message = shift;

	print $message;
	print "  " if( $USESLEEPTICKS);
	while( time < $time) {
		if( $USESLEEPTICKS) {
			printf( "\b%s", $SLEEPTICKS[ time % scalar( @SLEEPTICKS)]);
		}
		else {
			print ".";
		}
		sleep 1;
	}
	print "\b " if( $USESLEEPTICKS);
	print "\n";
}

# Sleep until a specified file is updated, displaying a message while sleeping
sub noisySleepUntilFileUpdate {
	my $file = shift;		# File whose mtime should be watched
	my $after = shift;		# Time after which file must be updated
	my $maxtime = shift;	# Maximum number of seconds after $after to wait
	my $message = shift;	# Message to display

	print $message;
	print "  " if( $USESLEEPTICKS);
	while(( ! -f $file) && ( time < ( $after + $maxtime))) {
		if( $USESLEEPTICKS) {
			printf( "\b%s", $SLEEPTICKS[ time % scalar( @SLEEPTICKS)]);
		}
		else {
			print ".";
		}
		sleep 1;
	}
	my ( $dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, 
			$ctime, $blksize, $blocks) = stat( $file);
	while(( $mtime <= $after) && ( time < ( $after + $maxtime))) {
		if( $USESLEEPTICKS) {
			printf( "\b%s", $SLEEPTICKS[ time % scalar( @SLEEPTICKS)]);
		}
		else {
			print ".";
		}
		sleep 1;
		( $dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, 
				$ctime, $blksize, $blocks) = stat( $file);
	}
	print "\b " if( $USESLEEPTICKS);
	print "\n";

	if( time >= ( $after + $maxtime)) {
		die "Maximum time exceeded waiting for $file to update\n";
	}
}

# Verify that a downtime is recorded correctly in a .dat file
sub verifyDowntime {
	my $service = shift;
	my $cfg = shift;
	my $downtimeFixed = shift;
	my $downtimeStart = shift;
	my $downtimeEnd = shift;
	my $downtimeDuration = shift;

	# Grab the status data
	print statusString( 2, "Reading status data.\n");
	my $datfile = readDatFile( $cfg->{ "status_file"});

	# Find the downtimes
	print statusString( 2, "Looking for downtimes.\n");
	my $downtimes = findServiceDowntimes( $datfile, $service);

	# Check to see if we found the correct number of downtimes (1)
	print statusString( 2, "Verifying the downtime.\n");
	if( scalar( @$downtimes) != 1) {
		print STDERR "Incorrect number of downtimes found: " . 
				scalar( @downtimes) . ".\n";
		return 0;
	}

	# Verify that the aspects of the downtime match
	if( $downtimes->[ 0]->{ "fixed"} != $downtimeFixed) {
		print STDERR sprintf( "Downtime type incorrect: should be %s, is %s.\n",
				( $downtimes->[ 0]->{ "fixed"} ? "Fixed" : "Flexible"),
				( $downtimeFixed ? "Fixed" : "Flexible"));
		return 0;
	}
	if( $downtimes->[ 0]->{ "start_time"} != $downtimeStart) {
		print STDERR sprintf( "Downtime start time incorrect: should be %lu, is %lu.\n",
				$downtimes->[ 0]->{ "start_time"}, $downtimeStart);
		return 0;
	}
	if( $downtimes->[ 0]->{ "end_time"} != $downtimeEnd) {
		print STDERR sprintf( "Downtime end time incorrect: should be %lu, is %lu.\n",
				$downtimes->[ 0]->{ "end_time"}, $downtimeEnd);
		return 0;
	}
	if( ! $downtimeFixed && ( $downtimes->[ 0]->{ "duration"} != 
			$downtimeDuration)) {
		print STDERR sprintf( "Downtime duration incorrect: should be %lu, is %lu.\n",
				$downtimes->[ 0]->{ "duration"}, $downtimeDuration);
		return 0;
	}
	return 1;
}

sub recoverService {
	my $service = shift;
	my $cfg = shift;
	my $message = shift;
	my $nagiosRunning = shift;
	my $versionMajor = shift;

	print statusString( 1, sprintf( "%s\n", $message));
	if( $service->{ "active_checks_enabled"}) {
		if(( $service->{ "host_name"} eq "localhost") && 
				( $service->{ "service_description"} eq "HTTP")) {
			system( "/sbin/service httpd start");
		}
		elsif(( $service->{ "host_name"} eq "localhost") && 
				( $service->{ "service_description"} eq "SSH")) {
			system( "/sbin/service sshd start");
		}
		else {
			die sprintf( "Don't know how to start service %s:%s",
					$service->{ "host_name"}, 
					$service->{ "service_description"});
		}
	}
	else {
		die "Nagios is not running" unless( $nagiosRunning);
		sendPassiveResults( $service, 0, "OK - Everything Okey Dokey");
	}

	# If Nagios is running, sleep so an undetected recovery doesn't 
	# interfere with future operations
	if( $nagiosRunning) {
		if( $service->{ "active_checks_enabled"}) {
			my $nextCheck = findServiceNextCheck( $cfg, $service);
			noisySleepUntilTime( $nextCheck, statusString( 2, 
					sprintf( "Waiting for next check to run at %s",
					humanTime( $nextCheck))));
		}
		if( $versionMajor < 4) {
			noisySleep( $cfg->{ "check_result_reaper_frequency"} + 2,
					statusString( 2, "Waiting for check results reaper to run"));
		}
		noisySleepUntilFileUpdate( $cfg->{ "status_file"}, time,
				$cfg->{ "status_update_interval"} * 
				$cfg->{ "StatusUpdateIntervalMultiplier"}, 
				statusString( 2, "Waiting for status to update")); 
	}

	return 1;
}

sub failService {
	my $service = shift;
	my $cfg = shift;
	my $schedule = shift;
	my $downtimeStart = shift;
	my $nagiosRunning = shift;
	my $versionMajor = shift;

	# downtimeEndCheckIncrement is how much time we're going to add before the
	# DowntimeEndCheck event due to a service check being delayed after the
	# service fail command is sent.
	my $downtimeEndCheckIncrement = 0;

	print statusString( 1, "Sending service failure.\n");
	if( $service->{ "active_checks_enabled"}) {
		if(( $service->{ "host_name"} eq "localhost") && 
				( $service->{ "service_description"} eq "HTTP")) {
			system( "/sbin/service httpd stop");
		}
		elsif(( $service->{ "host_name"} eq "localhost") && 
				( $service->{ "service_description"} eq "SSH")) {
			system( "/sbin/service sshd stop");
		}
		else {
			die sprintf( "Don't know how to stop service %s:%s",
					$service->{ "host_name"}, 
					$service->{ "service_description"});
		}
	}
	else {
		die "Nagios is not running" unless( $nagiosRunning);
		sendPassiveResults( $service, 2, "CRITICAL - Danger Will Robinson");
	}
	if( $nagiosRunning) {
		if( $service->{ "active_checks_enabled"}) {
			my $nextCheck = findServiceNextCheck( $cfg, $service);
			$downtimeEndCheckIncrement = $nextCheck - time - 
					maxDiscoveryTime( $cfg, $service, $versionMajor);
			$downtimeEndCheckIncrement = 0 if( $downtimeEndCheckIncrement < 0);
			noisySleepUntilTime( $nextCheck, statusString( 2, 
					sprintf( "Waiting for next check to run at %s",
					humanTime( $nextCheck))));
		}
		if( $versionMajor < 4) {
			noisySleep( $cfg->{ "check_result_reaper_frequency"} + 2,
					statusString( 2, "Waiting for check results reaper to run"));
		}
		noisySleepUntilFileUpdate( $cfg->{ "status_file"}, time,
				$cfg->{ "status_update_interval"} * 
				$cfg->{ "StatusUpdateIntervalMultiplier"}, 
				statusString( 2, "Waiting for status to update"));

		if( time < $downtimeStart + $schedule->{ "DowntimeEnd"}) {
			# We have not passed the end of the downtime waiting for 
			# a check that would trigger a downtime
			print statusString( 2, 
					"Verifying that flexible downtime is in effect.\n");
			my $datfile = readDatFile( $cfg->{ "status_file"});
			my $downtimes = findServiceDowntimes( $datfile, $service);
			if( scalar( @$downtimes) != 1) {
				print STDERR "Incorrect number of downtimes found: " . 
						scalar( @$downtimes) . "\n";
				return ( -1, undef);
			}
			if( ! $downtimes->[ 0]->{ "is_in_effect"}) {
				print STDERR "Downtime is not in effect\n";
				return ( -1, undef);
			}
			print statusString( 2, 
					sprintf( "The flexible downtime was triggered at %s.\n", 
					humanTime( $downtimes->[ 0]->{ "flex_downtime_start"})));
			return ( $downtimeEndCheckIncrement, $downtimes->[ 0]);
		}
		else {
			return ( $downtimeEndCheckIncrement, undef);
		}
	}
	else { 
		return ( 0, undef);
	}
}

sub checkStateAfterNagiosRestart {
	my $cfg = shift;
	my $service = shift;
	my $schedule = shift;
	my $downtimeStart = shift;
	my $downtimeFixed = shift;
	my $downtimeAfterServiceFail = shift;
	my $downtimesAtNagiosStop = shift;

	# If a flexible downtime was triggered and completed before the
	# Nagios restart, we have nothing to do.
	my $durationEndBeforeNagiosStop = ( ! $downtimeFixed && 
			( $schedule->{ "ServiceFail"} > 0) &&
			( $schedule->{ "NagiosStop"} > 0) &&
			( ref( $downtimeAfterServiceFail) eq "HASH") &&
			( $downtimeAfterServiceFail->{ "flex_downtime_start"} > 0) &&
			( $downtimeAfterServiceFail->{ "flex_downtime_start"} + 
			$schedule->{ "DowntimeDuration"} < $schedule->{ "NagiosStart"} + 
			$downtimeStart));
	print statusString( 3, 
			sprintf( "Downtime duration %s before Nagios was restarted\n", 
			( $durationEndBeforeNagiosStop ? "ended" : "did not end")));
	return 0 if ( $durationEndBeforeNagiosStop);

	# If the service failed AND recovered while Nagios was down, a flexible 
	# downtime would never be trigged.
	my $serviceCycleWhileNagiosDown = ( ! $downtimeFixed && 
			( $schedule->{ "ServiceFail"} > 0) &&
			( $schedule->{ "NagiosStop"} > 0) && 
			( $schedule->{ "NagiosStop"} < $schedule->{ "ServiceFail"}) && 
			( $schedule->{ "ServiceRecover"} < $schedule->{ "NagiosStart"}));
	print statusString( 3, sprintf( "Service %s while Nagios was down\n", 
			( $serviceCycleWhileNagiosDown ? "cycled" : "did not cycle")));
	return 0 if( $serviceCycleWhileNagiosDown);

	# downtimeEndCheckIncrement is how much time we're going to add before the
	# DowntimeEndCheck event due to the downtime not being triggered until
	# after a Nagios restart, if that even happens
	my $downtimeEndCheckIncrement = 0;

	# Determine whether the Nagios restart occurred after the end of the
	# downtime.
	my $nagiosRestartAfterDowntimeEnd = ( $schedule->{ "NagiosStart"} >
			$schedule->{ "DowntimeEnd"});

	# Use the above information and the schedule to determine whether a 
	# flexible downtime should have been triggered or should yet be triggered.
	my $downtimeTriggeredBeforeNagiosStop = 0;
	my $downtimeTriggeredAfterNagiosStart = 0;
	if( ! $downtimeFixed) {
		if(( $schedule->{ "ServiceFail"} > 0) && 
				( $schedule->{ "ServiceFail"} < $schedule->{ "NagiosStop"})) {
			$downtimeTriggeredBeforeNagiosStop = 1;
			print statusString( 3, "Downtime should have been triggered " .
					"before Nagios stopped\n");
		}
		else {
			print statusString( 3, "Downtime would not have been triggered " .
					"before Nagios stopped\n");
		}
		if( ! $nagiosRestartAfterDowntimeEnd && 
				( $schedule->{ "ServiceFail"} > 0 ) &&
				( $schedule->{ "NagiosStop"} < $schedule->{ "ServiceFail"}) &&
				( $schedule->{ "ServiceFail"} < $schedule->{ "NagiosStart"})) {
			$downtimeTriggeredAfterNagiosStart = 1;
			print statusString( 3, "Downtime should be triggered after " .
					"Nagios start\n");
		}
		else {
			print statusString( 3, "Downtime would not be triggered after " .
					"Nagios start\n");
		}
	}


	# If the downtime should have been triggered, but was not detected prior
	# to the Nagios stop, wait for it to be detected.
	if( $downtimeTriggeredAfterNagiosStart) {
		if( $service->{ "active_checks_enabled"}) {
			my $nextCheck = findServiceNextCheck( $cfg, $service);
			$downtimeEndCheckIncrement = ( $nextCheck - time) +
					( $schedule->{ "NagiosStart"} - 
					$schedule->{ "ServiceFail"});
			noisySleepUntilTime( $nextCheck, statusString( 1, 
					sprintf( "Waiting for next check to run at %s",
					humanTime( $nextCheck))));
		}
		if( time > $downtimeStart + $schedule->{ "DowntimeEnd"}) {
			# We passed the end of the downtime waiting for a check that
			# would trigger a downtime
			print statusString( 2, 
					"Downtime ended while waiting for next check.\n");
			return 0;
		}
	}

	# Next determine whether the downtime should still exist
	my $downtimeShouldExist = 0;
	if( $downtimeFixed) {
		if( time < $downtimeStart + $schedule->{ "DowntimeEnd"}) {
			# If the downtime is fixed and we have not reached the 
			# downtime end, it should still exists so check it below
			$downtimeShouldExist = 1;
		}
	}
	else {
		if( ! $downtimeTriggeredBeforeNagiosStop && 
				! $downtimeTriggeredAfterNagiosStart) {
			if( time < $downtimeStart + $schedule->{ "DowntimeEnd"}) {
				# If the downtime is flexible, it was never triggered 
				# and we have not reached the downtime end, it should 
				# still exist so check it below
				print statusString( 2, "We have not reached the downtime " .
						"end, so it should still exist.\n");
				$downtimeShouldExist = 1;
			}
		}
		else {
			# In this case the downtime is flexible and should have been
			# triggered.
			print statusString( 3, 
					sprintf( "There were %d downtime(s) when Nagios stopped.\n",
					scalar( @$downtimesAtNagiosStop)));
			if( scalar( @$downtimesAtNagiosStop) == 1) {
				print statusString( 3, 
						sprintf( "The flex downtime start for the downtime " .
						"existing when\nNagios stopped was %s\n", 
						humanTime( $downtimesAtNagiosStop->[ 0]->{ "flex_downtime_start"})));
			}
			if(( scalar( @$downtimesAtNagiosStop) == 1) && 
					( $downtimesAtNagiosStop->[ 0]->{ "flex_downtime_start"} > 0)) {
				print statusString( 2, 
						"The downtime was triggered before Nagios stopped.\n");
				if( time < 
					( $downtimesAtNagiosStop->[ 0]->{ "flex_downtime_start"} + 
					$downtimesAtNagiosStop->[ 0]->{ "duration"})) {
					# If the trigger was detected before Nagios was stopped and
					# we are still in the flexible downtime, it should exist
					print statusString( 2, "We have not reached the duration " .
							"end; it should still exist.\n");
					$downtimeShouldExist = 1;
				}
				else {
					print statusString( 2, "We have reached the duration " .
							"end; it should not exist.\n");
				}
			}
			elsif( time < ( $downtimeStart + $schedule->{ "NagiosStart"} + 
					$schedule->{ "DowntimeDuration"})) {
				# If the downtime was not triggered until after Nagios restarted
				# and we have not reached the duration end, it should still 
				# exist so check it below
				print statusString( 2, "We have not reached the duration " .
						"end; it should still exist.\n");
				$downtimeShouldExist = 1;
			}
		}
	}


	# If the downtime should exist, verify the downtime is back. If it should
	# not exist, verification that the downtime is gone happens at the 
	# DowntimeEndCheck event.
	if( $downtimeShouldExist) {
		print statusString( 1, "Verifying downtime after Nagios restart.\n");
		if( !verifyDowntime( $service, $cfg, $downtimeFixed, 
				$downtimeStart,
				$downtimeStart + ( $schedule->{ "DowntimeEnd"} - 
				$schedule->{ "DowntimeStart"}), 
				$schedule->{ "DowntimeDuration"})) {
			return -1;
		}
	}
	else {
		print statusString( 1, 
				"The downtime should have ended; it will be checked later.\n");
	}

	return $downtimeEndCheckIncrement;
}

# Perform the check of a downtime sequence
sub checkDowntime {
	my $testComment = shift;
	my $service = shift;
	my $cfg = shift;
	my $downtimeFixed = shift;
	my $schedule = shift;
	my $versionMajor = shift;

	my $startTime = time;
	my $datfile;
	my $nagiosRunning = 1;
	my $downtimesAtNagiosStop;
	my $downtimeAfterServiceFail;
	my $downtimeEndCheckIncrement = 0;

	print "$testComment\n";

	# Verify parameters
	die "No downtime start specified" unless( exists( $schedule->{ "DowntimeStart"}));
	die "No downtime end specified" unless( $schedule->{ "DowntimeEnd"} >= 0);
	die "No downtime duration specified" unless( $downtimeFixed || 
			$schedule->{ "DowntimeDuration"} >= 0);

	# Determine the amount of time to discover a service failure above
	# the normal check results reaper frequency
	my $maxDiscoveryTime = maxDiscoveryTime( $cfg, $service, $versionMajor);

	# Create an events list
	my @events;
	my $eventTime;

	# Figure out the time for the check at the end of the downtime
	if( ! $downtimeFixed && $schedule->{ "ServiceFail"} > 0) {
		if( $schedule->{ "NagiosStart"} > $schedule->{ "ServiceFail"} + 
				$schedule->{ "DowntimeDuration"} + $maxDiscoveryTime) {
			$eventTime = $schedule->{ "NagiosStart"} + 1;
		} 
		else {
			$eventTime = $schedule->{ "ServiceFail"} + 
					$schedule->{ "DowntimeDuration"};
			if( $service->{ "active_checks_enabled"}) {
				$eventTime += $maxDiscoveryTime;
			}
		}
	}
	else {
		if( $schedule->{ "NagiosStart"} > ( $schedule->{ "DowntimeEnd"} - 
				$schedule->{ "DowntimeStart"})) {
			$eventTime = $schedule->{ "NagiosStart"} + 1;
		}
		else {
			$eventTime = ( $schedule->{ "DowntimeEnd"} - 
					$schedule->{ "DowntimeStart"});
		}
	}
	push( @events, { "time" => $eventTime + $cfg->{ "status_update_interval"}, 
			"type" => "DowntimeEndCheck"});

	# Add the Nagios stop and start events if they're in the permutation
	if( $schedule->{ "NagiosStop"} > 0) {
		push( @events, { "time" => $schedule->{ "NagiosStop"}, 
				"type" => "NagiosStop"});
		if( $schedule->{ "NagiosStart"} > 0) {
			push( @events, { "time" => $schedule->{ "NagiosStart"}, 
					"type" => "NagiosStart"});
		}
		else {
			die "No Nagios restart time specified";
		}
	}

	# Add the service failure and recover events if they're in the permutation
	if( $schedule->{ "ServiceFail"} > 0) {
		push( @events, { "time" => $schedule->{ "ServiceFail"}, 
				"type" => "ServiceFail"});
		if( $schedule->{ "ServiceRecover"} > 0) {
			push( @events, { "time" => $schedule->{ "ServiceRecover"}, 
					"type" => "ServiceRecover"});
		}
		else {
			die "No service recovery time specified";
		}
	}

	# Sort the events in chronological order
	my @sortedEvents = sort { $a->{ "time"} <=> $b->{ "time"}} @events;

	# Make sure the service starts in a good state
	if( ! recoverService( $service, $cfg, 
			"Making sure the service is in a good state.", $nagiosRunning,
			$versionMajor)) {
		return -1;
	}

	# Cancel all current downtimes
	print statusString( 1, "Deleting current downtimes: ");
	$datfile = readDatFile( $cfg->{ "status_file"});
	$downtimes = findServiceDowntimes( $datfile, $service);
	for( my $x = 0; $x < @$downtimes; $x++) {
		print ", " if( $x > 0);
		print $downtimes->[ $x]->{ "downtime_id"};
		deleteDowntime( $downtimes->[ $x]->{ "downtime_id"}, $cfg, $service);
	}
	print "\n";

	# Schedule the downtime
	my $downtimeStart = time + $schedule->{ "DowntimeStart"};
	if( $downtimeFixed) {
		print statusString( 1, 
				sprintf( "Scheduling fixed downtime\n\t\tfrom %s to %s.\n",
				humanTime( $downtimeStart), 
				humanTime( $downtimeStart + ( $schedule->{ "DowntimeEnd"} - 
				$schedule->{ "DowntimeStart"}))));
	}
	else {
		print statusString( 1, sprintf( "Scheduling flexible downtime of " .
				"%d seconds\n\t\tfrom %s to %s.\n",
				$schedule->{ "DowntimeDuration"}, humanTime( $downtimeStart),
				humanTime( $downtimeStart + ( $schedule->{ "DowntimeEnd"} - 
				$schedule->{ "DowntimeStart"}))));
	}
	scheduleDowntime( $service, $downtimeStart,
			$downtimeStart + ( $schedule->{ "DowntimeEnd"} - 
			$schedule->{ "DowntimeStart"}), $downtimeFixed, 
			$schedule->{ "DowntimeDuration"}, $testComment);
	
	# Wait for the status file to update
	noisySleepUntilFileUpdate( $cfg->{ "status_file"}, time,
			$cfg->{ "status_update_interval"} * 
			$cfg->{ "StatusUpdateIntervalMultiplier"}, 
			statusString( 2, "Waiting for status to update"));

	# Verify the downtime 
	if( !verifyDowntime( $service, $cfg, $downtimeFixed, $downtimeStart,
			$downtimeStart + ( $schedule->{ "DowntimeEnd"} - 
			$schedule->{ "DowntimeStart"}), 
			$schedule->{ "DowntimeDuration"})) {
		return -1;
	}

	# Enter the event loop
	my $eventIndex = 0;
	while( $eventIndex < @sortedEvents) {

		# Sleep until the next event is to to be executed
		if( time < ( $downtimeStart + $sortedEvents[ $eventIndex]->{ "time"})) {
			my $message = statusString( 1, 
					sprintf( "Waiting for %s event at %s", 
					$sortedEvents[ $eventIndex]->{ "type"},
					humanTime( $downtimeStart + 
					$sortedEvents[ $eventIndex]->{ "time"})));
			noisySleepUntilTime(( $downtimeStart + 
					$sortedEvents[ $eventIndex]->{ "time"}), $message);
		}

		# Verify the downtime is gone at the end of the check
		if( $sortedEvents[ $eventIndex]->{ "type"} eq "DowntimeEndCheck") {
			noisySleepUntilFileUpdate( $cfg->{ "status_file"}, time,
					$cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"}, 
					statusString( 2, "Waiting for status to update"));
					# Grab the status data again
			print statusString( 2, "Rereading status data.\n");
			$datfile = readDatFile( $cfg->{ "status_file"});

			# Find the downtimes
			print statusString( 2, "Looking for downtimes again.\n");
			$downtimes = findServiceDowntimes( $datfile, $service);
			print statusString( 1, "Verifying the downtime has ended.\n");
			if( scalar( @$downtimes) > 0) {
				for( my $y = 0; $y < @$downtimes; $y++) {
					if( $downtimes->[ $y]->{ "is_in_effect"}) {
						print STDERR "Downtime still exists.\n";
						return -1;
					}
				}
			}
		}

		# Restart Nagios
		elsif( $sortedEvents[ $eventIndex]->{ "type"} eq "NagiosStart") {
			print statusString( 1, "Starting Nagios.\n");
			system( "/sbin/service nagios start");
			system( "/bin/chgrp nagcmd /usr/local/nagios/var/rw/nagios.cmd");

			# Wait for the status file to update
			noisySleepUntilFileUpdate( $cfg->{ "status_file"}, time,
					$cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"},
					statusString( 2, "Waiting for status to update"));

			# Check the status of things after the Nagios restart	
			$downtimeEndCheckIncrement = checkStateAfterNagiosRestart( $cfg, 
					$service, $schedule, $downtimeStart, $downtimeFixed, 
					$downtimeAfterServiceFail, $downtimesAtNagiosStop);
			if( $downtimeEndCheckIncrement < 0) {
				return -1;
			}
			elsif( $downtimeEndCheckIncrement > 0) {
				print statusString( 3, 
					sprintf( "Extending the DowntimeEndCheck time by %d " .
					"seconds\n", $downtimeEndCheckIncrement));
				$sortedEvents[ @sortedEvents - 1]->{ "time"} += 
						$downtimeEndCheckIncrement;
			}
			$nagiosRunning = 1;
		}

		# Stop Nagios
		elsif( $sortedEvents[ $eventIndex]->{ "type"} eq "NagiosStop") {
			$datfile = readDatFile( $cfg->{ "status_file"});
			$downtimesAtNagiosStop = findServiceDowntimes( $datfile, $service);
			print statusString( 3, sprintf( 
					"There are %d downtime(s) just before stopping Nagios.\n",
					scalar( @$downtimesAtNagiosStop)));
			print statusString( 1, "Stopping Nagios.\n");
			system( "/sbin/service nagios stop");
			$nagiosRunning = 0;
		}

		# Cause the service to fail
		elsif( $sortedEvents[ $eventIndex]->{ "type"} eq "ServiceFail") {
			( $downtimeEndCheckIncrement, $downtimeAfterServiceFail) = 
					failService( $service, $cfg, $schedule, $downtimeStart, 
					$nagiosRunning, $versionMajor);
			if( $downtimeEndCheckIncrement < 0) {
				return -1;
			}
			elsif( $downtimeEndCheckIncrement > 0) {
				print statusString( 3, 
					sprintf( "Extending the DowntimeEndCheck time by %d " .
					"seconds\n", $downtimeEndCheckIncrement));
				$sortedEvents[ @sortedEvents - 1]->{ "time"} += 
						$downtimeEndCheckIncrement;
			}
		}

		# Cause the service to recover
		elsif( $sortedEvents[ $eventIndex]->{ "type"} eq "ServiceRecover") {
			if( !recoverService( $service, $cfg, "Sending service recovery.", 
					$nagiosRunning, $versionMajor)) {
				return -1;
			}
		}

		# Something is dreadfully wrong
		else {
			die "Unhandled event type: " . $sortedEvents[ $eventIndex]->{ "type"};
		}
		$eventIndex++;
	}

	return( time - $startTime);
}

# Determine whether a event permutation is valid
sub isValidEventPermutation {
	my $permutation = shift;
	my $activeChecks = shift;

	# Create a hash of the events with the event name as the key and the order
	# as the value.
	my %permutation;
	for( my $x = 0; $x < @$permutation; $x++) {
		$permutation{ $permutation->[ $x]} = $x;
	}

	# The downtime cannot be created after it ends
	if( exists( $permutation{ "DowntimeEnd"}) &&
			exists( $permutation{ "EntryTime"}) && 
			( $permutation{ "DowntimeEnd"} < $permutation{ "EntryTime"})) {
		return 0;
	}

	# The downtime cannot end before it starts
	if( exists( $permutation{ "DowntimeEnd"}) &&
			exists( $permutation{ "DowntimeStart"}) && 
			( $permutation{ "DowntimeEnd"} < $permutation{ "DowntimeStart"})) {
		return 0;
	}

	# The service cannot recover before it fails
	if( exists( $permutation{ "ServiceFail"}) && 
			exists( $permutation{ "ServiceRecover"}) && 
			( $permutation{ "ServiceRecover"} < $permutation{ "ServiceFail"})) {
		return 0;
	}

	# Nagios cannot be restarted before it is initially stopped
	if( exists( $permutation{ "NagiosStop"}) && 
			exists( $permutation{ "NagiosStart"}) && 
			( $permutation{ "NagiosStart"} < $permutation{ "NagiosStop"})) {
		return 0;
	}

	# It doesn't make sense to test when a service failure occurs after the 
	# end of the downtime
	if( exists( $permutation{ "ServiceFail"}) &&
			exists( $permutation{ "DowntimeEnd"}) && 
			( $permutation{ "ServiceFail"} > $permutation{ "DowntimeEnd"})) {
		return 0;
	}

	# It doesn't make sense to test when a service failure occurs before the 
	# downtime is created
	if( exists( $permutation{ "ServiceFail"}) &&
			exists( $permutation{ "EntryTime"}) && 
			( $permutation{ "ServiceFail"} < $permutation{ "EntryTime"})) {
		return 0;
	}

	# It doesn't make sense to test when a service failure occurs before the 
	# downtime starts
	if( exists( $permutation{ "ServiceFail"}) &&
			exists( $permutation{ "DowntimeStart"}) && 
			( $permutation{ "ServiceFail"} < $permutation{ "DowntimeStart"})) {
		return 0;
	}

	# It doesn't make sense to test when a Nagios shutdown occurs after the 
	# end of the downtime
	if( exists( $permutation{ "NagiosStop"}) &&
			exists( $permutation{ "DowntimeEnd"}) && 
			( $permutation{ "NagiosStop"} > $permutation{ "DowntimeEnd"})) {
			return 0;
	}

	# A downtime cannot be created when Nagios is shutdown and it doesn't 
	# make sense to test when Nagios is shutdown and restarted before the
	# downtime is created
	if( exists( $permutation{ "NagiosStop"}) &&
			exists( $permutation{ "EntryTime"}) && 
			( $permutation{ "NagiosStop"} < $permutation{ "EntryTime"})) {
		return 0;
	}

	# It doesn't make sense for the duration to be so short it ends before
	# any real testing occurs
	if( exists( $permutation{ "DurationEnd"}) &&
			( $permutation{ "DurationEnd"} < 1)) {
		return 0;
	}

	# The following only applies to passive checks
	unless( $activeChecks) {
		# A passive service failure cannot be sent to Nagios when it is not
		# running
		if( exists( $permutation{ "ServiceFail"}) &&
				exists( $permutation{ "NagiosStop"}) && 
				exists( $permutation{ "NagiosStart"}) && 
				( $permutation{ "ServiceFail"} > $permutation{ "NagiosStop"}) &&
				( $permutation{ "ServiceFail"} < $permutation{ "NagiosStart"})) {
			return 0;
		}

		# A passive service recovery cannot be sent to Nagios when it is not
		# running
		if( exists( $permutation{ "ServiceRecover"}) &&
				exists( $permutation{ "NagiosStop"}) && 
				exists( $permutation{ "NagiosStart"}) && 
				( $permutation{ "ServiceRecover"} > $permutation{ "NagiosStop"}) &&
				( $permutation{ "ServiceRecover"} < $permutation{ "NagiosStart"})) {
			return 0;
		}
	}
	return 1;
}

# Display a time in human-readable format
sub humanTime {
	my $epoch = shift;
	my $longFormat = 0;
	$longFormat = shift if( @_);

	my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) =
			localtime( $epoch);	
	if( $longFormat) {
		return sprintf( "%04d-%02d-%02d %02d:%02d:%02d", $year+1900, $mon+1, 
				$mday, $hour, $min, $sec);
	}
	else {
		return sprintf( "%02d:%02d:%02d", $hour, $min, $sec);
	}
}

# Display a time duration in human-readable format
sub humanDuration {
	my $duration = shift;
	my $wordyFormat = 0;
	$wordyFormat = shift if( @_);

	my ( $durationLeft, $hours, $minutes, $seconds);
	$durationLeft = $duration;
	$hours = int( $durationLeft / ( 60 * 60));
	$durationLeft -= $hours * 60 * 60;
	$minutes = int( $durationLeft / 60);
	$durationLeft -= $minutes * 60;
	$seconds = $durationLeft;

	if( $wordyFormat) {
		if( $hours > 0) {
			return sprintf( "%d hour%s, %d minute%s, %d second%s", 
					$hours, ( $hours == 1 ? "" : "s"), 
					$minutes, ( $minutes == 1 ? "" : "s"), 
					$seconds, ( $seconds == 1 ? "" : "s"));
		}
		elsif( $minutes > 0) {
			return sprintf( "%d minute%s, %d second%s", 
					$minutes, ( $minutes == 1 ?  "" : "s"), 
					$seconds, ( $seconds == 1 ? "" : "s"));
		}
		else {
			return sprintf( "%d second%s", $duration, 
					( $duration == 1 ?  "" : "s"));
		}
	}
	else {
		return sprintf( "%02d:%02d:%02d (%ds)", $hours, $minutes, $seconds,
				$duration);
	}
}

# Schedule a downtime series' event times
sub scheduleDowntimePermutation {
	my $service = shift;
	my $cfg = shift;
	my $permutation = shift;
	my $versionMajor = shift;

	# Initialize times
	my %schedule;
	$schedule{ "DowntimeStart"} = -1;
	$schedule{ "DowntimeEnd"} = -1;
	$schedule{ "DowntimeDuration"} = -1;
	$schedule{ "NagiosStop"} = -1;
	$schedule{ "NagiosStart"} = -1;
	$schedule{ "ServiceFail"} = -1;
	$schedule{ "ServiceRecover"} = -1;

	# Determine the amount of time to discover a service failure above
	# the normal check results reaper frequency
	my $maxDiscoveryTime = maxDiscoveryTime( $cfg, $service, $versionMajor);

	# Get the current time, to be used for all future times
	my $currentTime = time;

	# Last event time
	my $lastEventTime;

	if( $permutation->[ 0] eq "EntryTime") {
		$schedule{ "DowntimeStart"} = 10;
		$lastEventTime = $schedule{ "DowntimeStart"};
	}
	elsif( $permutation->[ 0] eq "DowntimeStart") {
		$schedule{ "DowntimeStart"} = -10;
		$lastEventTime = $schedule{ "DowntimeStart"} + 
				( $versionMajor < 4 ? 
				$cfg->{ "check_result_reaper_frequency"} : 0) + 2 ;
	}
	else {
		die "Unsupported permutation order: " . $permutation->[ 0] . " => " .
				$permutation->[ 1];
	}

	print join( " => ", @$permutation) . "\n";
	printf( "    Downtime Start: %s\n", $schedule{ "DowntimeStart"});
	# Iterate through the remaining events, determining the time for each	
	for( my $x = 0; $x < @$permutation; $x++) {
		if( $permutation->[ $x] eq "DowntimeEnd") {
			$schedule{ "DowntimeEnd"} = $lastEventTime + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"});
			$lastEventTime = $schedule{ "DowntimeEnd"};
			printf( "    Downtime End: %s\n", $schedule{ "DowntimeEnd"});
		}
		elsif( $permutation->[ $x] eq "DowntimeStart") {
			; # Do nothing - Downtime Start was determined earlier
		}
		elsif( $permutation->[ $x] eq "DurationEnd") {
			$schedule{ "DowntimeDuration"} = ( $lastEventTime - 
					$schedule{ "DowntimeStart"}) + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"} + 
					$maxDiscoveryTime);
			$lastEventTime = $schedule{ "DowntimeDuration"} + 
					$schedule{ "DowntimeStart"};
			printf( "    Duration: %d\n", $schedule{ "DowntimeDuration"});
		}
		elsif( $permutation->[ $x] eq "EntryTime") {
			; # Do nothing
		}
		elsif( $permutation->[ $x] eq "NagiosStop") {
			$schedule{ "NagiosStop"} = $lastEventTime + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"});
			$lastEventTime = $schedule{ "NagiosStop"};
			printf( "    Nagios Stop: %s\n", $schedule{ "NagiosStop"});
		}
		elsif( $permutation->[ $x] eq "NagiosStart") {
			$schedule{ "NagiosStart"} = $lastEventTime + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"});
			$lastEventTime = $schedule{ "NagiosStart"};
			printf( "    Nagios Start: %s\n", $schedule{ "NagiosStart"});
		}
		elsif( $permutation->[ $x] eq "ServiceFail") {
			$schedule{ "ServiceFail"} = $lastEventTime + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"});
			$lastEventTime = $schedule{ "ServiceFail"} + 
					( $versionMajor < 4 ? 
					$cfg->{ "check_result_reaper_frequency"} : 0) + 
					$maxDiscoveryTime;
			printf( "    Service Fail: %s\n", $schedule{ "ServiceFail"});
		}
		elsif( $permutation->[ $x] eq "ServiceRecover") {
			$schedule{ "ServiceRecover"} = $lastEventTime + 
					( $cfg->{ "status_update_interval"} * 
					$cfg->{ "StatusUpdateIntervalMultiplier"});
			$lastEventTime = $schedule{ "ServiceRecover"} +
					( $versionMajor < 4 ? 
					$cfg->{ "check_result_reaper_frequency"} : 0) + 
					$maxDiscoveryTime;
			printf( "    Service Recover: %s\n", $schedule{ "ServiceRecover"});
		}
		else {
			die "Unhandled event type: " . $permutation->[ $x];
		}
	}

	return \%schedule;
}

# Generate a list of valid permutations for fixed downtime events
sub fixedDowntimeTestPermutations {
	my $service = shift;

	my @allPermutations;

	my @noRestart = ( "EntryTime", "DowntimeStart", "DowntimeEnd");
	my $iteratorNoRestart= Algorithm::Permute->new ( \@noRestart);
	while (my @perm1 = $iteratorNoRestart->next) {
		if( isValidEventPermutation( \@perm1, 
				$service->{ "active_checks_enabled"})) {
#			print "(@perm1)\n";
			push( @allPermutations, \@perm1);
		}
	}

	my @withRestart = ( "EntryTime", "DowntimeStart", "NagiosStop",
			"NagiosStart", "DowntimeEnd");
	my $iteratorWithRestart = 
		Algorithm::Permute->new ( \@withRestart);
	while (my @perm2 = $iteratorWithRestart->next) {
		if( isValidEventPermutation( \@perm2, 
				$service->{ "active_checks_enabled"})) {
#			print "(@perm2)\n";
			push( @allPermutations, \@perm2);
		}
	}

	return \@allPermutations;

}

# Generate a list of valid permutations for flexible downtime events
sub flexibleDowntimeTestPermutations {
	my $service = shift;

	my @allPermutations;

	my @noRestartOrTrigger = ( "EntryTime", "DowntimeStart", "DurationEnd", 
			"DowntimeEnd");
	my $iteratorNoRestartOrTrigger = 
			Algorithm::Permute->new ( \@noRestartOrTrigger );
	while (my @perm1 = $iteratorNoRestartOrTrigger->next) {
		if( isValidEventPermutation( \@perm1, 
				$service->{ "active_checks_enabled"})) {
			push( @allPermutations, \@perm1);
		}
	}

	my @noRestart = ( "EntryTime", "DowntimeStart", "DurationEnd", 
			"DowntimeEnd", "ServiceFail", "ServiceRecover");
	my $iteratorNoRestart = Algorithm::Permute->new ( \@noRestart );
	while (my @perm2 = $iteratorNoRestart->next) {
		if( isValidEventPermutation( \@perm2, 
				$service->{ "active_checks_enabled"})) {
			push( @allPermutations, \@perm2);
		}
	}

	my @noTrigger = ( "EntryTime", "DowntimeStart", "DurationEnd", 
			"DowntimeEnd", "NagiosStop", "NagiosStart");
	my $iteratorNoTrigger = Algorithm::Permute->new ( \@noTrigger );
	while (my @perm3 = $iteratorNoTrigger->next) {
		if( isValidEventPermutation( \@perm3, 
				$service->{ "active_checks_enabled"})) {
			push( @allPermutations, \@perm3);
		}
	}

	my @allEvents = ( "EntryTime", "DowntimeStart", "DurationEnd", 
			"DowntimeEnd", "NagiosStop", "NagiosStart", "ServiceFail", 
			"ServiceRecover");
	my $iteratorAllEventns = Algorithm::Permute->new ( \@allEvents );
	while (my @perm4 = $iteratorAllEventns->next) {
		if( isValidEventPermutation( \@perm4, 
				$service->{ "active_checks_enabled"})) {
			push( @allPermutations, \@perm4);
		}
	}

	return \@allPermutations;
}

# This function verifies the global Nagios configuration
sub verifyConfig {

	my $cfg = shift;
	my $versionMajor = shift;

	# Verify the status update interval
	if( $cfg->{ "status_update_interval"} < 5) {
		print STDERR sprintf( "status_update_interval (%d) is too short\n",
				$cfg->{ "status_update_interval"});
		return 0;
	}
	if( $cfg->{ "status_update_interval"} >= 10) {
		print sprintf( "The status_update_interval (%d) is long. " .
				"It should optimally be 5 seconds.\n", 
				$cfg->{ "status_update_interval"});
		print "Press ENTER to continue...";
		<STDIN>;
	}

	if( $versionMajor < 4) {
		# Verify the check_result_reaper_frequency
		if( $cfg->{ "check_result_reaper_frequency"} < 5) {
			print STDERR sprintf( "check_result_reaper_frequency (%d) is too " .
					"short\n", $cfg->{ "check_result_reaper_frequency"});
			return 0;
		}
		if( $cfg->{ "check_result_reaper_frequency"} >= 10) {
			print sprintf( "The check_result_reaper_frequency (%d) is long. " .
					"It should optimally be 5 seconds.\n", 
					$cfg->{ "check_result_reaper_frequency"});
			print "Press ENTER to continue...";
			<STDIN>;
		}
	}

	return 1;
}

sub verifyService {
	my $cfg = shift;
	my $hostName = shift;
	my $serviceDescription = shift;

	# Make sure there is an appropriately configured service before we 
	# start testing.
	my $objects = readObjectCache( $cfg->{ "object_cache_file"});
	my $service = findService( $objects, $hostName, $serviceDescription);
	if( !defined( $service)) {
		print STDERR sprintf( "No service found for host '%s' and " .
				"service description '%s'.\n", $hostName, $serviceDescription);
		return undef;
	}
	if( !( $service->{ "active_checks_enabled"} || 
			$service->{ "passive_checks_enabled"})) {
		print STDERR sprintf( "Either active or passive checks must be " .
				"enabled for %s:%s\n", $hostName, $serviceDescription);
		return undef;
	}
	if( $service->{ "notifications_enabled"}) {
		print STDERR sprintf( "Notifications must not be enabled for %s:%s\n",
				$hostName, $serviceDescription);
		return undef;
	}
	if( $service->{ "flap_detection_enabled"}) {
		print STDERR sprintf( "Flap detection must not be enabled for %s:%s\n",
				$hostName, $serviceDescription);
		return undef;
	}
	if( $service->{ "process_perf_data"}) {
		print STDERR sprintf( "Performance data processing must not be " .
				"enabled for %s:%s\n", $hostName, $serviceDescription);
		return undef;
	}
	if( $service->{ "active_checks_enabled"}) {
		if( $service->{ "max_check_attempts"} > 1) {
			print sprintf( "The max_check_attempts (%d) is high. " .
					"It should optimally be 1.\n", 
					$cfg->{ "max_check_attempts"});
			print "Press ENTER to continue...";
			<STDIN>;
		}
		if( $service->{ "check_interval"} > 1) {
			print sprintf( "The normal_check_interval (%d) is long. " .
					"It should optimally be 1 minute.\n", 
					$cfg->{ "normal_interval"});
			print "Press ENTER to continue...";
			<STDIN>;
		}
		if(( $service->{ "max_check_attempts"} > 1) && 
				( $service->{ "retry_interval"} > 1)) {
			print sprintf( "The retry_check_interval (%d) is long. " .
					"It should optimally be 1 minute.\n", 
					$cfg->{ "retry_interval"});
			print "Press ENTER to continue...";
			<STDIN>;
		}
	}
	else {
		if( $service->{ "check_freshness"}) {
			print STDERR sprintf( "check_freshness must not be enabled " .
					"for %s:%s\n", $hostName, $serviceDescription);
			return undef;
		}
		if( $service->{ "max_check_attempts"} != 1) {
			print STDERR sprintf( "max_check_attempts must be 1 for %s:%s\n",
					$hostName, $serviceDescription);
			return undef;
		}
	}
	return $service;
}

sub getVersion {
	my $cfg = shift;

	my $status = readDatFile( $cfg->{ "status_file"});
	my @versionparts = split( /\./, $status->{ "info"}->{ "version"});
	my %version;
	$version{ "major"} = $versionparts[ 0];
	$version{ "minor"} = $versionparts[ 1];
	$version{ "micro"} = $versionparts[ 2];

	return \%version;
}

sub isInArray {
	my $string = shift;
	my $array = shift;

	for( my $x = 0; $x < @$array; $x++) {
		return 1 if( $array->[ $x] eq $string);
	}
	return 0;
}

# This is the SIGUSR1 handler - it sets a flag that causes the test to 
# pause.
sub pauseTest {
	$testPaused = 1; 
}

sub usage {
	my $cmd = shift;

	print <<EOU;
This command runs one or more downtime checks. With only the required 
options, the test will run all appropriate permutations of downtime 
event sequences (see --single-run below for a list of events).

The service checked must be an existing service and can be either an 
actively or passively checked service. Note that this test can run a 
very long time, over 24 hours for active checks.

Sending the program the USR1 single will cause it to pause after the
current permutation has completed testing.

Usage: $cmd --host-name <host> --service-description <service>
       [--single-run <test>] [--no-fixed-tests] [--no-flexible-tests]
       [--skip-logged <logfile>] [--help|-?]

Where:
    --host-name <host> specifies the name of the host whose service 
        is to be used for the downtime check.
    --service-description <service> specifies the name of the service 
        to be used for the downtime check.
    --single-run <test> indicates that the single specified test is 
        to be run.  <test> must be of the form "(Fixed|Flexible) 
        downtime: <eventlist>" <eventlist> is a comma and space 
        separated list of the following events: EntryTime, 
        DowntimeStart, [NagiosStop, NagiosStart,] [ServiceFail, 
        ServiceRecover,] DurationEnd, DowntimeEnd. The events within 
        the brackets are optional, but if one of the pair is used, 
        both must be used. Hint: the format of <test> is the same as 
        is displayed at the beginning of each downtime check when 
        --single-run is not specified.
    --no-fixed-tests indicates that no fixed tests should be run. 
        This option is ignored is --single-run is specified.
    --no-flexible-tests indicates that no flexible tests should be 
        run.  This option is ignored is --single-run is specified.
    --skip-logged <logfile> indicates that any tests logged in 
        <logfile> should be skipped. This is useful for start over 
        where you left off if the test had been aborted prematurely.
    --help|-? displays this help message.
EOU
}

# Host and service to be used for testing
my $hostName = "localhost";
my $serviceDescription = "Downtime Test";
my $singleRun = undef;
my $logFile = undef;
my $noFixedTests = 0;
my $noFlexibleTests = 0;
my $showUsage = 0;

$result = GetOptions (	"host-name=s"			=> \$hostName,
						"service-description=s"	=> \$serviceDescription,
						"single-run=s"			=> \$singleRun,
						"no-fixed-tests"		=> \$noFixedTests,
						"no-flexible-tests"		=> \$noFlexibleTests,
						"skip-logged=s"			=> \$logFile,
						"help|?"				=> \$showUsage);

die "Error processing command line options" unless( $result);

if( $showUsage) {
	usage( $PROGRAM_NAME);
	exit( 0);
}

# Install signal handle to enable test pausing
$SIG{ "USR1"} = "pauseTest";

# Make sure we're running as root
if( $EUID != 0) {
	print STDERR "This script must be run as root.\n";
	exit 1;
}

# Make sure Nagios is running
my $status = `/sbin/service nagios status`;
if( $status !~ /nagios \(pid( \d+)+\) is running.../) {
	print STDERR "Nagios must be running in order to run this test.\n";
	exit 1;
}

# Read the Nagios configuration
my $cfg = readCfgFile( "/usr/local/nagios/etc/nagios.cfg");
exit 1 unless( defined( $cfg));

# Get the version
my $version = getVersion( $cfg);
printf( "Nagios version is %d.%d.%d\n", $version->{ "major"}, 
		$version->{ "minor"}, $version->{ "micro"});

# Verify the global configuration
unless( verifyConfig( $cfg, $version->{ "major"}) > 0) {
	exit 1;
}
$cfg->{ "StatusUpdateIntervalMultiplier"} = 4;

# Verify the service configuration
my $service = verifyService( $cfg, $hostName, $serviceDescription);
exit 1 unless( defined( $service));

# Enable autoflush on STDOUT so updates occur
my $old_fh = select(STDOUT);
$| = 1;
select($old_fh); 

my $activeChecks = 0;
my $permutationTime = 0;
my $elapsedTime = 0;
my $schedule;
my $comment;

#my @testPerm = ( "DowntimeStart", "DurationEnd", "EntryTime", "ServiceFail", "NagiosStop", "DowntimeEnd", "NagiosStart", "ServiceRecover");
#my $testSchedule = scheduleDowntimePermutation( $service, $cfg, \@testPerm, 
#		$version->{ "major"});
#my $testComment = join( ", ", @testPerm);
#checkDowntime( "Flexible downtime: $testComment", $service, $cfg, 0, 
#		$testSchedule, $version->{ "major"});
#exit( 0);

if( defined( $singleRun)) {
	if( $singleRun =~ /^(Fixed|Flexible) downtime: (.*)/) {
		my $downtimeType = $1;
		my @singleRunPerm = split( /, /, $2);
		my $fixed = (( $singleRun =~ /^Fixed/) ? 1 : 0);
		my $testSchedule = scheduleDowntimePermutation( $service, $cfg, 
				\@singleRunPerm, $version->{ "major"});
		my $testComment = join( ", ", @singleRunPerm);
		checkDowntime( "$downtimeType downtime: $testComment", $service, $cfg, 
				$fixed, $testSchedule, $version->{ "major"});
		exit 0;
	}
	else {
		die "Unrecognized single run format: $singleRun";
	}
}
else {
	my @skipTests = ();
	if( defined( $logFile)) {
		open( LOG, "$logFile") || 
				die "Unable to open log file $logFile for reading";
		while( <LOG>) {
			chomp;
			push( @skipTests, $_) if( /^(Fixed|Flexible) downtime: /);
		}
		close( LOG) || die "Unable to close $logFile";
	}

	my $lastTriedPermutation = -1;
	unless( $noFixedTests) {
		my $fixedDowntimeTests = fixedDowntimeTestPermutations( $service);
		for( my $x = 0; $x < @$fixedDowntimeTests; $x++) {
			if( $testPaused) {
				print "Testing paused as requested.\n";
				print "Press ENTER to continue...";
				<STDIN>;
				$testPaused = 0;
			}
			$comment = join( ", ", @{ $fixedDowntimeTests->[ $x]});
			unless( isInArray( "Fixed downtime: $comment", \@skipTests)) {
				printf( "Fixed Test %d of %d\n", $x + 1, 
						scalar( @$fixedDowntimeTests));
				$schedule = scheduleDowntimePermutation( $service, $cfg, 
						$fixedDowntimeTests->[ $x], $version->{ "major"});
				$permutationTime = checkDowntime( "Fixed downtime: $comment", 
						$service, $cfg, 1, $schedule, $version->{ "major"});
				if( $permutationTime == -1) {
					if( $lastTriedPermutation != $x) {
						print "Downtime test failed: retrying\n";
						$lastTriedPermutation = $x;
						$x--;
					}
					else {
						die "Downtime test failed after retry\n";
					}
				}
				else {
					$lastTriedPermutation = $x;
					$elapsedTime += $permutationTime;
					printf( "Last test time: %s\n", 
							humanDuration( $permutationTime));
					printf( "Elapsed time thus far: %s\n", 
							humanDuration( $elapsedTime));
				}
			}
			else {
				$lastTriedPermutation = $x;
				printf( "Skipping fixed downtime: %s\n", $comment);
			}
		}
	}
	
	$lastTriedPermutation = -1;
	unless( $noFlexibleTests) {
		my $flexibleDowntimeTests = flexibleDowntimeTestPermutations( $service);
		for( my $y = 0; $y < @$flexibleDowntimeTests; $y++) {
			if( $testPaused) {
				print "Testing paused as requested.\n";
				print "Press ENTER to continue...";
				<STDIN>;
				$testPaused = 0;
			}
			$comment = join( ", ", @{ $flexibleDowntimeTests->[ $y]});
			unless( isInArray( "Flexible downtime: $comment", \@skipTests)) {
				printf( "Flexible Test %d of %d\n", $y + 1, 
						scalar( @$flexibleDowntimeTests));
				$schedule = scheduleDowntimePermutation( $service, $cfg, 
						$flexibleDowntimeTests->[ $y], $version->{ "major"});
				$permutationTime = checkDowntime( "Flexible downtime: $comment", 
						$service, $cfg, 0, $schedule, $version->{ "major"});
				if( $permutationTime == -1) {
					if( $lastTriedPermutation != $y) {
						print "Downtime test failed: retrying\n";
						$lastTriedPermutation = $y;
						$y--;
					}
					else {
						die "Downtime test failed after retry\n";
					}
				}
				else {
					$lastTriedPermutation = $y;
					$elapsedTime += $permutationTime;
					printf( "Last test time: %s\n", 
							humanDuration( $permutationTime));
					printf( "Elapsed time thus far: %s\n", 
							humanDuration( $elapsedTime));
				}
			}
			else {
				$lastTriedPermutation = $y;
				printf( "Skipping flexible downtime: %s\n", $comment);
			}
		}
	}
}