#!/usr/bin/perl
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
#NAME
#  Test - run a suite of tests
#
#SYNOPSIS
#  Test [name]..
#
#KEYWORDS
#  regression test
# 
#DESCRIPTION
#  This is a test generator.  The purpose is to automate  testing  of
#  software,  by supplying a number of test programs plus their input
#  and expected output.  This program runs the tests and  reports  on
#  their  success.   It is designed to be run from a makefile, so you
#  can just type "make test" and watch the tests run automatically.
#
#  Here is a typical makefile entry:
#  
#  test: tests
#  tests: Test Makefile $(PROGS) test/*
#  	-/bin/rm -f test/err/* test/out/* test/tmp*/* test/log/*
#  	Test
#  	touch tests
#
#  This program expects to find a subdirectory "test" which  contains
#  several subdirectories:
#    test/prog     programs to run.
#    test/check    programs to test output for validity.
#    test/stdin    programs' standard input, if any.
#    test/stdout   programs' standard output, if any.
#    test/stderr   programs' standard error output, if any.
#    test/out      receives programs' standard output.
#    test/err      receives programs' error output.
#
#  If there are command-line arguments, Test treats them as a list of
#  the  tests  to run.  For each arg "foo", Test check to see whether
#  test/prog/*foo* matches anything, and if so, executes all  of  the
#  matching tests (which are executable).
#
#  If there are no command-line arguments, Test script  runs  through
#  the  test/prog directory in alphabetical order, and executes every
#  executable file whose name does not start with a dot ('.').   Test
#  reports  the  success  or  failure  of  each  such  program on its
#  standard output.
#
#  Success or failure can be based on the program's exit  status,  or
#  on  comparing the program's output with known "correct" output, or
#  by providing a check program to validate the output.
#
#  If  it  is  important that tests be done in a special order, their
#  names should indicate the order.  The suggested convention  is  to
#  make  initial,  low-level  tests  have  names  like  test/prog/00,
#  test/prog/01, and so on.  After these have been run,  higher-level
#  tests  are  usually  independent of each other, so the name should
#  generally start with a "package" name,  such  as  test/prog/xyz00,
#  test/prog/xyz01, and so on.
#
#  If  there  is  a  test/stdin/foo file, the test/prog/foo program's
#  standard input will come from that file; otherwise it will be left
#  at the control terminal.  Thus test programs may interact with the
#  user in the normal fashion, or may have packaged input.
#
#  If the file test/stdout/xxx exists, then test/prog/xxx will be run
#  with its standard output redirected to test/out/xxx; otherwise its
#  standard output will go to the terminal.  Similarly, if  the  file
#  test/stderr/xxx  exists,  then  test/prog/xxx will be run with its
#  error output  redirected  to  test/err/xxx;  otherwise  its  error
#  output  will go to the terminal.  The output files in test/out and
#  test/err will be compared with the corresponding  test/stdout  and
#  test/stderr files, and if they are different, failure is reported.
#
#  If a program has a corresponding file in the test/check directory,
#  it will be run to check the program's output. Thus if the programs
#  test/prog/foo and test/check/foo both exists, we  will  first  run
#  test/prog/foo,  and  when it finishes, we will run test/check/foo,
#  which should generally look for test/out/foo and test/err/foo, and
#  verify  their  contents.   Thus, if test/check/foo exists, it is a
#  good idea to create test/stdout/foo and test/stderr/foo,  even  if
#  they  are null, so that test/check/foo can look at the output.  If
#  test/check/foo returns a  zero  status,  the  test  is  considered
#  successful; if not, the test is reported as having failed. In this
#  case, we ignore the exit status of test/prog/foo.
#
#  If  there is no test/check/foo, we use test/prog/foo's exit status
#  as an indication of success or failure.  If  the  exit  status  is
#  nonzero,  we  report the test as failed.  Otherwise we examine the
#  output files as follows.
#
#  We compare test/stdout/foo with test/out/foo, and  test/stderr/foo
#  with test/err/foo, if they exist.  If they differ, we include that
#  fact in the report, and consider the test  to  have  failed.   You
#  should  examine  the test/out and test/err files in such cases, to
#  determine the problem.
#
#  If neither test/stdout/foo nor test/stderr/foo exists, then we use
#  the  exit status of test/prog/foo as the sole indicator of success
#  or failure.
#
#  Test ignores other test/* subdirectories.  A recommended  practice
#  is to create test/doc/foo files for each test/prog/foo program, to
#  explain to users what the test does and how to deal with problems.
#
#  All the test programs are run with the test directory as the cwd.
#
#  First written by John Chambers 1990/9/17.  Modified  somewhat  for
#  assorted  projects  for assorted employers since then, and emailed
#  to assorted people on the Net who expressed an interest.
#
#AUTHOR
#  Copyright (C) 1990, 1995, 2002 by John Chambers.  This  script  is
#  made  freely  available  to  the public under the terms of the GPL
#  (GNU Public License).  You may use it for  your  own  testing  and
#  distribute  it  with  your software, as long as you give me credit
#  (and give credit to anyone who makes changes or improvements).
#
#  Send comments, suggestions, and improvements to:
#    To: <jc@trillian.mit.edu>  (preferred when this was written)
#    To: <jc@eddie.mit.edu>     (older address)
#    To: <jmchambers@rcn.com>   (commercial ISP, may change)
#    To: <jcsd@world.std.com>   (commercial ISP, may change)
#  The GPL requires that you share your improvements with the rest of
#  us.   However, you will probably make a few small changes for your
#  own projects.  Unless you think they are of wider interest,  don't
#  worry about sharing them.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

$| = 1;
($P = $0) =~ s".*/"";	# This program's name
$V = $ENV{"V_$P"} || 1;	# Our verbose level
chop($cwd = `pwd`);		# What directory we're in, for messages

die "$0: No $cwd/test directory.\n"      if (! -d "test");
die "$0: No $cwd/test/prog directory.\n" if (! -d "test/prog");

chdir 'test';		# Run everything from the test directory.
chop($cwd = `pwd`);	# For diagnostics
$ENV{'PATH'} = '.:..:' . $ENV{'PATH'};

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
if (! -d "out") {
	mkdir("out",0775)
		|| die "$0: Can't create $cwd/out [$!]\n";
}
if (! -d "err") {
	mkdir("err",0775)
		|| die "$0: Can't create $cwd/err [$!]\n";
}
if (! -d "tmp") {
	mkdir("tmp",0777)
		|| die "$0: Can't create $cwd/tmp [$!]\n";
}

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
for $a (@ARGV) {
	print "Arg: \"$a\"\n";
	@alist = <prog/*$a*>;
	die "$0: Arg \"$a\" doesn't match anything in $cwd/prog/\n"
		unless @alist;
	for $p (@alist) {
		$p =~ s'^prog/'';
		push @tests, $p;
	}

}
unless (@tests) {
	foreach $p (`ls prog`) {
		$p =~ s/\s+$//;
		push @tests, $p;
	}
}

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
program: 
foreach $p (sort @tests) {
	unless (-x ($prog = "prog/$p")) {
		print "$when ----- Test $p not executable\n";
		next program;
	}
	$when = &dt();
	print "$when ----- Test $p\n";
	if ($child = fork) {	# Parent
		waitpid($child,0);	# Wait for test program to quit
		$stat = $?;			# Note its exit status
		$code = ($stat >>8);
		$sgnl = ($stat & 0xFF);
		print "Process $child exited with code $code signal $sgnl (status $stat)\n" if $V>2;
	} else {					# Child
		if (-f "stdin/$p") {	# Does test suite have input for this test?
			open(STDIN,"<stdin/$p")
				|| die "$0: Can't read $cwd/stdin/$p [$!]\n";
		}
		if (-f "stdout/$p") {	# Does test suite have sample stdout?
			open(STDOUT,">out/$p")
				|| die "$0: Can't write $cwd/out/$p [$!]\n";
		}
		if (-f "stderr/$p") {	# Does test suite have sample stderr?
			open(STDERR,">err/$p")
				|| die "$0: Can't write $cwd/err/$p [$!]\n";
		}
		exec $prog;
		print STDERR "$0: Can't exec $prog [$!]\n";
		exit $!;				# Return exec's error code.
	}
	if (-x "$cwd/check/$p") {	# Is there a check program?
		print "Test $p check ...\n";
		if ($child = fork) {	# Parent.
			waitpid($child,0);	# Wait for check program to finish
		} else {				# Child.
			exec "check/$p";	# Run the check program
		}
		$stat = $?;
		$code = ($stat >>8);
		$sgnl = ($stat & 0xFF);
		print "Process $child exited with code $code signal $sgnl (status $stat)\n" if $V>2;
		if ($?) {print "Process $child exited with code $code signal $sgnl (status $stat)\n";
		} else  {print "Test $p success.\n";
		}
		next program;
	}
	# No check program for this one, so we check the stdout and stderr files.
	if (-f "stdout/$p") {	# Does it have a stdout file?
		$diff = `diff out/$p stdout/$p`;
		$stat = $?;
		$code = ($stat >>8);
		$sgnl = ($stat & 0xFF);
		print "Diff out/$p stdout/$p gave code $code signal $sgnl (status $stat)\n" if $V>2;
		if ($stat = $?) {		# No difference.
			print "Test $p $cwd/out/$p differs from $cwd/stdout/$p\n";
		}
	}
	if (-f "stderr/$p") {	# Does it have a stderr file?
		$diff = `diff err/$p stderr/$p`;
		$stat = $?;
		$code = ($stat >>8);
		$sgnl = ($stat & 0xFF);
		print "Diff err/$p stderr/$p gave code $code signal $sgnl (status $stat)\n" if $V>2;
		if ($stat = $?) {			# No difference.
			print "Test $p $cwd/err/$p differs from $cwd/stderr/$p\n";
		}
	}
	$when = &dt();
	if ($stat) {print "$when ----- Test $p failure: code $code signal $sgnl (status $stat).\n";
	} else     {print "$when ----- Test $p success.\n";
	}
}

exit 0;

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Return time stamp in ISO format.  Note that we return  GM/UTC,  not #
# local time. This is done so we can easily compare runs on different #
# machines that may be in different time zones.                       #
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
sub dt {
	($sec,$min,$hour,$mday,$mon,$year) = gmtime(time);
	return sprintf("%04d-%02d-%02d %02d:%02d:%02d",
		1900+$year,1+$mon,$mday,$hour,$min,$sec);
}