#! /bin/sh

# 06 July 2016

# This is the official template for pipetool regression test scripts.
# In addition to supporting the "SHORTTEST" option, this script also
# allows the user to run individual subtests from the command line.
# The script will accept a series of test identifiers, generally of
# the form "test1" "test2" ... which are to be run.

# Portions of the script which must be customized are marked with "!!",
# below.

# The complete list of tests must be placed in "alltests", !!3, below.
# The test[s] for the SHORTTEST must be placed in "shortlist", !!4 below.

# !!1
# sherpa.t
# test script for sherpa


# !!2
# syntax:
# sherpa.t [<testid> ... ]
 



######################################################################
# subroutine
# error_exit <message>
# Fatal error exit

error_exit()
{
  echo "$1" | tee -a $LOGFILE
  echo "${toolname} : FAIL" | tee -a $LOGFILE
  exit 1
}

######################################################################
# subroutine
# keyfilter infile outfile
# filters out CHECKSUM, Dataset, CREATOR, HISTORY, DATASUM, 
#             ASCDSVER, HISTNUM, and DATE
# To filter additional keywords, add s/KEYWORD/Dataset/g; for each.

keyfilter()
{
  cat $1 | sed -e 's/CHECKSUM/Dataset/g;s/COMMENT/Dataset/g;
  s/DATE/Dataset/g;s/CREATOR/Dataset/g;s/HISTORY/Dataset/g;
  s/DATASUM/Dataset/g;s/ASCDSVER/Dataset/g;s/HISTNUM/Dataset/g' | \
  grep -v Dataset > $2
  zerotest $2
}

######################################################################
# subroutine
# find_tool <toolname>
# checks that tool exists and is runnable

find_tool()
{
  s1=`type $1`
  s2=`echo $s1 | awk -F" " '{ print $3}'`
  if test -x $s2 ; then
    :
  else
    error_exit "tool $1 not found"
  fi
}

######################################################################
# subroutine
# zerotest <file> 
# Makes sure that file is not 0 length.
# Use this to protect yourself against empty files  (which will 
# 'diff' without error).  This can happen when the input file to
# cat $infile | do_something >> $outfile
# is missing.  This is used by keyfilter(), above.

zerotest()
{
 if test -s $1 ;
 then
   :
 else
   echo "ERROR: file $1 is of zero length" >> $LOGFILE
   #  Indicate failure, but do not exit.acis_build_badpix.t
   mismatch=0
 fi
}


######################################################################
# Initialization

# !!3
toolname="sherpa"

# set up list of tests
# !!4
alltests="test1"

# "short" test to run
# !!5
shortlist="test1"


# compute date string for log file
DT=`date +'%d%b%Y_%T'`

# check for directory environment variables
if test "x${TESTIN}" = "x" -o "x${TESTOUT}" = "x" -o "x${TESTSAV}" = "x" \
   -o "x${TESTLOG}" = "x" ; then
  error_exit "one or more of TESTIN/TESTOUT/TESTSAV/TESTLOG not defined" 
fi

# convenience definitions
OUTDIR=$TESTOUT/$toolname
SAVDIR=$TESTSAV/$toolname
INDIR=$TESTIN/$toolname
LOGDIR=$TESTLOG/$toolname

# set up log file name
LOGFILE=$LOGDIR/${toolname}_log.$DT

#get rid of old logs
rm -f $LOGDIR/${toolname}_log.*

# Any tests specified on command line?
if test $# -gt 0; then
  # yes, do those tests
  testlist=$*
else
  # No, see if we are to do "short" test
  if test "x$SHORTTEST" = "x" ; then
    # No, do everything
    testlist=$alltests
  else
    # yes, do short test
    testlist=$shortlist
  fi
fi


# Make sure we have a log directory
if test -d $LOGDIR ; then
 :
else
  mkdir -p $LOGDIR 
  if test $? -ne 0 ; then
    error_exit ""
  fi
fi


# Make sure we have an output directory
if test -d $OUTDIR ; then
 :
else
  mkdir -p $OUTDIR >> $LOGFILE 2>&1
  if test $? -ne 0 ; then
    error_exit "can't create output directory $OUTDIR"
  fi
fi


# announce ourselves
echo ""
echo "${toolname} regression" | tee $LOGFILE
echo ""

script_succeeded=0

######################################################################

# sherpa test specific setup

# Get sherpa location. Currently needed by py.test to find a 
# configuration file (sherpa/conftest.py). Should be unnecessary 
# eventually...

SHERPA_LOC=`python - <<EOF
import os
import sherpa
print(os.path.dirname(sherpa.__file__))
EOF`

######################################################################
# Begin per-test loop

for testid in $testlist
do
    
  # delete old outputs
  rm -f $OUTDIR/${testid}*

  # Set up file names
  outfile=$OUTDIR/${testid}.txt

  echo "" | tee -a $LOGFILE
  echo "running $testid" | tee -a $LOGFILE
  echo "" | tee -a $LOGFILE

  ####################################################################
  # run the tool
  case ${testid} in
    # !!6
    # SH-2: ignoring test that mocks chips
    test1 ) test1_string="py.test $SHERPA_LOC -k 'not test_chips_unit' 
                          --test-data ${INDIR} --verbose"
            echo $test1_string | tee -a  $LOGFILE 
	    eval $test1_string  2>&1 | tee -a $outfile
            ;;
  esac

  ####################################################################
  # check the outputs

  # if different tests need different kinds of comparisons, use a 
  #  case ${testid} in...  here

  # Unlike other CIAO regression tests, we do not compare the sherpa
  # test outputs to a baseline file as passing and failing tests are
  # handled in Python (with py.test). This script uses the exit 
  # status of the Python tests (0=pass, !0=fail) instead.

  # Known warnings are whitelisted in the sherpa Python tests in, so
  # we do not need to filter out warnings from the test output.

  # Did we get an error?
  if test ${PIPESTATUS[0]} -ne 0 ; then
  # Yes
      echo "ERROR: FAILURES in ${outfile}" >> $LOGFILE
      echo "${testid} NOT-OK"
      script_succeeded=1
  else
  # No
      echo "${testid} OK"
      script_succeeded=0
  fi

done

# end per-test loop
######################################################################
# report results

# blank line
echo ""

if test $script_succeeded -eq 0; then
    echo "${toolname} : PASS" | tee -a $LOGFILE
else
    echo "${toolname} : FAIL" | tee -a $LOGFILE
fi

echo "log file in ${LOGFILE}"


exit $script_succeeded
