#!/opt/conda/conda-bld/pyciao_1701809863847/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placeh/bin/python3
from __future__ import print_function
from pycrates import *
from history import HistoryRecord

import numpy as np
import sys
import os
import unittest
import testpycratesheader

# set the format datatype to use in astype() function (for printing byte strings)
if sys.version_info[0] < 3:
    tt = "S"
else:
    tt = "U"

class TestCrates:
    """
    Class to test the python Crates module.
    Arguments:
        indir   (optional)  <path>
          Directory containing input files
          default = "./" or  Environment variable 'TESTIN'
        outdir  (optional)  <path>
          Directory to write output files
          default = "./" or Environment variable 'TESTOUT'
        basedir (optional)  <path>
          Directory containing baseline files for comparison
          default = "./" or Environment variable 'TESTSAV'
        tests   (optional)  [list]
          List of test #s to run
          default = all tests
        log     (optional)  <True|False>
          Flag to control log file generation.
          True  = generate log as "<outdir>/cxcdm_verbose.txt"
          False = no log, output goes to stdout

    """
    def __init__(self, *args):
        self.indir  = "./"
        self.outdir = "./"
        self.basdir = "./"
        self.verbose_fh = sys.stdout
        self.runlog_fh = sys.stdout
        self._log   = True
        self._chklog= True
        self.verbose = 0
        self.anybad = False

        self.ntests = 21
        self.tests  = list(range(1,self.ntests+1))
        self.infiles = [
            "arf.fits",               #  0
            "image.fits",             #  1
            "pha.fits",               #  2
            "pha2.fits",              #  3
            "rmf.fits",               #  4
            "table.fits",             #  5
            "vector_array.fits",      #  6
            "vector.fits",            #  7
            "vector2.fits",           #  8
            "single_row_table.fits",  #  9
            "fov1.fits",              # 10
            "cube.fits",              # 11
            "img1d.fits",             # 12
            "varlen.fits",            # 13
            "blah.dtf",               # 14
            "history.fits",           # 15
            "reg.fits",               # 16
            "pha_blankWCS.fits",      # 17 test_21
            "pha_identWCS.dtf",       # 18 test_22
            ]

        self.outfiles = [
            "pycrates_m001.fits",  # empty file
            "pycrates_m002.fits",  # no longer empty file
            "pycrates_m003.fits",  # file w/ keyword removed
            "pycrates_m004.fits",  # copy table
            "pycrates_m005.fits",  # append new crate, remove a crate from dataset
            "pycrates_m006.fits", 
            "pycrates_m007.fits",
            "pycrates_m008.fits",
            "pycrates_m009.fits",
            "pycrates_m010.fits",
            "pycrates_m011.fits",
            "pycrates_m012.fits",
            "pycrates_m013.fits",
            "pycrates_m014.fits",
            "pycrates_pha_01.fits", # outfiles index 14
            "pycrates_pha_02.fits", # outfiles index 15
            "pycrates_pha_03.fits", # outfiles index 16
            "pycrates_pha_04.fits", # outfiles index 17
            "pycrates_m015.fits",   # outfiles index 18
            "pycrates_m016.fits",
            "pycrates_m017.fits",
            "pycrates_m018.fits",
            "pycrates_m019.fits",  # history and comments, plus updated histnum 
            "pycrates_m020.fits",  # no history written out 
            "pycrates_m021.fits",  # history/comments in update mode
            "pycrates_m022.fits",  # write_dataset test
            "pycrates_m023.fits",  # test reading empty dataset where all values filtered out
        ]


        # Check for environment variable directory defaults
        if 'TESTIN' in os.environ:
            self.indir = os.environ['TESTIN']
            self.indir += "/test_pycrates"

        if 'TESTOUT' in os.environ:
            self.outdir = os.environ['TESTOUT']
            self.outdir += "/pycrates"

        if 'TESTSAV' in os.environ:
            self.basdir = os.environ['TESTSAV']
            self.basdir += "/pycrates"

        cnt = 0
        # Parse and process any arguments
        for item in args:
            if item.count('=') == 1:
                (key,val)=item.split('=')
                if key == 'indir':
                    self.indir = val
                elif key == 'outdir':
                    self.outdir = val
                elif key == 'basdir':
                    self.basdir = val
                elif key == 'tests':
                    tlist = val.split(',')
                    self.tests = [ int(tnum) for tnum in tlist ]
                    self._chklog = False
                elif key == 'log':
                    if val == 'True':
                        self._log = True
                    else:
                        self._log = False
                elif key == 'checklog':
                    if val == 'True':
                        self._chklog = True
                    else:
                        self._chklog = False
                elif key == 'verbose':
                    self.verbose = int(val)
                else:
                    raise TypeError ("TestCrates() got an unexpected keyword argument '%s'" % key)
            else:
                if cnt == 0:
                    self.indir = item
                elif cnt == 1:
                    self.outdir = item
                elif cnt == 2:
                    self.basdir = item
                elif cnt == 3:
                    self.verbose = int(item)

            cnt = cnt + 1


        if not self.indir.endswith('/'):
            self.indir = self.indir + '/'
        if not self.outdir.endswith('/'):
            self.outdir = self.outdir + '/'
        if not self.basdir.endswith('/'):
            self.basdir = self.basdir + '/'

        # Add test bin to PATH for output comparison
        path = os.environ['PATH']
        testbin = os.environ['ASCDS_INSTALL'] + "/test/bin/pycrates"
        os.environ['PATH'] = path + ":" + testbin
        path = os.environ['PATH']

        # Clean up existing output files
        self.cleanup()


    def __repr__(self):
        buf =  "pycrates Test Suite:\n"
        buf += "------------------ \n"
        buf += "   Indir  = %s\n" % self.indir
        buf += "   Outdir = %s\n" % self.outdir
        buf += "   Basdir = %s\n" % self.basdir
        buf += "   NTests = %d\n" % len(self.tests)
        buf += "\n"
        return buf


    def cleanup(self):
        """
        Delete output files from 'outdir' if present.
        """

        if os.path.exists( self.outdir ):
            for file in self.outfiles:
                path = self.outdir + file
                if os.path.isfile( path ):
                    os.system("rm " + path)
        else:
            os.system("mkdir " + self.outdir)
            

    def run(self):
        """
        Loop over tests to execute and run them.
        Report PASS|FAIL for each test.
        Returns overall grade for tests run (PASS|FAIL)
        """
        if self.verbose >= 1:
            print (self)

        # Open Log file for verbose output
        if self._log == True:
            file = self.outdir + "crates_verbose.txt"
            self.verbose_fh = open( file, 'w')

            file2 = self.outdir + "crates_run_log.txt"
            self.runlog_fh = open( file2, 'w')

            if self.verbose >= 1:
                print ("Running pycrates tests...")

            print("Running pycrates tests...", file=self.runlog_fh)


        for ii in self.tests:
            try:
                exec ('self.test_%d ()'%ii)

            except Exception as e:
                print(("  ", e))
                grade = "FAIL"
                self.anybad = True
            else:
                grade = "PASS"

            if self.verbose > 0:
                print(("  test_%d: %s" % (ii,grade)))
            print("  test_%d: %s" % (ii,grade), file=self.runlog_fh)

            if not self._log:
                print() 
                
        if self._log == True:
            self.verbose_fh.close()

        if self.anybad:
            return "FAIL"
        else:
            return "PASS"



    def check_file(self, file ):
        """
        Will find matching file in 'basedir' and compare.
          file = full path to file being checked

        Returns True if OK, False otherwise
        """
        result = False
        tfile  = self.outdir + "tmp.txt"

        # check specified file exists
        if os.path.isfile( file ):
            fname = os.path.basename( file )
            basefile = self.basdir + fname
        else:
            raise IOError("check_file(): Specified file does not exist %s" % file)

        if self.verbose >= 1:
            print(("Checking file: %s" % fname))

        print("Checking file: %s " % fname, file=self.runlog_fh)

        # convert provided file to ASCII DTF format
        tfile1 = file+ "tmp_new.dtf"
        comstr = "dmcopy infile=%s outfile=%s'[opt kernel=text/dtf]' opt=all clobber=yes" % (file, tfile1)

        if os.system( comstr ):
            raise RuntimeError("check_file(): Problem converting file for comparison %s" % file)

        # convert baseline file to ASCII DTF format
        tfile2 = file+ "tmp_bas.dtf"
        comstr = "dmcopy infile=%s outfile=%s'[opt kernel=text/dtf]' opt=all clobber=yes" % (basefile, tfile2)

        if os.system( comstr ):
            raise RuntimeError("check_file(): Problem converting file for comparison %s" % basefile)

        # filter out stuff that changes.
        comstr = "cat %s | egrep -v \"ASCDSVER|CHECKSUM|DATASUM|HISTORY|HISTNUM\" > %s ; mv %s %s" % ( tfile1, tfile, tfile, tfile1 )
        os.system( comstr )
        comstr = "cat %s | egrep -v \"ASCDSVER|CHECKSUM|DATASUM|HISTORY|HISTNUM\" > %s ; mv %s %s" % ( tfile2, tfile, tfile, tfile2 )
        os.system( comstr )

        # compare the ascii files.
        comstr = "diff %s %s > /dev/null" % (tfile1, tfile2)
        if os.system( comstr ):
            result = False
        else:
            result = True

        if os.path.isfile( tfile1 ):
            os.system("rm " + tfile1 )
        if os.path.isfile( tfile2 ):
            os.system("rm " + tfile2 )
                
        if result == True:
            if self.verbose >= 1:
                print ("   OK")
            print("  OK", file=self.runlog_fh)
        else:
            if self.verbose >= 1:
                print ("   Problem")
            print("  Problem", file=self.runlog_fh)

        return result


    def verify(self):
        if self.verbose >= 1:
            print("Verifying pycrates tests...")

        print("Verifying pycrates tests...", file=self.runlog_fh)

        result = True
        grade = ""

        try:
            for ii in range(len(self.outfiles)):
                try:
                    result = self.check_file(self.outdir + self.outfiles[ii])
                except IOError:
                    pass

                if result is False:
                    self.anybad = True

        except RuntimeError as e:
            print(("  ", e))
            self.anybad = True
            pass

        # check log file
        if self._chklog == True:
            if self.check_log() == False:
                self.anybad = True

        if self.anybad:
            grade = "FAIL"
        else:
            grade = "PASS"
            self.cleanup()

        if self.verbose > 0:  
            print()
            print(("pycrates Tests: %s" % grade))

        print("", file=self.runlog_fh)
        print("pycrates Tests: \n%s" % grade, file=self.runlog_fh)
           
        if self._log == True:
            self.runlog_fh.close()

        return grade


    def test_1( self ):
        print("\nRunning Test 1 - Bad file input", file=self.verbose_fh)

        try:
            print("Try to open a non-existent file...", file=self.verbose_fh)
            tab = TABLECrate("bogus.fits")
        except IOError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        try:
            print("Try to open an existing file with bad rw mode argument...", file=self.verbose_fh)
            tab = TABLECrate("table.fits", mode='q')
        except AttributeError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        try:
            print("Try to open an existing file with bad syntax...", file=self.verbose_fh)
            tab = TABLECrate("table.fits[ccd_id=3")
        except IOError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass


    def test_2( self ):
        print("\nRunning Test 2 - Create an empty file.", file=self.verbose_fh)

        outfile = self.outdir + self.outfiles[0]

        tab = TABLECrate()
        print(tab, file=self.verbose_fh)

        try:
            print("Try to write file without a destination...", file=self.verbose_fh)
            tab.write()
        except IOError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        print("Write file to " + outfile, file=self.verbose_fh)        
        tab.write(outfile, clobber=True)


    def test_3( self ):
        print("\nRunning Test 3 - Populate empty file with one key and one column.  Write file in place.", file=self.verbose_fh)
        # Assign input/output file names

        infile  = self.outdir + self.outfiles[0]
        outfile = self.outdir + self.outfiles[1]
        
        if os.path.isfile(infile) is False:
            os.system("cp " + self.basdir + self.outfiles[0] + " " + infile)
  
        # copy input file to output location for update
        os.system("cp " + infile + " " + outfile)
        
        tab = TABLECrate(outfile)

        # create new key and add to table
        print("Create new key and add to table", file=self.verbose_fh)
        key = CrateKey( ("ORIGIN", "CXC", "foo", "bar") )
        print("Key Info:  ",file=self.verbose_fh)
        print(key, file=self.verbose_fh)

        # test bad input index
        try:
            tab.add_key(key, index=300)
        except IndexError as ex:
            pass
       
        tab.add_key(key)

        # create new column and add to table
        print("Create new column and add to table", file=self.verbose_fh)
        col = CrateData()
        col.name = "COLUMN1"
        col.values = np.array([[[10,20,30],
                                [11,22,33],
                                [100,200,300]]])
        col.unit = None
        col.desc = None
        print("Column Info:  ", file=self.verbose_fh)
        print(col, file=self.verbose_fh)

        # test bad input index
        try:
            tab.add_column(col, index=5)
        except IndexError as ex:
            pass

        tab.add_column(col)

        # create another new column and add to table
        print("Create another new column without a name and add to table", file=self.verbose_fh)
        col2 = CrateData()
        col2.name = ""
        col2.values = np.array([875,234,258])
        
        print("Column Info:  ", file=self.verbose_fh)
        print(col2, file=self.verbose_fh)

        tab.add_column(col2)

        print(tab.get_colnames(), file=self.verbose_fh)

        print("Write file in place", file=self.verbose_fh)

        tab.write()

        # another bad input test for add_col
        # placing this after the table write() so as not to impact the output file
        x = CrateData()
        x.name = '(x)'
        tab.add_column(x)
        print(tab.COLUMN3, file=self.verbose_fh) 


    def test_4( self ):
        print("\nRunning Test 4 - Read table file w/ one block, delete keyword, and write file in place", file=self.verbose_fh)

        infile = self.indir + self.infiles[5]
        outfile = self.outdir + self.outfiles[2]

        # copy input file to output location for update
        os.system("cp " + infile + " " + outfile)

        tab = TABLECrate( input= outfile, mode="rw" )
        print("Open table:", file=self.verbose_fh)
        print(tab, file=self.verbose_fh)

        ds = tab.get_dataset()
        print("Parent Dataset:", file=self.verbose_fh)
        print(ds, file=self.verbose_fh)


        print("Has table been modified?  ", tab.is_modified(), file=self.verbose_fh)        
        print("Table signature:", file=self.verbose_fh)
        print(tab.get_signature(), file=self.verbose_fh)

        print("", file=self.verbose_fh)
        print("Does keyword FOO exist?", tab.key_exists("FOO"), file=self.verbose_fh)
        print("Does keyword OBS_ID exist?", tab.key_exists("obs_ID"), file=self.verbose_fh)
        print("OBS_ID value = ", tab.get_key_value("Obs_ID"), file=self.verbose_fh)
        print("Removing OBS_ID keyword", file=self.verbose_fh)
        tab.delete_key("OBS_ID")
        print("Does keyword OBS_ID exist?", tab.key_exists("obs_id"), file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Has table been modified?  ", tab.is_modified(), file=self.verbose_fh)        
        print("Table signature:", file=self.verbose_fh)     
        print(tab.get_signature(), file=self.verbose_fh)

        print("Print the SKY column", file=self.verbose_fh)
        print(tab.get_column("sky"), file=self.verbose_fh)
        print(tab.sky.values, file=self.verbose_fh)
        
        print("Modify SKY values", file=self.verbose_fh)
        tab.sky.values[0][1] = 99999.99
        print(tab.sky.values, file=self.verbose_fh)
        
        print("Has file been modified?  ", tab.is_modified(), file=self.verbose_fh)

        print("Write file in place", file=self.verbose_fh)
        tab.write(clobber=True)



    def test_5( self ):

        print("\nRunning Test 5 - Read table file with filter and copy", file=self.verbose_fh)
        
        infile = self.indir + self.infiles[7]
        outfile = self.outdir + self.outfiles[3]

        os.system("touch " + outfile)

        tab = TABLECrate( input= infile+"[EVENTS]", mode="rw" )

        print("Filename:  ", tab.get_filename(), file=self.verbose_fh)

        print("Open table:", file=self.verbose_fh)
        print(tab, file=self.verbose_fh)

        print("Read in data", file=self.verbose_fh)
        tab._read_data()
        
        print("Print scalar column", file=self.verbose_fh)
        print(tab.pha, file=self.verbose_fh)
        print(tab.pha.values, file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Print array column", file=self.verbose_fh)
        print(tab.phas, file=self.verbose_fh)
        print(tab.phas.values, file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Print vector column", file=self.verbose_fh)
        print(tab.sky, file=self.verbose_fh)
        print(tab.sky.values, file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Print virtual vector column", file=self.verbose_fh)
        print(tab.get_column("MSC"), file=self.verbose_fh)
        print(tab.get_column("msc").values, file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Print virtual vector column", file=self.verbose_fh)
        print(tab.get_column("EQPOS"), file=self.verbose_fh)
        print(tab.get_column("EQPOS").values, file=self.verbose_fh)
        print("", file=self.verbose_fh)


        print("Try to write in place", file=self.verbose_fh)
        try:
            print("Is writeable?  ", tab.get_dataset().is_writeable(), file=self.verbose_fh)
            tab.write()
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass
        

        print("Write (copy) EVENTS table to existing output file", file=self.verbose_fh)

        print("Setting clobber to False", file=self.verbose_fh)
        try:
            tab.write(outfile, clobber=False)
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        print("Setting clobber to True", file=self.verbose_fh)
        tab.write(outfile, clobber=True)
        

    def test_6( self ):

        print("\nRunning Test 6 - Open a dataset with more than one table, append a crate from another dataset, delete a crate, and write in place  ", file=self.verbose_fh)

        infile1 = self.indir + self.infiles[7]
        infile2 = self.indir + self.infiles[0]
        outfile = self.outdir + self.outfiles[4]

        os.system("cp " + infile1 + " " + outfile)

        print("Read in first dataset", file=self.verbose_fh)
        ds1 = CrateDataset(outfile)
        print(ds1, file=self.verbose_fh)

        print("Read in second dataset", file=self.verbose_fh)
        ds2 = CrateDataset(infile2)
        print(ds2, file=self.verbose_fh)

        tab2 = ds2.get_crate(2)

        print("Add crate to first dataset", file=self.verbose_fh)
        ds1.add_crate(tab2)
        print(ds1, file=self.verbose_fh)

        print("Delete crate from first dataset", file=self.verbose_fh)
        ds1.delete_crate("GtI7")
        print(ds1, file=self.verbose_fh)

        # check that PRIMARY NULL header is updated
        key = CrateKey( ("FOO", "BAR", "Blah", "Blah Description") )
        ds1.get_crate("PRIMARY").add_key( key )

        print("Write file in place", file=self.verbose_fh)
        ds1.write()


    def test_7( self ):

        print("\nRunning Test 7 - Examine file with vector array and virtual vector array columns", file=self.verbose_fh)

        infile = self.indir + self.infiles[6]
        outfile1 = self.outdir + self.outfiles[5]
        outfile2 = self.outdir + self.outfiles[6]

        os.system("cp " + infile + " " + outfile1)
        
        ds = CrateDataset(outfile1)
        print(ds, file=self.verbose_fh)

        tab = ds.get_crate(2)

        print("Print column names (vectors=True, rawonly=True)", file=self.verbose_fh)        
        print(tab.get_colnames(), file=self.verbose_fh)
        print("", file=self.verbose_fh)

        print("Print column names (vectors=False, rawonly=False)", file=self.verbose_fh)
        print(tab.get_colnames(vectors=False, rawonly=False), file=self.verbose_fh)
        print("", file=self.verbose_fh)


        print("Print column names (vectors=True, rawonly=False)", file=self.verbose_fh)
        print(tab.get_colnames(vectors=True, rawonly=False), file=self.verbose_fh)
        print("", file=self.verbose_fh)
        
        print("Print column names (vectors=False, rawonly=True)", file=self.verbose_fh)
        print(tab.get_colnames(vectors=False, rawonly=True), file=self.verbose_fh)
        print("", file=self.verbose_fh)


        print("Print vector array column", file=self.verbose_fh)
        print(tab.POS, file=self.verbose_fh)
        print(tab.POS.values, file=self.verbose_fh)
        print("", file=self.verbose_fh)


        print("Print virtual vector array column", file=self.verbose_fh)
        print(tab.get_column(b"EQsrc"), file=self.verbose_fh)
        print(tab.get_column(u"eqsrc").values, file=self.verbose_fh)
        print("", file=self.verbose_fh)


        print("Removing column COMPONENT", file=self.verbose_fh)
        tab.delete_column(b"COMPONENT")  ## delete_column is case-sensitive

        print("Write file in place", file=self.verbose_fh)
        ds.write()

        print("Write file to " + outfile2, file=self.verbose_fh)
        ds.write(outfile2, clobber=True)


    def test_8( self ):
        print("\nRunning Test 8 - Image", file=self.verbose_fh)

        infile = self.indir + self.infiles[1]
        outfile1 = self.outdir + self.outfiles[7]
        outfile2 = self.outdir + self.outfiles[8]
        
        os.system("cp " + infile + " " + outfile1)

        img = IMAGECrate(input=outfile1, mode="rw")
        img.update_signature()
        print(img, file=self.verbose_fh)
        print("Image Data:", file=self.verbose_fh)
        print(img.get_image(), file=self.verbose_fh)
        print("Image signature:  ", img.get_signature(), file=self.verbose_fh)

        print("Maximum image array value:  ", img.get_max_value(), file=self.verbose_fh)
        print("Minimum image array value:  ", img.get_min_value(), file=self.verbose_fh)
        print("Image array shape:          ", img.get_shape(), file=self.verbose_fh)
        
        print("Ncoords:    ", img.get_ncoords(), file=self.verbose_fh)
        print("Axisnames:  ", img.get_axisnames(), file=self.verbose_fh)

        pos = img.get_axis(b"POS")

        print(pos, file=self.verbose_fh)
        print(pos.values, file=self.verbose_fh)
        print("", file=self.verbose_fh)

        eqpos = img.get_axis(u"EQPOS")
        print(eqpos, file=self.verbose_fh)
        print(eqpos.values, file=self.verbose_fh)

        print("", file=self.verbose_fh)

        print("Try to get a non-existent axis", file=self.verbose_fh)        
        try: 
            foo = img.get_axis(u"FOO")
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        print("Try get_transform() with bad input", file=self.verbose_fh) 
        try:
            tr = img.get_transform(123)
        except TypeError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass
        except:
            raise

        print("Try get_transform() with non-existent axis", file=self.verbose_fh) 
        try:
            tr = img.get_transform("EQP")
        except KeyError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass
        except:
            raise


        print("Get POS transform and parameter list: ", file=self.verbose_fh)
        tr = img.get_transform(b"POS")
        matrix = tr.get_transform_matrix()
        print(matrix, file=self.verbose_fh)

        print(tr.print_parameter_list(), file=self.verbose_fh)
        
        print("", file=self.verbose_fh)
        print("Create a new image", file=self.verbose_fh)
        piximg = CrateData()
        piximg.name = "image"
        piximg.values = np.array([[10,20,30],
                                [11,22,33],
                                [100,200,300]])

        img.delete_image()
        img.add_image(piximg)

        print(img.get_image(), file=self.verbose_fh)
       
        print("Replace image data", file=self.verbose_fh)

        print("New image data: ", file=self.verbose_fh)
        print(img.get_image(), file=self.verbose_fh)

        img.update_signature()
        print("Updated signature:  ", img.get_signature(), file=self.verbose_fh)

        print("Write in place, sending its own filename as input", file=self.verbose_fh)
        fname = img.get_filename()
        img.write(fname, clobber=False)
        
        print("Write image to a different file", file=self.verbose_fh)
        img.write(outfile2, clobber=True)

        try:
            print("Try writing to same file with clobber set to False", file=self.verbose_fh)
            img.write(outfile2)
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass

        print("Read and write image/transform with all values filtered out", file=self.verbose_fh)
        empty_img = IMAGECrate(self.indir + self.infiles[1] + "[pos=region(" + self.indir + self.infiles[16] + ")]")
        empty_img.write(self.outdir + self.outfiles[26], clobber=True)


    def test_9( self ):
        print("\nRunning Test 9 - RMFCrateDataset test", file=self.verbose_fh)
        
        infile = self.indir + self.infiles[4]
        outfile = self.outdir + self.outfiles[9]

        rmf = RMFCrateDataset(infile)
        print(str(rmf), file=self.verbose_fh)

        rmf.write(outfile)
        try:
            rmf.write(outfile)
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass


    def test_10( self ):
        print("\nRunning Test 10 - PHACrateDataset test for PHA:TypeI", file=self.verbose_fh)
        
        infile = self.indir + self.infiles[2]
        outfile = self.outdir + self.outfiles[10]

        pha1 = PHACrateDataset(infile)
        print(pha1, file=self.verbose_fh)

        print("Is PHA Type:I ?  ", pha1.is_pha_type1(), file=self.verbose_fh)
        print("Is PHA Type:II ?  ", pha1.is_pha_type2(), file=self.verbose_fh)

        pha1.write(outfile, clobber=True)


    def test_11( self ):
        print("\nRunning Test 11 - PHACrateDataset test for PHA:TypeII", file=self.verbose_fh)
        
        infile = self.indir + self.infiles[3]
        outfile1 = self.outdir + self.outfiles[11]
        outfile2 = self.outdir + self.outfiles[12]

        pha2 = PHACrateDataset(infile)
        print(pha2, file=self.verbose_fh)

        print("Is PHA Type:I ?  ", pha2.is_pha_type1(), file=self.verbose_fh)
        print("Is PHA Type:II ?  ", pha2.is_pha_type2(), file=self.verbose_fh)

        print("Writing PHA Type:II dataset as Type:II", file=self.verbose_fh)
        pha2.write(outfile1, clobber=True)

        print("Writing row 2 of a PHA Type:II dataset as a Type:I dataset", file=self.verbose_fh)
        pha2.write_pha1_dataset(2, outfile2, clobber=True)

        print("Writing all rows of a PHA Type:II dataset as a Type:I dataset", file=self.verbose_fh)
        root = self.outdir + "pycrates_pha"
        pha2.write_pha1_files( root, clobber=True)


    def test_12( self ):
        print("\nRunning Test 12 - Crates HLUI tests", file=self.verbose_fh)

        #--------------------------------------------------------------------------

        print("\nCreate empty TABLECrate.....", file=self.verbose_fh)
        cr = TABLECrate()
        
        print("Create CrateKey.....", file=self.verbose_fh)
        key = CrateKey()
        key.name = b"FOO"
        key.value = 3.0
        key.unit = u"bar"
        key.desc = b"blah"
        
        print(key, file=self.verbose_fh)
        
        print("Add key to TABLECrate.....", file=self.verbose_fh)
        add_key(cr, key)
        
        print("Key exists? ", key_exists(cr, key.name), file=self.verbose_fh)
        
        print("\nSet key value.....", file=self.verbose_fh)
        set_keyval(cr, key.name, key.value)
        print(key, file=self.verbose_fh)
        
        print("\nPrint key names in TABLECrate.....", file=self.verbose_fh)
        print(print_key_names(cr), file=self.verbose_fh)
        
        print("\nGet key.....", file=self.verbose_fh)
        samekey = get_key(cr, key.name)
        print(samekey, file=self.verbose_fh)
        
        print("Change key name.....", file=self.verbose_fh) 
        samekey.name = u"FOOTWO"
        print(key, file=self.verbose_fh)
        
        print("\nDelete key.....", file=self.verbose_fh)
        delete_key(cr, 0)
        print("Key exists? ", key_exists(cr, key.name), file=self.verbose_fh)
        
        #--------------------------------------------------------------------------

        print("\nCreate CrateData.....", file=self.verbose_fh)
        myData = CrateData()
        
        myData.name = u"energy"
        myData.unit = b"keV"
        myData.desc = u"photon energy"
        
        larr2 = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9)

        print(myData, file=self.verbose_fh)

        print("Add column to TABLECrate.....", file=self.verbose_fh)
        add_col(cr, myData)
        print("Column exists? ",col_exists(cr, myData.name), file=self.verbose_fh)

        print("\nSet column values.....", file=self.verbose_fh)
        set_colvals(cr, myData.name, larr2)
        print(myData, file=self.verbose_fh)

        print("\nSet the Crate and CrateData checksums.....", file=self.verbose_fh)
        update_crate_checksum(cr)
        update_cratedata_checksum(myData)
        update_all_cratedata_checksums(cr)
        print("Crate checksum:  ", cr.get_signature(), file=self.verbose_fh)
        print("CrateData checksum:  ", myData.get_signature(), file=self.verbose_fh)
        
        print("\nGet column values.....", file=self.verbose_fh)
        vals = get_colvals(cr, myData.name)
        print(vals, file=self.verbose_fh)
        
        print("\nModify column values.....", file=self.verbose_fh)
        vals[1] = 99
        vals[5] = 99
        print(vals, file=self.verbose_fh)
        
        print("\nHas the CrateData been modified?  ", check_cratedata_modified(myData), file=self.verbose_fh)
        print(get_cratedata_checksum(myData), file=self.verbose_fh)
        
        print("\nGet number of cols: ", get_number_cols(cr), file=self.verbose_fh)
        
        print("\nPrint column names in TABLECrate.....", file=self.verbose_fh)
        print(print_col_names(cr), file=self.verbose_fh)
        
        print("\nDelete column.....", file=self.verbose_fh)
        delete_col(cr, myData.name)
        print("Column exists? ", col_exists(cr, myData.name), file=self.verbose_fh)
        
        print("\nHas the crate been modified?  ", check_crate_modified(cr), file=self.verbose_fh)
        print(get_crate_checksum(cr), file=self.verbose_fh)

        #--------------------------------------------------------------------------

        print("\nRead in vector file.....", file=self.verbose_fh)

        basefile  = self.indir  + 'vector.fits'
        infile =  self.outdir  + 'vector.fits'
        outfile = self.outdir  + self.outfiles[25]

        os.system('cp ' + basefile + " " + infile )

        ds = read_dataset(infile)

        print(ds, file=self.verbose_fh)
        print("Delete crate GTI7 from vector file.....", file=self.verbose_fh)
        delete_crate(ds, "GTI7")
        print(ds, file=self.verbose_fh)
        
        print("Get EVENTS crate from vector file.....", file=self.verbose_fh)
        veccrate = get_crate(ds, "EVENTS")

        print("File type: ", get_crate_type(veccrate), file=self.verbose_fh)
        
        print("Number of rows:  ", get_number_rows(veccrate), file=self.verbose_fh)
        
        print(print_key_names(veccrate), file=self.verbose_fh)
        print("\nNumber of columns:  ", get_number_cols(veccrate), file=self.verbose_fh)
        
        print("\nShow Regular columns: ", file=self.verbose_fh)
        print(print_col_names(veccrate), file=self.verbose_fh)
        
        print("\nShow Regular and Virtual columns:  ", file=self.verbose_fh)
        print(print_col_names(veccrate, True, False), file=self.verbose_fh)
        
        print("\nGet transform.....", file=self.verbose_fh)
        trans = get_transform(veccrate, b"EQPOS")
        
        print("Get transform matrix.....", file=self.verbose_fh)
        mat = get_transform_matrix(veccrate, u"EQPOS")
        print(mat, file=self.verbose_fh)
        
        print("\nGet SKY column...", file=self.verbose_fh)
        sky = get_col(veccrate, b"SKY")
        print(sky, file=self.verbose_fh)
        
        print("\nSKY is Virtual?  ", is_virtual(veccrate, u"SKY"), file=self.verbose_fh)
        skyvals = copy_colvals(veccrate, b"SKY")
        print("\nGet a COPY of SKY values:", file=self.verbose_fh)
        print(skyvals, file=self.verbose_fh)
        
        skyvals[0][0] = 4444
        skyvals[0][1] = 95555
        
        print("\nUpdate SKY values [x1,y1]:", file=self.verbose_fh)
        print(skyvals, file=self.verbose_fh)
        
        print("\nShow X and Y values; should be the same as original SKY component values...\n", file=self.verbose_fh)
        print("X vals: ", file=self.verbose_fh)
        print(get_colvals(veccrate, b"X"), file=self.verbose_fh)
        print("Y vals: ", file=self.verbose_fh)
        print(get_colvals(veccrate, u"Y"), file=self.verbose_fh)

        print("\nSet all X component values for SKY vector to zero...",
              file=self.verbose_fh)
        x = copy_colvals(veccrate, b"X")
        print(x, file=self.verbose_fh)
        zeros = np.zeros_like( get_colvals(veccrate, b"X")) 
        set_colvals(veccrate, b"X", zeros)
        iszerod = np.all(get_colvals(veccrate, b"X")==0)
        zerox = get_colvals(veccrate, b"X")
        print("\nIs X zerod? {}".format(iszerod),
              file=self.verbose_fh)
        print(zerox, file=self.verbose_fh)
        print("\nReset zeroed X component values to original...",
              file=self.verbose_fh)
        set_colvals(veccrate, b"X", x)
        orig = np.all(get_colvals(veccrate, b"X") == x)
        x = copy_colvals(veccrate, b"X")
        print("\nIs X back to original values? {}".format(orig),
              file=self.verbose_fh)
        print(x, file=self.verbose_fh)

        print("\nGet EQPOS column...", file=self.verbose_fh)
        eq = get_col(veccrate, b"EQPOS")
        print("EQPOS is Virtual?  ", is_virtual(veccrate, u"EQPOS"), file=self.verbose_fh)
        eqvals = get_colvals(veccrate, b"EQPOS")
        print(eqvals, file=self.verbose_fh)
        
        print("\nGet RA column values...", file=self.verbose_fh)
        ravals = get_colvals(veccrate, u"RA")
        print(ravals, file=self.verbose_fh)
        
        print("\nGet DEC column values...", file=self.verbose_fh)
        decvals = get_colvals(veccrate, b"DEC")
        print(decvals, file=self.verbose_fh)
        
        print("\nWrite Dataset...", file=self.verbose_fh)
        write_dataset(ds, outfile, clobber=True)

        #--------------------------------------------------------------------------
        print("\nPrint History", file=self.verbose_fh)
        print(print_history(veccrate), file=self.verbose_fh)

        print("\nDelete CALDBVER keyword", file=self.verbose_fh)
        delete_key(veccrate, "CALDBVER")

        if "CALDBVER" in str(get_key_names(veccrate)):
            raise Exception("delete CALDBVER keyword FAILED")

        print(get_key_names(veccrate), file=self.verbose_fh)

        print("\nAdd Comments", file=self.verbose_fh)

        try:
            add_comment(None, ["This is a comment record"])
        except ValueError:
            pass
        else:
            raise Exception("check invalid Crate FAILED")

        add_comment(veccrate, None)
        add_record(veccrate, "COMMENT", ["Small block of comment lines"])
        add_record(veccrate, "COMMENT", "to test the Crates add_record() method")
        add_record(veccrate, "COMMENT", "which adds them to a file")

        print("\nAdd History", file=self.verbose_fh)
        
        try:
            add_history(None, ["This is a non-ASCFITS compliant history record"])
        except ValueError:
            pass
        else:
            raise Exception("check invalid Crate FAILED")


        add_record(veccrate, "HISTORY", ["This is a non-ASCFITS compliant history record"])
        
        rec = HistoryRecord( tool="dmsomething", date="2017-12-01T14:00:00", param_names=["infile", "outfile", "tol"], param_values=["in.fits", "out.fits", "tolerance"], asc_start = 34)
        add_history(veccrate, rec)

        add_record(veccrate, "HISTORY", rec.as_FITS() )


        histlist = ["HISTORY  TOOL  :dmcopy  2016-08-11T20:27:22                             ASC00041",
                    "HISTORY  PARM  :infile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____5ASC00042",
                    "HISTORY  CONT  :87231988n944/output/acisf05017_000N022_r0001b_rayevt3.fiASC00043",
                    "HISTORY  CONT  :ts[sky=bounds(region(/export/proc/tmp.ascdsl3/prs_run/tmASC00044",
                    "HISTORY  CONT  :p//L3AMRF____587231988n944/input/acisfJ0331514m274138_00ASC00045",
                    "HISTORY  CONT  :1N021_r0001_srbreg3.fits[bkgreg]))][bin sky=1][opt type=ASC00046",
                    "HISTORY  CONT  :i4]                                                     ASC00047",
                    "HISTORY  PARM  :outfile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____ASC00048",
                    "HISTORY  CONT  :587231988n944/output/acisf05017_000N022_r0001b_fbinpsf3.ASC00049",
                    "HISTORY  CONT  :fits                                                    ASC00050",
                    "HISTORY  PARM  :kernel=default                                          ASC00051",
                    "HISTORY  PARM  :option=image                                            ASC00052",
                    "HISTORY  PARM  :verbose=0                                               ASC00053",
                    "HISTORY  PARM  :clobber=yes                                             ASC00054"]

        add_record(veccrate, "HISTORY", histlist)

        rec = HistoryRecord(content="Random string without label, string value, or ASC sequence number")
        add_record(veccrate, "HISTORY", rec)

        try:
            add_record(veccrate, "HISTORY", [True, False, 10, 1000.999])
        except TypeError:
            pass
        else:
            raise Exception("check for add_record() with non-string input FAILED")

        print("\nCheck invalid record tag", file=self.verbose_fh)
        try:
            add_record(veccrate, "INVALID", ["blah blah blah blah blah"])
        except ValueError:
            pass
        else:
            raise Exception("check invalid record tag FAILED")

        print("\nGet ALL records", file=self.verbose_fh)
        print(get_all_records(veccrate), file=self.verbose_fh)
        print("\nGet History records", file=self.verbose_fh)
        print(get_history_records(veccrate), file=self.verbose_fh)
        print("\nGet Comment records", file=self.verbose_fh)
        print(get_comment_records(veccrate), file=self.verbose_fh)


        print("\nPrint full header", file=self.verbose_fh)
        print(print_full_header(veccrate), file=self.verbose_fh)

        print("\nDelete history records", file=self.verbose_fh)
        delete_history(veccrate)

        print("\nDelete comment records", file=self.verbose_fh)
        delete_comments(veccrate)

        print("\nGet ALL records", file=self.verbose_fh)
        print(get_all_records(veccrate), file=self.verbose_fh)

        #--------------------------------------------------------------------------

        print("\nRead in virtual+vector+array file.....", file=self.verbose_fh)

        basefile  = self.indir  + 'vector_array.fits'
        infile =  self.outdir  + 'vector_array.fits'

        os.system('cp ' + basefile + " " + infile )

        ds = CrateDataset(infile)
        print(ds, file=self.verbose_fh)
        
        print(veccrate, file=self.verbose_fh)
        
        print("Add EVENTS crate to virtual+vector+array file.....", file=self.verbose_fh)
        add_crate(ds, veccrate)
        print(ds, file=self.verbose_fh)
        
        print("Get SRCREG crate from virtual+vector+array file.....", file=self.verbose_fh)
        tab = get_crate(ds, u"SRCREG")
        print(tab, file=self.verbose_fh)
        print("\nShow POS values (vector arrays) \n", file=self.verbose_fh)
        posvals = get_colvals(tab, b"POS")
        print(posvals, file=self.verbose_fh)
        
        print("\nUpdate posvals[0][1][9] = 5000.9999999\n", file=self.verbose_fh)
        posvals[0][1][9] = 5000.9999999
        
        print("\nShow a COPY of the updated POS values\n", file=self.verbose_fh)
        posvals = copy_colvals(tab, u"POS")
        print(posvals, file=self.verbose_fh)
        
        print("\nShow EQSRC values (virtual vector arrays) \n", file=self.verbose_fh)
        eqvals = get_colvals(tab, b"EQSRC")
        print(eqvals, file=self.verbose_fh)
        
        print("\nShow DEC_SRC values (virtual vector component arrays) \n", file=self.verbose_fh)
        decvals = get_colvals(tab, u"DEC_SRC")
        print(decvals, file=self.verbose_fh)
        
        #--------------------------------------------------------------------------

        print("\nRead in image file.....", file=self.verbose_fh)

        basefile  = self.indir  + 'image.fits'
        infile  = self.outdir  + 'image.fits'

        os.system('cp ' + basefile + " " + infile )

        outfile = self.outdir + self.outfiles[13]

        imds = CrateDataset(infile)
        imcr = imds.get_crate(1)

        print("File type: ", get_crate_type(imcr), file=self.verbose_fh)
        
        print("\nKey names: ", get_key_names(imcr), file=self.verbose_fh) 
        print("\nDelete OBSERVER keyword", file=self.verbose_fh)
        delete_key(imcr, u"OBSERVER")
        print("\nKey names minus OBSERVER : ", get_key_names(imcr), file=self.verbose_fh) 
        
        print("\nImage info:", file=self.verbose_fh)
        print(get_piximg(imcr), file=self.verbose_fh)
        
        print("\nImage shape: ", get_piximg_shape(imcr), file=self.verbose_fh)
        
        print("\nImage values: ", get_piximgvals(imcr), file=self.verbose_fh)
        
        print("\nImage axes: ", file=self.verbose_fh)
        print(print_axis_names(imcr), file=self.verbose_fh)
        
        print("\nImage world coord transform:", file=self.verbose_fh)
        wtran = get_axis_transform( imcr, b"WORLD")
        if wtran: 
            print(wtran.print_parameter_list(), file=self.verbose_fh)
        else:
            print("ERROR, No WORLD coord system found\n", file=self.verbose_fh)
            
        print("\nImage physical coord transform:", file=self.verbose_fh)
        ptran = get_axis_transform( imcr, u"PHYSICAL")
        if ptran:
            print(ptran.print_parameter_list(), file=self.verbose_fh)
        else:
            print("ERROR, No PHYSICAL coord system found\n", file=self.verbose_fh)

        print("\nWrite file", file=self.verbose_fh)
        write_file(imcr, outfile, clobber=True, history=False)

        print("\nCreate empty IMAGECrate.....", file=self.verbose_fh)
        imcr2 = IMAGECrate()
        
        print("Create CrateData.....", file=self.verbose_fh)
        cd = CrateData()
        cd.name = "IMAGE"
        larr3 = ( (0,0,0,0,0,0,0,0,0,0),
                  (1,1,1,1,1,1,1,1,1,1),
                  (2,2,2,2,2,2,2,2,2,2),
                  (3,3,3,3,3,3,3,3,3,3),
                  (4,4,4,4,4,4,4,4,4,4) )

        cd.unit = "blahs"
        cd.desc = "image data"
        
        print("\nAdd image to IMAGECrate.....", file=self.verbose_fh)
        add_piximg(imcr2, cd)
        
        set_piximgvals( imcr2, larr3 )
        
        print("\nGet image.....", file=self.verbose_fh)
        print(get_piximg(imcr2), file=self.verbose_fh)
        
        print("\nSet image vals.....", file=self.verbose_fh)
        set_piximgvals(imcr2, larr3)
        print(get_piximgvals(imcr2), file=self.verbose_fh)
        
        print("\nDelete image.....", file=self.verbose_fh)
        delete_piximg(imcr2)


    def test_13(self):
        print("\nRunning Test 13a - File with a single row", file=self.verbose_fh)
        infile = self.indir + self.infiles[9]
        infile2 = self.indir + self.infiles[10]

        cr = TABLECrate(infile)
        print(cr, file=self.verbose_fh)
        
        print("\nPrint column POS, a vector", file=self.verbose_fh)
        print(cr.get_column('pos'), file=self.verbose_fh)
        print(cr.get_column('pos').values, file=self.verbose_fh)

        print("\nPrint column Y, a vector component", file=self.verbose_fh)
        print(cr.get_column('y'), file=self.verbose_fh)
        print(cr.get_column('y').values, file=self.verbose_fh)

        print("\nPrint column ROTANG, an array", file=self.verbose_fh)
        print(cr.get_column('rotang'), file=self.verbose_fh)
        print(cr.get_column('rotang').values, file=self.verbose_fh)

        print("\nPrint column SHAPE, an scalar", file=self.verbose_fh)
        print(cr.get_column('shape'), file=self.verbose_fh)
        print((cr.get_column('shape').values).astype(tt), file=self.verbose_fh)

 
        print("\nRunning Test 13b - File with a single row due to filtering", file=self.verbose_fh)

        cr2 = TABLECrate(infile2+"[ccd_id=7]")
        print(cr2, file=self.verbose_fh)

        print("\nPrint column X, a vector array component", file=self.verbose_fh)
        print(cr2.get_column('x'), file=self.verbose_fh)
        print(cr2.get_column('x').values, file=self.verbose_fh)

        print("\nPrint column R, an array", file=self.verbose_fh)
        print(cr2.get_column('r'), file=self.verbose_fh)
        print(cr2.get_column('r').values, file=self.verbose_fh)

        print("\nPrint column SHAPE, an scalar", file=self.verbose_fh)
        print(cr2.get_column('shape'), file=self.verbose_fh)
        print((cr2.get_column('shape').values).astype(tt), file=self.verbose_fh)

        print("\nPrint column COMPONENT, an scalar", file=self.verbose_fh)
        print(cr2.get_column('component'), file=self.verbose_fh)
        print(cr2.get_column('component').values, file=self.verbose_fh)


    def test_14(self):
        print("\nRunning Test 14 - Virtual data and Transforms", file=self.verbose_fh)
        infile = self.indir + self.infiles[8]
        infile2 = self.indir + self.infiles[12]
        outfile = self.outdir + self.outfiles[19]

        cr = TABLECrate(infile)

        print("\nTry get_transform() with non-existent column", file=self.verbose_fh)
        try:
            tr = cr.get_transform("SRC") 
        except ValueError as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass
        except:
            raise

        print("\nPrint column POS", file=self.verbose_fh)
        print(cr.get_column('pos'), file=self.verbose_fh)
        print(cr.get_column('pos').values, file=self.verbose_fh)

        print("\nPrint column EQSRC, WCSTransform applied to POS", file=self.verbose_fh)
        print(cr.get_column('eqsrc'), file=self.verbose_fh)
        print(cr.get_column('eqsrc').values, file=self.verbose_fh)        
        print(cr.get_column('eqsrc').get_transform().print_parameter_list(), file=self.verbose_fh)
        print(cr.get_column('eqsrc').get_transform_matrix(), file=self.verbose_fh)


        print("\nPrint column R", file=self.verbose_fh)
        print(cr.get_column('r'), file=self.verbose_fh)
        print(cr.get_column('r').values, file=self.verbose_fh)


        print("\nPrint column EQR, LINEARTransform applied to R", file=self.verbose_fh)
        print(cr.get_column('eqr'), file=self.verbose_fh)
        print(cr.get_column('eqr').values, file=self.verbose_fh)        
        print(cr.get_column('eqr').get_transform().print_parameter_list(), file=self.verbose_fh)
        print(cr.get_column('eqr').get_transform_matrix(), file=self.verbose_fh)


        cr2 = read_file(infile2)
        ax = cr2.get_axis('col4')

        print("\nShow Physical Axis values of a 1D image", file=self.verbose_fh)
        print(ax.values, file=self.verbose_fh)

        print("\nPrint 1D Image parameters", file=self.verbose_fh)
        print(ax.get_transform().print_parameter_list(), file=self.verbose_fh)

        print("\nIs there a transform matrix?  ", ax.get_transform().get_transform_matrix(), file=self.verbose_fh)
        
        print("\nShow the inverted values of the Physical Axis (i.e. the Logical Axis values)", file=self.verbose_fh)
        print(ax.get_transform().invert(ax.values), file=self.verbose_fh)

        cr2.write(outfile, clobber=True)


    def test_15(self):

        print("\nRunning Test 15 - Subspace", file=self.verbose_fh)
        infile = self.indir + self.infiles[11]
        outfile = self.outdir + self.outfiles[18]

        cr = read_file( infile )
        print("\nPrint all subspace data", file=self.verbose_fh)
        print(cr.print_subspace(), file=self.verbose_fh)


        print("\nPrint subspace data for component 4", file=self.verbose_fh)
        print(cr.print_subspace_component(4), file=self.verbose_fh)


        print("\nPrint data for CCD_ID column in component 3", file=self.verbose_fh)
        print(cr.get_subspace_data(3, u"ccd_id"), file=self.verbose_fh)


        print("\nTry to get non-existent subspace component data", file=self.verbose_fh)
        try:
            sd = cr.get_subspace_data(6, b"ccd_id")
        except Exception as ex:
            print(ex, file=self.verbose_fh)
            print("   Expected... OK", file=self.verbose_fh)
            pass


        cr.write(outfile, clobber=True)


    def test_16(self):

        print("\nRunning Test 16 - Varlen", file=self.verbose_fh)
        infile = self.indir + self.infiles[13]
        outfile = self.outdir + self.outfiles[20]

        ds = CrateDataset(infile)
        cr = ds.get_crate(2)

        print(cr.VSCAL1, file=self.verbose_fh)
        print(cr.VSCAL1.values, file=self.verbose_fh)
        
        print(cr.VSCAL2, file=self.verbose_fh)
        print(cr.VSCAL2.values, file=self.verbose_fh)


        print(cr.VSCAL3, file=self.verbose_fh)
        print(cr.VSCAL3.values, file=self.verbose_fh)
        
        print(cr.VSCAL4, file=self.verbose_fh)
        print(cr.VSCAL4.values, file=self.verbose_fh)

        print(cr.VSCAL5, file=self.verbose_fh)
        print(cr.VSCAL5.values, file=self.verbose_fh)
        
        print(cr.VSCAL6, file=self.verbose_fh)
        print(cr.VSCAL6.values, file=self.verbose_fh)

        print(cr.FSCAL0, file=self.verbose_fh)
        print(cr.FSCAL0.values, file=self.verbose_fh)


        cr2 = ds.get_crate(3)

        print(cr2.VVECT1, file=self.verbose_fh)
        print(cr2.VVECT1.values, file=self.verbose_fh)
        
        print(cr2.VVECT2, file=self.verbose_fh)
        print(cr2.VVECT2.values, file=self.verbose_fh)

        print(cr2.VVECT3, file=self.verbose_fh)
        print(cr2.VVECT3.values, file=self.verbose_fh)
        
        print(cr2.VVECT3, file=self.verbose_fh)
        print(cr2.VVECT3.values, file=self.verbose_fh)

        print(cr2.VVECT5, file=self.verbose_fh)
        print(cr2.VVECT5.values, file=self.verbose_fh)
        
        print(cr2.VVECT6, file=self.verbose_fh)
        print(cr2.VVECT6.values, file=self.verbose_fh)
        
        ds.write(outfile, clobber=True)



    def test_17(self):

        print("\nRunning Test 17 - Bit arrays", file=self.verbose_fh)
        infile = self.indir + self.infiles[14]
        outfile = self.outdir + self.outfiles[21]


        cr = read_file(infile)

        d = cr.get_column(u"scalar_bit")
        d.name = b'test'

        tab = TABLECrate()

        print(" Scalar Bit column (1X)", file=self.verbose_fh)
        c = CrateData()
        c.name = 'scalar_bit'
        c.values = np.array([255,64,32], dtype='uint8')
        c.convert_bytes_to_bits(3)
        
        print(c, file=self.verbose_fh)
        print(c.values, file=self.verbose_fh)
        tab.add_column(c)

        print("Scalar Byte column (1B)", file=self.verbose_fh)
        c = CrateData()
        c.name = 'scalar_byte'
        c.values = np.array([128,64,32],dtype='uint8')
        print(c, file=self.verbose_fh)
        print(c.values, file=self.verbose_fh)
        tab.add_column(c)

        print("Bit Array column (16X) - all bits used", file=self.verbose_fh)
        c = CrateData()
        c.name = 'array_bit'
        c._bit_array = True
        c.values = np.array([[1,128],[1,64],[1,32]],dtype='uint8')
        c.convert_bytes_to_bits(16)
        print(c, file=self.verbose_fh)
        print(c.values, file=self.verbose_fh)
        tab.add_column(c)
        
        print("Byte Array column (2B)", file=self.verbose_fh)
        c = CrateData()
        c.name = 'array_byte'
        c.values = np.array([[1,128],[1,64],[1,32]],dtype='uint8')
        print(c, file=self.verbose_fh)
        print(c.values, file=self.verbose_fh)
        tab.add_column(c)
        
        tab.add_column(d)
        
        tab.write(outfile, clobber=True)


    def test_18(self):
        print("\nRunning Test 18 - History and Comments Records", file=self.verbose_fh)
        from io import StringIO

        infile = self.indir + self.infiles[11]
        outfile = self.outdir + self.outfiles[22]
        outfile2 = self.outdir + self.outfiles[23]
        cr = read_file(infile)

        # Print existing HISTORY|COMMENT content
        print(cr.print_history(), file=self.verbose_fh)

        # Delete keyword which has HISTORY content associated with it.
        #  o tests that the code retains the records appropriately
        print("Delete ASPTYPE keyword", file=self.verbose_fh)
        cr.delete_key("ASPTYPE")

        if "ASPTYPE" in str(cr.get_keynames()):
            raise Exception("delete ASPTYPE keyword FAILED")

        #  o this just shows that the key was removed.. not really being tested here.
        print(cr.get_keynames(), file=self.verbose_fh)

        # Add comment records to crate:  Associated with last non-structural key ( ??? )
        print("Add Comments", file=self.verbose_fh)
        try:
            #  o empy record
            cr.add_record("COMMENT", "")
            #  o list of records
            cr.add_record("COMMENT", ["Small block of comment lines", "to test the Crates add_record() method", "which adds them to a file"])
            #  o record with unicode content
            ustring = "© Whizbang The Flübmaster"
            cr.add_record("COMMENT", ustring )
            #  o really long record (more than FITS card length)
            cr.add_record("COMMENT", "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGGGGGGGGGHHHHHHHHHHIIIIIIIIIIJJJJJJJJJJKKKKKKKKKKLLLLLLLLLLMMMMMMMMMMNNNNNNNNNNOOOOOOOOOOPPPPPPPPPPQQQQQQQQQQRRRRRRRRRRSSSSSSSSSSTTTTTTTTTTUUUUUUUUUUVVVVVVVVVVWWWWWWWWWWXXXXXXXXXXYYYYYYYYYYZZZZZZZZZZaaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjjkkkkkkkkkkllllllllllmmmmmmmmmmnnnnnnnnnnooooooooooppppppppppqqqqqqqqqqrrrrrrrrrrssssssssssttttttttttuuuuuuuuuuvvvvvvvvvvwwwwwwwwwwxxxxxxxxxxyyyyyyyyyyzzzzzzzzzz")
            #  o a single BLANK line
            cr.add_record("COMMENT", None)
        except Exception as ex:
            print(ex)
            raise Exception("Unexpected exception occurred")

        # Add History records to crate:  Associated with last non-structural key ( ??? ); but flushed to file at end of block.
        print("Add History", file=self.verbose_fh)

        #  o From Single string
        cr.add_record("HISTORY", "This is a non-ASCFITS compliant history record")

        #  o From HistoryRecord object
        rec1 = HistoryRecord( tool="dmsomething", date="2017-12-01T14:00:00", param_names=["infile", "outfile", "tol"], param_values=["@in.lis", "out.fits", "tolerance"], param_stacks=[[], [], []], asc_start=34)
        recstr = rec1.as_ASC_FITS(with_tag=False)
        add_history(cr, recstr)

        #  o From HistoryRecord object (with stacks)
        rec2 = HistoryRecord( tool="dmsomething2", date="2019-01-01T12:11:22", param_names=["infile", "outfile", "tol"], param_values=["@in.lis", "out.fits", "tolerance"], param_stacks=[["stk1.fits", "stk2.fits"], [], []],asc_start=55)
        recstr = rec2.as_ASC_FITS(with_tag=False, stk_expand=True)
        reclist = recstr.split("\n")
        add_history(cr, reclist)

        #  o From List of strings
        histlist = ["HISTORY  TOOL  :dmcopy  2016-08-11T20:27:22                             ASC00041",
                    "HISTORY  PARM  :infile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____5ASC00042",
                    "HISTORY  CONT  :87231988n944/output/acisf05017_000N022_r0001b_rayevt3.fiASC00043",
                    "HISTORY  CONT  :ts[sky=bounds(region(/export/proc/tmp.ascdsl3/prs_run/tmASC00044",
                    "HISTORY  CONT  :p//L3AMRF____587231988n944/input/acisfJ0331514m274138_00ASC00045",
                    "HISTORY  CONT  :1N021_r0001_srbreg3.fits[bkgreg]))][bin sky=1][opt type=ASC00046",
                    "HISTORY  CONT  :i4]                                                     ASC00047",
                    "HISTORY  PARM  :outfile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____ASC00048",
                    "HISTORY  CONT  :587231988n944/output/acisf05017_000N022_r0001b_fbinpsf3.ASC00049",
                    "HISTORY  CONT  :fits                                                    ASC00050",
                    "HISTORY  PARM  :kernel=default                                          ASC00051",
                    "HISTORY  PARM  :option=image                                            ASC00052",
                    "HISTORY  PARM  :verbose=0                                               ASC00053",
                    "HISTORY  PARM  :clobber=yes                                             ASC00054"]
        cr.add_record("HISTORY", histlist)

        #  o Really long HISTORY record (longer than FITS card length)
        cr.add_record("HISTORY", "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEEFFFFFFFFFFGGGGGGGGGGHHHHHHHHHHIIIIIIIIIIJJJJJJJJJJKKKKKKKKKKLLLLLLLLLLMMMMMMMMMMNNNNNNNNNNOOOOOOOOOOPPPPPPPPPPQQQQQQQQQQRRRRRRRRRRSSSSSSSSSSTTTTTTTTTTUUUUUUUUUUVVVVVVVVVVWWWWWWWWWWXXXXXXXXXXYYYYYYYYYYZZZZZZZZZZaaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjjkkkkkkkkkkllllllllllmmmmmmmmmmnnnnnnnnnnooooooooooppppppppppqqqqqqqqqqrrrrrrrrrrssssssssssttttttttttuuuuuuuuuuvvvvvvvvvvwwwwwwwwwwxxxxxxxxxxyyyyyyyyyyzzzzzzzzzz")

        #  o String with newline delimeters
        cr.add_record("HISTORY", "First History line \nSecond History line \nThird History line \nFourth History line \nFifth History line") 

        #  o Add block of comments which includes both an empty and blank line.
        content = ["Some arbitrary comment text, followed by an empty comment",
                   "", 
                   "With some more commentary, followed by a blank line",
                   None,
                   "and ending with a final statement", 
                   "Thank you for your attention!"]
        try:
            cr.add_record("COMMENT", content )
        except Exception as ex:
            raise Exception("check empty vs blank COMMENT FAILED")

        # Test Error case: tag != "COMMENT" | "HISTORY"
        print("Check invalid record tag", file=self.verbose_fh)
        try:
            cr.add_record("INVALID", ["blah blah blah blah blah"])
        except ValueError as ex:
            pass
        else:
            raise Exception("check invalid record tag FAILED")

        # Test Error case: COMMENT - single content != string or None
        print("Check invalid COMMENT content", file=self.verbose_fh)
        try:
            cr.add_record("COMMENT", 8 )
        except TypeError as ex:
            pass
        else:
            raise Exception("check invalid COMMENT content FAILED")

        # Test Error case: content != List of string
        try:
            cr.add_record("COMMENT", ["some text", 8, "more text"] )
        except TypeError as ex:
            pass
        else:
            raise Exception("check invalid COMMENT content FAILED")

        # Test Error case: HISTORY - single content == None
        print("Check invalid HISTORY content", file=self.verbose_fh)
        try:
            cr.add_record("HISTORY", None )
        except TypeError as ex:
            pass
        else:
            raise Exception("check invalid HISTORY content - A, FAILED")

        # Test Error case: HISTORY - single content != string
        print("Check invalid HISTORY content", file=self.verbose_fh)
        try:
            cr.add_record("HISTORY", 9 )
        except TypeError as ex:
            pass
        else:
            raise Exception("check invalid HISTORY content - B, FAILED")

        # Print sets of records (for verification)
        #  o exercises crate handling of history list and functionality of these methods
        print("Get ALL records", file=self.verbose_fh)
        print(cr.get_all_records(), file=self.verbose_fh)

        print("Get only comment records", file=self.verbose_fh)
        print(cr.get_comment_records(), file=self.verbose_fh)

        print("Get only history records", file=self.verbose_fh)
        print(cr.get_history_records(), file=self.verbose_fh)

        print("Print full header", file=self.verbose_fh)
        print(cr.print_full_header(), file=self.verbose_fh)


        # Write output files - 1 with history, 1 without
        try:
            cr.write(outfile, clobber=True, history=True)
            cr.write(outfile2, clobber=True, history=False)
        except UnicodeDecodeError as ux:
            #UnicodeDecodeError('ascii', '\xc2\xa9 Whizbang The Fl\xc3\xbcbmaster', 0, 1, 'ordinal not in range(128)'))
            print(ux)
            print("UnicodeDecodeError hit - to be fixed")
            pass
        except Exception as ex:
            print(ex)
            raise Exception("cr.write() - Unexpected exception occurred")

        # Test Delete methods
        #  o delete all HISTORY
        print("Delete history records", file=self.verbose_fh)
        cr.delete_history_records()

        #  o delete all COMMENTS
        print("Delete comment records", file=self.verbose_fh)
        cr.delete_comment_records()

        #  o show all are gone
        print("Get ALL records", file=self.verbose_fh)
        print(cr.get_all_records(), file=self.verbose_fh)

 

    def test_19(self):
        print("\nRunning Test 19 - History and Comments Records in Update Mode", file=self.verbose_fh)
        infile  = self.indir + self.infiles[15]
        outfile = self.outdir + self.outfiles[24]
  
        # copy input file to output location for update
        os.system("cp " + infile + " " + outfile)

        crate = read_file(outfile, mode="rw")
        histnum = crate.get_key("histnum")

        tool="kickin_it"
        pname = [ "this", "that", "the", "other"]
        pvalue = [ 99.78, True, "false", 10]
        
        my_history = HistoryRecord( tool=tool, date="2018-11-13T21:44:24", param_names=pname, param_values=pvalue, asc_start=histnum.value+1)

        add_history( crate, my_history)
        add_comment( crate, "testing for duplicates")

        crate.write() 

        print(crate.print_full_header(), file=self.verbose_fh)


    def test_20(self):
        print("\nRunning Test 20 - Standard Header", file=self.verbose_fh)
        # initialize the test suite
        loader = unittest.TestLoader()
        suite  = unittest.TestSuite()
        
        # add tests to the test suite
        suite.addTests(loader.loadTestsFromModule(testpycratesheader))
        
        # initialize a runner, pass it the test suite and run it
        runner = unittest.TextTestRunner(verbosity=1,stream=self.verbose_fh)
        result = runner.run(suite)        

        print("  Done", file=self.verbose_fh)

        if not result.wasSuccessful():
            raise Exception("Test 20 (Standard Header) FAILED")

    def test_21(self):

        print("\nRunning Test 21 - Blank Descriptor Names", file=self.verbose_fh)
 
        infile  = self.indir + self.infiles[17]
        # self validation and no outfile
 
        try:
            print("\n- Read in dataset", file=self.verbose_fh)
            ds = CrateDataset(infile)
            print(ds, file=self.verbose_fh)
        except:
            raise
  


        img=0
        tab=0
        axisnames=0
        ncoords  =0
        status = 0
        tol = 1e-6


        """ read image block """
        try:
            print("\n-- Image Block WMAP:", file=self.verbose_fh)
            # this section will access the affected functions:
            #  def __load_key
            #  def __load_data
            #  def __load_image_data
            #  etc.
            img = get_crate(ds, "WMAP")     # image block

            print("Array Dimn: ", img.get_shape(), file=self.verbose_fh)
            ncoords =  img.get_ncoords()
            print("Num Coords: ", ncoords, file=self.verbose_fh)
            axisnames =  img.get_axisnames()
            print("Axis Names: ", axisnames, file=self.verbose_fh)

            foo = img.get_axis(u"")
            print("\tExpected '"+foo.name+"'", file=self.verbose_fh)
            foo = img.get_axis(u"_2")
            print("\tExpected '"+foo.name+"'", file=self.verbose_fh)            
   
            print("\n--- Print Image Transform Parameters", file=self.verbose_fh)
            # this section will access the affected functions:
            #  def __create_linear_transform
            #  def __create_wcs_transform
            #  etc.
            for axis in axisnames: 
                lst = img.get_transform(axis).print_parameter_list()
                print(lst, file=self.verbose_fh )
   

            """ test transform  """
            print("\n--- Get Image Axis Scale and Offset Values", file=self.verbose_fh)
            for axis in axisnames:
                if axis != "" and axis != "_2":
                    continue;
 
                scle = img.get_transform(axis).get_parameter("SCALE").get_value()  # --> 0.027
                ofst = img.get_transform(axis).get_parameter("OFFSET").get_value() # --> -17.293499999999998
                print("'%8s' (%-14s): (%.6f, %.6f)" % ( axis, "SCALE, OFFSET", scle,ofst), file=self.verbose_fh )
                if abs(scle - 0.027) > tol:
                    status += 1
                    print("\t\tNot-Expected scale value",  file=self.verbose_fh)
                if abs(ofst - (-17.2935) ) > tol:
                    print("\t\tNot-Expected offset value",  file=self.verbose_fh)
                    status += 1



            """ test subspace """
            print("\n--- Get Subspace Items", file=self.verbose_fh)
            # this section will access the affected functions:
            #  def read_subspace
            #  etc.
            ssd1 = img.get_subspace_data(1, u"detx")
            ssd2 = img.get_subspace_data(1, u"dety")
            ssd3 = img.get_subspace_data(1, u"POS")
            if ssd1.name == "DETX" and ssd2.name == "DETY" and ssd3.name == "POS" :
                print("cptnames: %8s,%8s, %8s" % (ssd1.name, ssd2.name, ssd3.name), file=self.verbose_fh)
            else:
                raise
        except:
            status += 1
            print("\tNot-OK",  file=self.verbose_fh)
            pass


        """ test columns """
        try:
            print("\n-- Table Block REGION:", file=self.verbose_fh)
            # this section will access the affected functions:
            #  def read_header
            #  def read_columns
            #  def __create_linear_transform
            #  def __create_wcs_transform
            #  etc.
 
            tab = get_crate(ds, "REGION")     # table block

            numcols = get_number_cols(tab)
            namcols = tab.get_colnames()
            print("Number of columns:  ", numcols, file=self.verbose_fh)
            if numcols==6:
                pass
            else:
                print("\tNot-Expected number",  file=self.verbose_fh)
                status += 1

            print("Names of columns:  ", namcols, file=self.verbose_fh)
            if 'X' in namcols and 'Y' in namcols and 'SHAPE' in namcols:
                pass
            else:
                print("\tNot-Expected names",  file=self.verbose_fh)
                status += 1

            print("\n--- Get Table Transform Parameters: Scale and Offset Values", file=self.verbose_fh)
            
            axis = ""      # both x- & y-axis have blank descriptor names in original file
            try:
                trsfm = tab.get_transform(axis)
                lst = trsfm.print_parameter_list()
            except Exception as ex:
                print("\tException occured and caught at tab.get_transform('').", file=self.verbose_fh)
                print("\t\tExpected, as blank name is altetnated run-time.\n", file=self.verbose_fh)
                pass
            
            ncoords = 2
            for ii in range(ncoords):
                axis = "COLUMN7"      # both x- & y-axis updated run-time with 'COLUMN7,8' as descriptor names
                if ii==1 : axis = "COLUMN8"
                try: 
                    lst = tab.get_transform(axis).print_parameter_list()
                    print(lst, file=self.verbose_fh )
                except Exception as ex:
                    print("\t\tNot-Expected exception: "+ex)
        

                scle = tab.get_transform(axis).get_parameter("SCALE").get_value()  # --> 0.027
                ofst = tab.get_transform(axis).get_parameter("OFFSET").get_value() # --> -17.293499999999998
                print("'%8s' (%-14s): (%.6f, %.6f)" % ( axis, "SCALE, OFFSET", scle,ofst), file=self.verbose_fh )
                if abs(scle - 0.027) > tol:
                    status += 1
                    print("\t\tNot-Expected scale value",  file=self.verbose_fh)
                if abs(ofst - (-17.2935) ) > tol:
                    print("\t\tNot-Expected offset value",  file=self.verbose_fh)
                    status += 1

 
        except:
            status += 1
            print("\tNot-OK",  file=self.verbose_fh)
            pass


        if status != 0:
            raise Exception("Test 21 (Blank Decsriptor Names) FAILED")



    def test_22(self):
        #                                                                 #
        # NOT EXECUTED:                                                   #
        #   IDENTICAL AXIS/COLUMN NAME HANDLING TO BE ADDRESSED IN SL-272 #
        #                                                                 #
        print("\n\nRunning Test 22 - Identical Descriptor Names", file=self.verbose_fh)
 
        infile  = self.indir + self.infiles[18]
        # self validation and no outfile
 

        try:
            print("\n- Read in dataset", file=self.verbose_fh)
            ds = CrateDataset(infile)
            print(ds, file=self.verbose_fh)
        except:
            raise


        img=0
        tab=0
        ncoords = 0
        axisnames = 0
        tol = 1.e-6
        status = 0

        try:
            print("\n-- Image Block WMAP:", file=self.verbose_fh)
            img = get_crate(ds, "WMAP")     # image block

            ncoords =  img.get_ncoords()
            print("Ncoords:    ", ncoords, file=self.verbose_fh)
            axisnames =  img.get_axisnames()
            print("Axisnames: ", axisnames, file=self.verbose_fh)

            foo = img.get_axis(u"FOO")
            print("\tExpected '"+foo.name+"'", file=self.verbose_fh)
            foo = img.get_axis(u"FOO_2")
            print("\tExpected '"+foo.name+"'", file=self.verbose_fh)

            print("\n--- Print Image Transform Parameters", file=self.verbose_fh)
            for axis in axisnames: 
                lst = img.get_transform(axis).print_parameter_list()
                print(lst, file=self.verbose_fh )

            print("\n--- Get Image Axis Scale and Offset Values", file=self.verbose_fh)
             
            for axis in axisnames:
                if axis != "FOO" and axis != "FOO_2":
                    continue

                scle = img.get_transform(axis).get_parameter("SCALE").get_value()  # --> 0.027
                ofst = img.get_transform(axis).get_parameter("OFFSET").get_value() # --> -17.293499999999998
                print("'%8s' (%-14s): (%.6f, %.6f)" % ( axis, "SCALE, OFFSET", scle,ofst), file=self.verbose_fh )
                if abs(scle - 0.027) > tol:
                    status += 1
                    print("\t\tNot-Expected scale value",  file=self.verbose_fh)
                if abs(ofst - (-17.2935) ) > tol:
                    print("\t\tNot-Expected offset value",  file=self.verbose_fh)
                    status += 1
 
        except:
            status += 1
            print("\tNot-OK",  file=self.verbose_fh)
            pass


        """ test columns """        
        try:
            print("\n-- Table Block REGION:", file=self.verbose_fh)
            tab = get_crate(ds, "REGION")     # table block
            numcols = get_number_cols(tab)
            namcols = tab.get_colnames()
            print("Number of columns:  ", numcols, file=self.verbose_fh)
            if numcols != 6:
                print("\t\tNot-Expected number",  file=self.verbose_fh)
                status += 1

            print("Names of columns:  ", namcols, file=self.verbose_fh)
            if 'X' in namcols and 'Y' in namcols and 'SHAPE' in namcols:
                pass
            else:
                print("\t\tNot-Expected names",  file=self.verbose_fh)
                status += 1

            print("\n--- Get Table Transform Parameters: Scale and Offset Values", file=self.verbose_fh)
            axis_base = "BOO"   # both x- and y- descriptors are 'BOO'

            ncoords = 2
            for ii in range(ncoords):
                axis = axis_base      
                if ii==1 : axis = axis_base+"_2"  # expecting to be separated like this.

                try:
                    trsfm = tab.get_transform(axis)
                    lst = trsfm.print_parameter_list()
                except Exception as ex:
                    print("\n\tException occured and caught at tab.get_transform('",axis,"').", file=self.verbose_fh)
                    print("\t\tExpected, as the axis is a duplicate or lost (TB fixed).\n", file=self.verbose_fh)
                    continue
            
 
                print(lst, file=self.verbose_fh )
            
                scle = tab.get_transform(axis).get_parameter("SCALE").get_value()  # --> 0.027
                ofst = tab.get_transform(axis).get_parameter("OFFSET").get_value() # --> -17.293499999999998
                print("'%8s' (%-14s): (%.6f, %.6f)" % ( axis, "SCALE, OFFSET", scle,ofst), file=self.verbose_fh )
                if abs(scle - 0.027) > tol:
                    status += 1
                    print("\t\tNot-Expected scale value",  file=self.verbose_fh)
                if abs(ofst - (-17.2935) ) > tol:
                    print("\t\tNot-Expected offset value",  file=self.verbose_fh)
                    status += 1
  
        except:
            status += 1
            print("\tNot-OK",  file=self.verbose_fh)
            pass


        if status != 0:
            raise Exception("Test 22 (Identical Decsriptor Names) FAILED")


         
    def check_log(self):
        """
        Will compare log file ( if appropriate ) against the baseline version.
        Returns True if OK, False otherwise
        """
        result = False

        outfile  = self.outdir + "crates_verbose.txt"
        savfile  = self.outdir + "crates_verbose.txt_sav"
        filtfile = self.outdir + "crates_verbose.txt_filt"
        basefile = self.basdir + "crates_verbose.txt"
        difffile = self.outdir + "crates_verbose.txt_diff"

        if self.verbose > 0:
            print ("Checking log file:")

        print("Checking log file:", file=self.runlog_fh)

        self.filter_log_file( outfile )

        os.system('cp ' + basefile + " " + savfile )

        if self._log:
            if os.path.isfile( filtfile ):
                comstr = "diff %s %s > %s" % (filtfile, savfile, difffile)
                if os.system( comstr ):
                    result = False
                else:
                    result = True

        if result == True:
            if self.verbose > 0: 
                print ("   OK")
            print("   OK", file=self.runlog_fh)

        else:
            if self.verbose > 0:
                print ("   Problem")
            print("   Problem", file=self.runlog_fh) 
        return result


    def filter_log_file( self, outfile ):
        """
        Creates a filtered version of the verbose output file.  Directory paths 
        are replaced with either "<TESTIN>" or "<TESTOUT>" and "NaN"s are converted 
        to "nan"s.
        """
        filtfile = outfile + "_filt"

        # create filtered verbose file
        outfile_fh = open(outfile)
        filtfile_fh = open(filtfile, 'w')
        testin  = "<TESTIN>"
        testout = "<TESTOUT>"

        for line in outfile_fh:
            tokens = None
            if line.find("NaN") != -1:
                line = line.replace("NaN", "nan")

            if line.find("/") != -1:
                tokens = line.split('/')                

            if line.find(self.indir) != -1 and tokens is not None:
                print(tokens[0] + testin + '/' + tokens[len(tokens)-1].rstrip('\n'), file=filtfile_fh)

            elif line.find(self.outdir) != -1 and tokens is not None:
                print(tokens[0] + testout + '/' + tokens[len(tokens)-1].rstrip('\n'), file=filtfile_fh)

            elif line.find("string128") != -1 or line.find("str512") != -1:
                tokens = line.split("str")
                print(tokens[0]+"str", file=filtfile_fh)

            elif line.find("Ran 5 tests") != -1:
                tokens = line.split("in")
                print(tokens[0], file=filtfile_fh)
  
            else:
                print(line.rstrip('\n'), file=filtfile_fh)


        outfile_fh.close()
        filtfile_fh.close()


if __name__ == '__main__':
    t = TestCrates(*sys.argv[1:])

    anybad = False

    # Run the set of tests
    try:
        if (t.run() == "FAIL" ):
            anybad = True
    except TypeError as e:
        name = ""
        print(("    ", e))
        pass
    except:
        raise

    # Run the output files they create
    try:
        if ( t.verify() == "FAIL"):
            anybad = True
    except:
        raise
