# 
#  Copyright (C) 2010-2016,2018-2021  Smithsonian Astrophysical Observatory
#
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

from pycrates import CrateData
from pycrates import CrateKey
from pycrates.crateutils import parse_groupings
from collections import OrderedDict
from numpy import array
from numpy import array_str
from numpy import float64
import pycrates as pycr
import hashlib
import warnings
from history import *
from cxcstdhdr import *
import copy

class Crate:
    def __init__(self):
        """
        Initializes the base Crate class.
        """
        self.__clear__()

        self.__keylist = OrderedDict()
        self.__datalist = OrderedDict()
        self.__history = []
        self.__axes = OrderedDict()


    def __clear__(self):
        """
        Clears the Crate.
        """
        # public attributes
        self.name  = "" 

        # protected and private attributes
        self._crtype     = ""
        self._number     = 0
        self.__signature = ""

        if hasattr(self, "_Crate__keylist"):
            self.__keylist.clear()

        if hasattr(self, "_Crate__datalist"):
            self.__clear_vectors(self.__datalist)
            self.__datalist.clear()

        if hasattr(self, "_Crate__history") and self.__history is not None:
            if len(self.__history) > 0:
                del self.__history[:]
            del self.__history
            

        self.__subspace = []

        self.__key_flag = False
        self.__data_flag = False

        self.__key_trash = []
        self.__data_trash = []


        ## TABLE
        self.__nrows = 0
        self.__ncols = 0


        ## IMAGE
        if hasattr(self, "_Crate__axes"):
            self.__clear_vectors(self.__axes)
            self.__axes.clear()


    def __clear_vectors(self, in_dict):
        # loop through input dictionary, if column/axis is a vector, null out the
        # vector component's parent attribute
        datalist = list(in_dict.keys())

        for item in (datalist):
            col = in_dict[item]
            self.__clear_vector_parents(col)


    def __clear_vector_parents(self, in_col):
        # for each vector component, set parent pointer to None to break 
        # circular dependencies and allow vector CrateData objects to be freed
        
        if in_col.is_vector():
            for ii in range (0, in_col.vdim):
                try:  # in case column with vdim=2 does not actually have defined components
                    cpt = in_col.get_vector_component(ii)
                    cpt.parent = None
                except:
                    pass


    def __del__(self):
        """
        Clears and closes the Crate.
        """
        self.__clear__()


    def __str__(self):
        """
        Returns a formatted string representation of the Crate.
        """
        return self.__repr__()


    def __repr__(self):
        """
        Returns a formatted string representation of the Crate.        
        """
        retstr = ""

        if self._crtype == "IMAGE":
            retstr  = "   Crate Type:        <IMAGECrate>\n"
            retstr += "   Crate Name:        " + self.name + "\n"
            if len(self.__axes) > 0:
                retstr += "   Image Axes:        " + str(self.get_axisnames()) + "\n"

        else:
            retstr  = "   Crate Type:        <TABLECrate>\n"
            retstr += "   Crate Name:        " + self.name + "\n"
            retstr += "   Ncols:             " + str(self.__ncols) + "\n"
            retstr += "   Nrows:             " + str(self.__nrows) + "\n"
            
        return retstr
    

    def get_number(self):
        """
        Returns the crate number in the dataset.
        """
        return self._number


    def _set_number(self, val):
        """
        Sets the crate number in the dataset.
        """
        self._number = val


    def is_image(self):
        """
        Returns True if the crate is contains image data, False if not.
        """
        if self._crtype == "IMAGE":
            return True
        return False


    ## SUBSPACE functions ###############################################

    def _set_subspace(self, sslist):
        """
        Sets/replaces the subspace component list.
        """
        self.__subspace = sslist


    def _get_subspace(self):
        """
        Retrieves the list of subspace components.
        """
        return self.__subspace
        

    def get_subspace_data(self, cptnum, item):
        """
        Retrieves the subspace column information for a specified
        subspace component.  Component index number is one-based.
        """

        if cptnum is not None:
            try:
                cptnum = int(cptnum)
            except Exception :
                raise TypeError("Subspace component number is not an integer")
        else:
            raise ValueError("Subspace component value is {}".format(str(cptnum)))


        if (cptnum < 1) or (cptnum > len(self.__subspace)):
            raise IndexError("Subspace component number is out of range.")

        try: 
            ss = self.__subspace
            ssdata = ss[cptnum-1].get_subspace_data(item)
        except KeyError:
            raise IndexError("Cannot find subspace data item " + str(item) )
        except:
            raise

        return ssdata


    def print_subspace(self):
        """
        Prints all of the data contained each of the subspace components.
        """
        retstr = ""
        for ii in range( len(self.__subspace) ):
            retstr = retstr + "Component " + str(ii+1) + "\n"
            retstr = retstr + str(self.__subspace[ii])

        return retstr


    def print_subspace_component(self, cptnum):
        """
        Prints data contained the specified subspace component.   
        Component index number is one-based.
        """
        if (cptnum < 1) or (cptnum > len(self.__subspace)):
            raise IndexError("Subspace component number is out of range.")

        retstr = ""
        retstr = retstr + "Component " + str(cptnum) + "\n"
        retstr = retstr + str(self.__subspace[cptnum-1])

        return retstr
                

    ## HEADER KEYWORD functions #########################################

    def key_exists(self, keyname):
        """
        Searches header list for specified keyword.  If it exists, 
        returns True; if not, returns False.
        """
        
        keyname = pycr.convert_2_str(keyname)
        
        if keyname.upper() in list(self.__keylist.keys()):
            return True
        return False


    def get_key(self, key):
        """
        Searches header list for keyword by name or number and returns the 
        CrateKey if it exists.

        key - the name or number (zero-based) of the keyword to retrieve

        """
        key = pycr.convert_2_str(key)

        if not isinstance( key, str):
            keylist = list(self._get_keylist().keys())
            
            if len(keylist) == 0:
                return

            try:
                keyname = keylist[key]
            except:
                raise IndexError("Invalid key index number.")
        else:
            keyname = key

        if self.key_exists(keyname):
            return self.__keylist[keyname.upper()]
        # MCD - 20150526: back this out
        #   change in Crates API has ramification in user scripts and needs
        #   more vetting.  Effecting Sherpa; also changes behavior of 
        #   get_key_value() which does not appear to be set for the change.
        #   
        #else:
        #    raise ValueError("Keyword '"+ keyname + "' does not exist.")
        return None


    def get_key_value(self, key):
        """
        Finds the value of a keyword and returns it.  If the keyword 
        does not exist, returns None.
        """
        key = pycr.convert_2_str(key)
        keyobj = self.get_key(key)
        
        if keyobj is not None:
            return keyobj.value
        return None


    def add_key(self, key, index=None):
        """
        Appends a key to the end of header list or replaces an 
        already existing key in the list.
        """
        if not isinstance(key, pycr.CrateKey):
            raise TypeError("Input must be a CrateKey.")

        inkeyname = key.name.upper()         

        if index is not None:
            try:
                ndx = int(index)
            except:
                raise TypeError( "Index argument must be compatible with integer." )

            if (ndx > len(self.__keylist) or ndx < 0):
                raise IndexError("Key index is out of range")
                
            if (ndx == len(self.__keylist)):
                self.__keylist[inkeyname] = key

            else:
                newdict = OrderedDict()
                ii = 0
                for (itemkey, itemvalue) in (self.__keylist.items()):
                    if ndx == ii:
                        newdict[inkeyname] = key
                    
                    newdict[itemkey] = itemvalue
                    ii = ii + 1
                
                self.__keylist = newdict
            
        else:
            self.__keylist[inkeyname] = key


    def delete_key(self, key):
        """
        Removes a keyword from the header list.
        """
        key = pycr.convert_2_str(key)
        keylist = list(self.__keylist.keys())

        if not isinstance( key, str):
            if len(keylist) == 0:
                return

            try:
                keyname = keylist[key].upper()
            except:                
                return
        else:
            keyname = key.upper()


        if self.key_exists(keyname):

            # if needed, update history/comment records afterkey attribute
            currkeyindex = keylist.index(keyname)
            prevkeyname = keylist[currkeyindex-1]
            records = self._get_history()
            for rec in records:
                if keyname == rec[2]:                    
                    rec[2] = prevkeyname
                    
            # remove from header
            del self.__keylist[keyname]
            self.__key_trash.append(keyname)


    def get_nkeys(self):
        """
        Returns the number of keys read in from the header.
        """
        return len(self.__keylist)


    def get_keynames(self):
        """
        Returns a list of key names in the header list.
        """
        return list(self.__keylist.keys())


    def _get_keylist(self):
        """
        Retrieves the list containing the header keywords.
        """
        return self.__keylist


    def _copy_keylist(self):
        return self.__keylist.copy()


    def _set_keylist(self, in_list):
        """
        Replaces the keyword list.
        """
        self.__keylist = in_list
        self._set_key_flag(True)


    def _get_key_flag(self):
        """
        Returns the key flag.  'True' means that the keywords 
        have been read in from the input file.
        """
        return self.__key_flag


    def _set_key_flag(self, flag):
        """
        Sets the key flag.
        """
        self.__key_flag = flag


    def _get_key_trash(self):
        return self.__key_trash[:]



    ## Standard Header functions #############################

    def add_stdhdr(self, groupings, replace=False):
        """
        Generate a standard header keylist based on input groupings and merge it with the current Crate.keylist.

        If the 'replace' flag is True, existing keyword values will be overwitten. 
        If the 'replace' flag is False, the keys in the current Crate will retain their values (default).
        """

        # generate a CrateKey dict of all the keys in requested grouping (s)
        stdhdr = self._build_stdhdr(groupings)

        # merge the current keylist with the newly-created stdhdr keylist
        if replace is True:
            self.__keylist.update(stdhdr)
        else:
            # replace == False; update the current keylist with any missing values 
            for keyname in stdhdr.keys():
                if keyname not in self.__keylist.keys():
                    self.add_key( stdhdr[keyname] )


    def create_stdhdr(self, groupings):
        """
        Create a default-valued list of CrateKeys based on the input grouping(s) and add it to the Crate.
        If a cr.keylist already exists, an execption will be thrown. 
        """
        # check if keylist already exists
        if (self.__keylist is not None) and (len(self.__keylist) > 0):
            raise RuntimeError("This Crate already contains a list of keywords.")
        
        # build the standard header based on groupings
        stdhdr = self._build_stdhdr(groupings)
 
        self._set_keylist( stdhdr )  
 

    def _build_stdhdr(self, groupings, check_stdgrps=False):
        """
        Helper function to build the standard header list based on input groupings and convert it to a 
        list of CrateKeys.

        If 'check_stdgrps' is True, the function will separate out valid standard groupings from 
        user-defined groupings before sending into build_hdr(); otherwise, build_hdr() will throw an 
        exception with user-defined groupings which are also valid.
        """

        try:
            cxchdr = CXCStandardHeader()

            if check_stdgrps is True:
                expected = cxchdr.get_valid_groupings()
                groups = [ xx for xx in groupings if xx in expected ]
            else:
                groups = groupings

            cxchdr.build_hdr(groups)
            hdrlist = cxchdr._as_tuples()

        except Exception as ex:
            raise IOError(str(ex))
            
        # convert hdrlist to dict of CrateKeys
        stdhdr_dict = OrderedDict()

        # create a CrateKey for each hdrlist item
        # hdlist item = (keyname, value, unit, description, [groupings])
        for keyitem in hdrlist:
            key = CrateKey( keyitem )

            # append to dictionary
            stdhdr_dict[keyitem[0]] = key

        return stdhdr_dict


    def copy_stdhdr (self, groupings, targetcrate, add_missing):
        """
        Copies keys in given groupings from current Crate to the target Crate. 

        If add_missing flag is True, the targetcrate will contain all the keys belonging to 
        the input group(s) even if they are not in the current crate.

        If add_missing is False, only the keys from the origin Crate belonging to the input 
        group(s) will be copied to the targetccrate.

        """

        # most args checked in get_stdhdr()
        if not isinstance(targetcrate, (pycr.TABLECrate, pycr.IMAGECrate)):
            raise ValueError("The 'targetcrate' argument must be a TABLECrate or an IMAGECrate.")

        # generates a CrateKey dict of all the keys in requested grouping(s) in the target Crate
        header = self.get_stdhdr(groupings, add_missing)

        targetcrate.put_stdhdr(header, add_missing)


    def get_stdhdr (self, groupings, add_missing):
        """
        Loop through the existing CrateKeys and return a dict of keys that match the requested groupings.
        Returns None if there are no matching CrateKeys.
        """

        if len(self.__keylist) == 0:
            raise ValueError("This Crate does not contain any keywords.")

        if (add_missing is not True) and (add_missing is not False):
            raise TypeError("The 'add_missing' argument must be a boolean (True or False).")

        groupings = parse_groupings(groupings)
        
        ret_hdr = None
        
        # copy current CrateKeys matching the input grouping(s), including the 
        # Standard Header keys missing from the current Crate
        if add_missing is True:
            ret_hdr = self._build_stdhdr(groupings, True)

        # copy current CrateKeys matching the input grouping(s)
        else:
            ret_hdr = OrderedDict()

        for keyname in self.__keylist.keys():
            for grp in groupings:
                if grp in self.__keylist[keyname].groups or grp == "ALL":
                    ret_hdr[keyname] = copy.deepcopy(self.__keylist[keyname])   
                    break

        # if there are no matches, return None
        if len(ret_hdr) == 0:
            ret_hdr = None

        return ret_hdr


    def put_stdhdr(self, hdrdict, add_missing):
        """
        Put the provided header content into the Crate.

        'add_missing':
          - if True, will add any keys not currently in the Crate
          - if False, only matching keys will be updated

        """
        if not isinstance(hdrdict, OrderedDict):
            raise TypeError("Input header must be an OrderedDict")

        if (all(isinstance(hdrdict[hdrkey], CrateKey) for hdrkey in hdrdict.keys())) is False:
            raise TypeError("Input header must be a OrderedDict of CrateKeys.")

        if (add_missing is not True) and (add_missing is not False):
            raise TypeError("The 'add_missing' argument must be a boolean (True or False).")

        # copy over any hdrdict keys not listed the current Crate's keylist
        if (len(self.__keylist) > 0):  

            for keyname in hdrdict.keys():
                item = hdrdict[keyname]
                if keyname in self.__keylist.keys():
                    self.__keylist[keyname].value = item.value
                    self.__keylist[keyname].unit = item.unit
                    self.__keylist[keyname].desc = item.desc
                    self.__keylist[keyname].groups = item.groups

                # any keys from hdrdict not currently in cr will be added
                else:
                    if add_missing is True:
                        self.__keylist[keyname] = CrateKey((item.name, item.value, item.unit, item.desc, item.groups))

        # if this Crate does not have a header, do a direct copy of input header
        else:
            self._set_keylist( copy.deepcopy(hdrdict) )

        return



    ## Header HISTORY and COMMENT functions #############################

    def _get_history(self):
        """
        Returns the pointer to the CXCHistory object.
        """
        return self.__history

    def print_history(self):
        """
        Prints the string representation of the CXCHistory list.
        """
        retstr = ""
        for record in self.__history:
            retstr = retstr + record[0] + " " + record[1] + "\n"  

        return retstr

    def get_all_records(self):
        """
        Returns all records, both Comments and History, stored in the CXCHistory list.
        """
        return copy.deepcopy(self.__history)

    def get_comment_records(self):
        """
        Returns the CommentRecords stored in the CXCHistory list.
        """
        history = copy.deepcopy(self.__history)
        retlist = []

        for record in history:
            if record[0] == "COMMENT" or record[0] == "BLANK":
                retlist.append( record ) 

        return retlist

    def get_history_records(self):
        """
        Returns the HistoryRecords stored in the CXCHistory list.
        """
        history = copy.deepcopy(self.__history)
        retlist = []

        for record in history:
            if record[0] == "HISTORY":
                retlist.append( record ) 

        return retlist


    def add_record(self, tag, content):
        """
        Appends record(s) to the Crate history list.
        
        Parameters
        ----------
          tag       : string
                      Indicates the flavor of the record to be added.
                      valid options are: "HISTORY" and "COMMENT"
          content   : string or list of strings  (see special case)
                      Text of record(s), provided as a string or list of strings.

          Special Case
            tag = "COMMENT" + None;  either alone or within list.
            Will produce a completely BLANK line, as opposed to a COMMENT line with empty text.

        Returns
        -------
          None

        Examples
        --------
          1) add_record("HISTORY", "myTool infile=/data/a.fits outfile=/tmp/out.fits clobber=True")
             Adds a single HISTORY record to the stack.  There are no restrictions to the length,
             format or content of the record text.
          
          2) add_record("COMMENT", "---------- Observation Keys ---------- ")
             Adds a single COMMENT record to the stack.  The record is associated with the the last
             keyword currently in the Crate.
          
          3) add_record("COMMENT", None)
             Adds a single empty record to the stack.  The record is associated with the the last
             keyword currently in the Crate.  Unlike an empty comment record, this form will NOT
             include the "COMMENT" tag on output.
          
          4) histlist = ["HISTORY  TOOL  :dmcopy  2016-08-11T20:27:22                             ASC00041",        
                         "HISTORY  PARM  :infile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____5ASC00042",
                         "HISTORY  CONT  :87231988n944/output/acisf05017_000N022_r0001b_rayevt3.fiASC00043",
                         "HISTORY  CONT  :ts[sky=bounds(region(/export/proc/tmp.ascdsl3/prs_run/tmASC00044",
                         "HISTORY  CONT  :p//L3AMRF____587231988n944/input/acisfJ0331514m274138_00ASC00045",
                         "HISTORY  CONT  :1N021_r0001_srbreg3.fits[bkgreg]))][bin sky=1][opt type=ASC00046",
                         "HISTORY  CONT  :i4]                                                     ASC00047",
                         "HISTORY  PARM  :outfile=/export/proc/tmp.ascdsl3/prs_run/tmp//L3AMRF____ASC00048",
                         "HISTORY  CONT  :587231988n944/output/acisf05017_000N022_r0001b_fbinpsf3.ASC00049",
                         "HISTORY  CONT  :fits                                                    ASC00050",
                         "HISTORY  PARM  :kernel=default                                          ASC00051",
                         "HISTORY  PARM  :option=image                                            ASC00052",
                         "HISTORY  PARM  :verbose=0                                               ASC00053",
                         "HISTORY  PARM  :clobber=yes                                             ASC00054"]
             add_record("HISTORY", histlist) 
             Adds a sequence of HISTORY records to the stack.  This sequence corresponds to an
             ASC-FITS compliant history record of a dmcopy command. see CXC history module.
          
          5) content = ["Some arbitrary comment text, followed by an empty comment",
                        "",
                        "With some more commentary, followed by a blank line",
                        None,
                        "and ending with a final statement", 
	         	"Thank you for reading this!"]
          
          6) histstr = "First History line\\nSecond History line \\nThird History line\\nFourth History line "
             add_record("HISTORY", histstr) 
             Hidden feature; a string delimited with newline characters will be interpreted as 
                             a list of strings, adding multiple records.       
             This can be useful for adding records from a toString() style method.
        """
        if tag != "HISTORY" and tag != "COMMENT":
            raise ValueError("Invalid input; tag must be 'HISTORY' or 'COMMENT'.")
        
        if content is None:
            if tag == "HISTORY":
                raise TypeError("History record input cannot be None.")
            input = [content]
        elif type(content) is list:
            if all( ((tag == "COMMENT" and xx is None) or isinstance(xx, str)) for xx in content) is False:
                raise TypeError("Content list items must be strings")
            input = content
        elif isinstance(content, str):
            input = content.split("\n")
        else:
            raise TypeError("Invalid input; content must be a python string or list of strings.")
        
        for line in input:
            if line is not None:
                if (tag == "HISTORY" and "HISTORY " == line[0:8]) or (tag == "COMMENT" and "COMMENT " == line[0:8]):
                    line = line[8:]

            if tag == "COMMENT" and line is None:
                self.__history.append( ["BLANK", "", ""] )
            else: 
                self.__history.append( [tag, line, ""] )


    def delete_history_records(self):
        """
        Removes all History records from the CXCHistory list.
        """
        for ii in range( len(self.__history)-1, -1 , -1):
            if self.__history[ii][0] == "HISTORY":
                del self.__history[ii]


    def delete_comment_records(self):
        """
        Removes all Comment records from the CXCHistory list.
        """
        for ii in range( len(self.__history)-1, -1 , -1):
            if self.__history[ii][0] == "COMMENT" or self.__history[ii][0] == "BLANK":
                del self.__history[ii]


    def print_full_header(self):
        """
        Prints the entire header.  This includes standard keywords interleaved with 
        History and Comment keywords.
        """
        records = self.__history
        keylist = self.__keylist

        retstr = ""

        if (self.get_nkeys() > 0):       
            for name in list(keylist.keys()):
                key = keylist[name]
                retstr = retstr + "\nKEY      " + name

                for rec in records:
                    if name == rec[2]:
                        if rec[0] == "BLANK":
                            retstr = retstr + "\n"
                        else:
                            retstr = retstr + "\n" + rec[0] + " " + rec[1]

            for rec in records:
                if "" == rec[2]:
                    if rec[0] == "BLANK":
                        retstr = retstr + "\n"
                    else:
                        retstr = retstr + "\n" + rec[0] + " " + rec[1]

        else:
            for rec in records: 
                if rec[0] == "BLANK":
                    retstr = retstr + "\n"
                else:
                    retstr = retstr + "\n" + rec[0] + " " + rec[1] 

        return retstr


    def get_histnum(self):
        count = 0
        for record in self.__history:
            if record[0] == "HISTORY" and record[1][-8:-5] == "ASC":
                count += 1
        return count


    ## COLUMN functions #################################################

    def get_colnames(self, vectors=True, rawonly=True):
        """
        Outputs as an array of the column names found matching the 
        vectors and rawonly criteria.

        vectors - flag to control format for vector columns
                     True:  shows vector notation
                     False: displays individual component columns
                  default = True

        rawonly - flag for including/excluding virtual columns
                     True:  exclude virtual columns
                     False: include virtual columns
                  default = False

        """
        return self._get_datalist_itemnames(vectors, rawonly)


    def print_colnames(self, vectors=True, rawonly=True):
        """
        Prints to screen the column names found matching the 
        vectors and rawonly criteria.

        vectors - flag to control format for vector columns
                     True:  shows vector notation
                     False: displays individual component columns
                  default = True

        rawonly - flag for including/excluding virtual columns
                     True:  exclude virtual columns
                     False: include virtual columns
                  default = False

        """
        print_list = self._get_datalist_itemnames(vectors, rawonly)

        print_str = "        Colname\n"
        for ii in range( len(print_list) ):
            print_str += ' {0:3d})   {1!s}\n'.format(ii, print_list[ii])

        return print_str


    def column_exists(self, colname):
        """
        Checks to see if the input column name exists.  Returns 'True' 
        if column exists, 'False' if it does not.
        """
        colname = pycr.convert_2_str(colname)

        dlist = self._get_datalist_itemnames(vectors=True, rawonly=False)

        for item in dlist:
            items_lc = item.lower()

            # search
            if colname.lower() == items_lc:
                return True

            else:
                # tokenize if vector column
                if '(' in items_lc:
                    # remove close parenthesis and split on open parenthesis
                    items_lc = items_lc.replace(')', '')
                    items_tmp = items_lc.split('(')
                    items_lc = [items_tmp[0]]

                    # split 2nd part of string on comma+space
                    items_tmp = items_tmp[1].split(', ')

                    # create full list of column and component names
                    items_lc = items_lc + items_tmp

                    if colname.lower() in items_lc:
                        return True

        return False
    

    def add_column(self, col, index=None):
        """
        Adds a column to the TABLECrate.

        col   - the column (CrateData object) to add to the crate
        index - numerical position of where to insert the column
        """

        if not isinstance(col, CrateData ):
            raise TypeError("Input must be a CrateData.")

        # add name to the datalist
        if len(col.name) == 0:
            ii = 1
            while True:
                tmpname = "COLUMN" + str(self.__ncols+ii)
                ii = ii+1
                if not self.column_exists(tmpname):
                    col.name = tmpname
                    break
                   

        self._append_to_datalist( col, index=index )
        self.__dict__[ col.name ] = col
        
        # update number of rows and columns
        self.__ncols = len(self.__datalist)           
        if col.get_nsets() > self.__nrows:
            self.__nrows = col.get_nsets()
            
        # add subspace if subspace list is empty
        if not self.__subspace:
            self.__subspace = [pycr.CrateSubspace()]
            
        # add subspace data
        for ii in range(1, len(self.__subspace)+1):
            try:
                ssd = self.get_subspace_data(ii, col.name)
            except:
                ssdata = pycr.CrateSubspaceData()
                ssdata.name = col.name

                if col.values is not None:
                    dtype = col.values.dtype
                else:
                    dtype = float64  # default

                if col.vdim > 1:
                    colcpts = col.get_cptslist()

                    for kk in range(col.vdim):                    
                        cptname = colcpts[kk]
                        ssdata.cpts.append( cptname )
                        ssdata_cpt = pycr.CrateSubspaceData()
                        ssdata_cpt.name = cptname
                        ssdata_cpt.range_min = array(ssdata_cpt.range_min, dtype)
                        ssdata_cpt.range_max = array(ssdata_cpt.range_max, dtype)
                        
                        self.__subspace[ii-1].add_subspace_data(ssdata_cpt)

                        
                ssdata.range_min = array(ssdata.range_min, dtype)
                ssdata.range_max = array(ssdata.range_max, dtype)

                self.__subspace[ii-1].add_subspace_data(ssdata)


    def delete_column(self, col):
        """
        Removes a column from the Crate.
        
        col - the name or number (zero-based) of the column to delete

        """
        col = pycr.convert_2_str(col)

        datalist = list(self._get_datalist().keys())

        if not isinstance( col, str):
            if len(datalist) == 0:
                return
            
            try:
                colname = datalist[col]
            except:
                raise IndexError("Invalid column number.")
        else:
            colname = col

        if self.column_exists(colname):

            # delete virtual column(s) that are dependent on column to be deleted
            for col1name in (datalist):
                if col1name != colname:
                    col1 = self.get_column(col1name)
                    if col1.source is not None:
                        src_name = col1.source.name
                        if src_name == colname :
                            self._remove_from_datalist(col1name)
 
            found = 0

            for name in (datalist):
                if colname.lower() == name.lower():
                    # remove input column name from datalist
                    self._remove_from_datalist( name )
                    found = 1
                    break
                    
            if not found:
                raise ValueError("Column '" + colname + "' cannot be deleted since it is a component of '" + self.get_column(col).parent.name + "'.")

            self.__ncols = len(self.__datalist)

            if self.__ncols == 0:
                self.__nrows = 0
        else:
            raise ValueError("Column '" + colname + "' does not exist.")


    def get_column(self, col):    
        """
        Finds and returns a column from the TABLECrate.
        
        col - the name or number (zero-based) of the column to retrieve

        """
        col = pycr.convert_2_str(col)

        if not isinstance( col, str):
            if len(self.__datalist) == 0:
                return
            
            try:
                datalist = list(self._get_datalist().keys())
                colname = datalist[col]
            except:
                raise IndexError("Invalid column index number " + str(col) + ".")
        else:
            colname = col

        column = self._retrieve_from_datalist(colname)
        if column is None:
            raise ValueError("Column '" + colname + "' does not exist.")
        
        return column


    def _get_datalist(self):
        """
        Retrieves the python dictionary containing regular column names.
        """
        return self.__datalist


    def _get_datalist_itemnames(self, vectors=True, rawonly=True):
        """
        Retrieves the list containing regular column names plus
        vector component names and virtual columns.
        """
        retlist = []
        
        for item_name in list(self.__datalist.keys()):
            item = self.__datalist[item_name]

            if (rawonly == False and item.is_virtual()) or item.is_virtual() == False:

                if vectors == True:
                    if item.is_vector():
                        retlist.append( item._get_full_name() )
                    else:
                        retlist.append( item_name )
                else:
                    if item.is_vector():
                        cptslist = item.get_cptslist()
                        for ii in range(item.vdim):
                            retlist.append( cptslist[ii] )
                    else:
                        retlist.append( item_name )

        return retlist


    def _copy_datalist(self):
        """
        Returns a copy of the data list.
        """

        return self.__datalist.copy()


    def _set_datalist(self, in_dict):
        """
        Replaces the data list with input dictionary.
        """
        if isinstance(in_dict, OrderedDict):
            self.__datalist = in_dict
            self._set_data_flag(True)
        else:
            raise TypeError("Input must be a OrderedDict.")


    def _get_data_flag(self):
        """
        Returns the data flag.  'True' means that column data 
        has been read in from the input file.
        """
        return self.__data_flag


    def _set_data_flag(self, flag):
        """
        Sets the data flag.
        """
        self.__data_flag = flag


    def _append_to_datalist( self, dataitem, name=None, index=None ):
        """
        Adds a new column name to the data list.
        """
        if not isinstance(dataitem, pycr.CrateData):
            raise TypeError("Item must be a CrateData.")

        if name is not None and isinstance(name, str):
            itemname = name
        else:
            itemname = dataitem.name
        
        #check if index exists and is valid
        if index is not None :
            try:
                ndx = int(index)
            except:
                raise TypeError( "Index argument must be compatible with integer." )

            if (ndx > len(self.__datalist) or ndx < 0):
                raise IndexError("Column index is out of range")

            if (ndx == len(self.__datalist)):
                self.__datalist[ itemname ] = dataitem
            else:
                newdict = OrderedDict()
                ii = 0
                for (itemkey, itemvalue) in (self.__datalist.items()):
                    if ndx == ii:
                        newdict[itemname] = dataitem
                    
                    newdict[itemkey] = itemvalue
                    ii = ii + 1

                self.__datalist = newdict
        else:
            self.__datalist[ itemname ] = dataitem



    def _remove_from_datalist( self, itemname):
        """
        Removes a column name from the data list.
        """
        if itemname in self.__datalist:
            if itemname == "image":
                self.__data_trash.append( self.__datalist["image"].name )
            else:
                self.__data_trash.append( itemname.lower() )

            self.__clear_vector_parents(self.__datalist[itemname])
            del self.__datalist[itemname]


    def _retrieve_from_datalist( self, itemname ):
        """
        Searches the datalist for the input item name and 
        returns the item if found.
        """
        # Try with the requested capitalization
        if itemname in self.__datalist:
            return self.__datalist[itemname]

        # Fall back to looking for different capitalizations or vectors
        itemname = itemname.lower()
        for colname in list(self.__datalist.keys()):
            if colname.lower() == itemname:
                return self.__datalist[colname]

            # loop through items in datalist for vector columns
            column = self.__datalist[colname]
            if column.is_vector():
                cpts = column.get_cptslist()
                for cptname in cpts:
                    if cptname.lower() == itemname:
                        return column.get_vector_component(cptname)


    def _get_data_trash(self):
        return self.__data_trash[:]


    ## SIGNATURE functions ##############################################

    def is_modified(self):
        """
        Returns True is the Crate information has been changed or 
        updated, False if not.
        """
        current_sig = self.__calculate_signature()

        if current_sig != self.get_signature():
            return True
        return False


    def __calculate_signature(self):
        """
        Calculates checksum of the Crate in it's current state.
        """
        sigstr = self.__str__()
        sigstr += str(list(self.__keylist.values()))
        sigstr += str(list(self.__datalist.values()))

        try: 
            for item_name in list(self.__datalist.keys()):
                vals = self.__datalist[item_name].values
                if vals is not None:
                    sigstr += str(vals)
        except MemoryError:
            sigstr = None
            return None

        return hashlib.sha256( sigstr.encode('utf-8') ).hexdigest()


    def get_signature(self):
        """
        Retrieves the stored checksum of the Crate.
        """

        if len(self.__signature) == 0:
            self.update_signature()
        return self.__signature


    def update_signature(self):
        """
        Recalculates and stores the new checksum.
        """
        self.__signature = self.__calculate_signature()


    ## TABLE related funcions ###########################################

    def get_ncols(self):
        """
        Retrieves the number of columns in the TABLECrate.
        """
        return self.__ncols


    def get_nrows(self):
        """
        Retrieves the number of rows in the TABLECrate.
        """
        return self.__nrows


    def _set_ncols(self, val):
        """
        Sets the number of columns.
        """
        self.__ncols = val


    def _set_nrows(self, val):
        """
        Sets the number of rows.
        """
        self.__nrows = val


    ## IMAGE related funcions ###########################################

    def get_axis(self, axis_name):
        """
        Returns the requested axis.
        """
        axis_name = pycr.convert_2_str(axis_name)

        retaxis = None

        axislist = list(self.__axes.keys())
        for name in (axislist):
            if axis_name.lower() == name.lower():
                retaxis = self.__axes[ name ]
                break

        if retaxis:
            return retaxis
        else:
            raise KeyError(axis_name + " not found in list of image axes.")


    def get_axisnames(self):
        """
        Returns an array of the names of the image axes.
        """ 
        return list(self.__axes.keys())


    def _set_axes(self, axis_dict):
        """
        Sets the Crate's axes.
        """
        if isinstance(axis_dict, OrderedDict):
            self.__axes = axis_dict
        else:
            raise TypeError("Input must be a OrderedDict.")
        

    def _get_axes(self):
        """
        Returns the python dictionary of coordinate transforms associated 
        with the image. 

        """
        return self.__axes
    

    def get_ncoords(self):
        """
        Returns the number of coordinate transforms associated 
        with the image. 
        """
        return len(self.__axes)


    def get_image(self):
        """
        Returns the image in the Crate.
        """
        if "image" in self.__datalist:
            return self._retrieve_from_datalist( "image" )
        else:
            raise LookupError("Image data does not exist.")


    def add_image(self, in_image):
        """
        Adds an image to the Crate or overwrites the existing image.
        """
        if isinstance(in_image, pycr.CrateData):
            in_image._image = True
            self._append_to_datalist(in_image, name="image")
            self._set_data_flag(True)

            # add subspace if subspace list is empty
            if not self.__subspace:
                self.__subspace = [pycr.CrateSubspace()]
            
            # add subspace data
            for ii in range(1, len(self.__subspace)+1):

                for axisname in ('X', 'Y'):
                    try:
                        ssd = self.get_subspace_data(ii, axisname)
                    except:
                        ssdata = pycr.CrateSubspaceData()
                        ssdata.name = axisname

                        if in_image.values is not None:
                            dtype = in_image.values.dtype
                        else:
                            dtype = float64  # default
 
                        ssdata.range_min = array(ssdata.range_min, dtype)
                        ssdata.range_max = array(ssdata.range_max, dtype)
                            
                        self.__subspace[ii-1].add_subspace_data(ssdata)
                            
        else:
            raise TypeError("Input must be a CrateData.")


    def delete_image(self):
        """
        Removes image from the Crate.
        """
        if "image" in self.__datalist:
            self._remove_from_datalist("image")
        else:
            raise LookupError("Image data does not exist.")


    ## TRANSFORM related funcions #######################################

    def get_transform(self, trname):
        """
        Searches the axes or columns lists for named transform.  
        If found, the transform is returned, else raises one of the following exceptions:

            TypeError: Input transform name must be a string.
            KeyError: <name> not found in list of image axes.
            ValueError: Column <name> does not exist.
        """

        trname = pycr.convert_2_str(trname)
        if not isinstance(trname, str):
            raise TypeError("Input transform name must be a string.")

        item = None
        
        if self.is_image():
            item = self.get_axis(trname)
        else:        
            item = self.get_column(trname)

        return item.get_transform()

