
# 
#  Copyright (C) 2011-2014,2016,2020-2021  Smithsonian Astrophysical Observatory
#
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#


# Oct -21 -2010

# caldb4.py
# this file

"""
A python module for CALibration Data Base library querying.

Exported Classes:

Caldb -- User Interface Python Class Object


Usage:

1> 'Caldb' object Creation

    To create the Caldb object, 'c', make following call with
    three arguments, telescope, instrume, and product,
    
      c = Caldb(telescope='tele', instrume='inst', product='prod')

    These arguments can be also optional as

      c = Caldb('tele', 'inst')
    for product defult to 'None'.

    Or
    
      c = Caldb('tele')
    for additonal instrume default to 'default',

    or
    
      c = Caldb()
    for additonal telescope default to 'chandra'

  Note that, both 'telescope' and 'instrume' are immutable attributes of the
  current object or their values are not updatable. Once the object is
  launched, the session of the object is to hold the values of telescope and
  instrume until the end.
  
    
2> The Object Attribute Assignments 
    
    The assignment syntax is
    
      object.attr=value
    or

      object.attr=value,unit

    For example,

      c.start='1999-02-12T06-09-35'

    asign object 'c' attribute of 'start' with the string value.

      c.fp_temp=148.0,'K'
      
    assign the 'c' 'fp_temp' with an integer value and 'K' unit.


3> The Object Attribute Updates

   The 'Caldb' object attributes can be updated with new values or new types.
   But attributes of 'telescope' and 'instrume' are exceptional which are
   fixed at the time of launching Caldb object.
   
   Any  unwanted attribute is deletable. The syntax of deleting is
   
    del object.attr

   For example,
   
    del c.fp_temp

   will delete 'fp_temp' from the object, 'c'

4> The Method Properties of the Caldb object:

   c.search                  - search for the queried resutls
   c.close                   - end the current session


5> Summary of User Interface Syntax
   (o=object)

   - set parameter
     o.name=val   or/and    o.name=val,unit

   - get the named value
    myval = o.name  or  myval,myunit = o.name

    or
    print o.name

   - get all inputs of (name, value) pairs
     get the need parameter to be queried
    print o

   - delete a named parameter
    del o.name

6> Get start
   % python
   >>> from caldb4 import *
   >>> calhelp()


Run Example

see "def usage()"

"""

# Support python 2 and 3
from __future__ import print_function

import sys, os
from types import *                     # for python types operation
from . caldblib import *                 # c-function module
from . calattr  import Calattr,Caldict   # the Python Magic methods
# Py3:
# use relative path, '.', here is because 
# caldblib,calattr are local modules which are should
# be instructed in relative path fashion
import collections       # for ordered dictionary   



# python2 cmp is removed in python3, so re-define it here
# to be done
def cmp_mydicts(A, B):
  if len(A) != len(B):
    return 1 #len(A)-len(B)
  for ak,av in A.items():
    if ak in B.keys() and av != B[ak]:
      return 1  #unmatched
    
  return 0;  #matched
      
#---------
# read this for convert Py2 cmp to Py3 compatible
# functools.cmp_to_key(func)
# https://docs.python.org/3/howto/sorting.html#sortinghowto

#=============================================================================
#
#   --- The Class and the Function ---
#
#=============================================================================


#Clsneeds=Caldict()     #the need-parameter container
#Clsattrs=Caldict()     #the input-attributes log, updateable

Clsneeds = collections.OrderedDict()
Clsattrs = collections.OrderedDict()

class Caldb(CDBlib, Calattr):
  """
  User-Interface CALDB querying
  """
  # get rid of __dict__ object listed below, which created by
  # interpreter for each instance of class.
  # 
  __slots__ = ['_curprod','_curtele','_curinst', '_infile', 'maphdr', # control flags
               'libcptr', 'srchptr']                                  # defined in caldblib.py 
               
  _confLst = ["telescope", "instrume", "infile","caldb"]
  

  def __str__( self): return self.__repr__()
  def __repr__(self):  
    if not self.errmsg:
      return self.__print__() + self._lstattrs()+ self._lstneeds()
    elif self.errmsg.find("ERROR") != -1 or self.errmsg.find("Error") != -1 :
      return "\n"+self.errmsg
    elif not self.libcptr or self.libcptr==0:
      return "\nError: null caldb pointer."
    else :
      return self.__print__() + self._lstattrs()+ self._lstneeds()
  
  def __print__(self):
    """print the defaul Class info"""

    attrLst = self.__class__._confLst

    classinfo = "\n\t%s\
    \n\tThe '%s' configuration:\n" % (50*'=',self.__class__.__name__)
    
    classinfo = classinfo+"\n".join(["\t%s: '%s'" % (key.rjust(10), self.__dict__[key]) for key in attrLst])
    classinfo = classinfo+"\n\t%s\n" %(50*"=")

    return classinfo

  def _lstattrs(self):
    """ list the input attributes.
    """

    # allinp=self.attrdicts.copy()
    allinp=convert_to_ascii_dictval(self.attrdicts)
      
    attrLst = self.__class__._confLst

    lstattrs=""
    for key in allinp.keys():
      if key in attrLst or key.lower() in attrLst or key == "errmsg":
        continue
      
      val=allinp[key]
      if isinstance(val, str):
        lstattrs=lstattrs + "\t%s = '%s'\n" % (key.rjust(10), val)
      else:
        lstattrs=lstattrs + "\t%s = %s\n" % (key.rjust(10), str(val))

    if lstattrs != "":
      lstattrs="\n\t***** The currently queried *****\n"+lstattrs

    return lstattrs




  @property
  def allneeded(self):
    """
    allneeded - Get all parameters that might be turned into attributes.
                Return a list of '(name, unit)' pairs.

    """
    self._updProd()
    self._updNeeded()        # update the current needed params
    allneeds=Clsneeds.copy() # need copy() as dict changes size during iteration

    attrLst = self.attrdicts.keys()  #tb done on the list
    if attrLst=={}:
      return list(allneeds.items())

    for key in Clsneeds.keys():
      if key in attrLst or key.lower() in attrLst:
        del allneeds[key]
        continue      

    return list(allneeds.items()) # compatible in Py2 and Py3
  
  def _lstneeds(self):
    """
    list the needed parameters.
    return string type
    """


    allneed=self.allneeded
    lstattrs=""

    if not allneed:
      return ""

    for (key,val) in allneed:
      if isinstance(val, str):
        lstattrs=lstattrs + "\t%s = '%s'\n" % (key.rjust(10), val)
      else:
        lstattrs=lstattrs + "\t%s = %s\n" % (key.rjust(10), str(val))

    if lstattrs !="":
      lstattrs="\n\t***** The to-be queried *****\n"+lstattrs

    return  lstattrs

  def __init__(self, telescope="chandra", instrume="default", product="", infile=""):
    """
    Caldb(tele, inst, prod, infile) - Launch Caldb with arguments.

          Arguments:
            telescope   -- string of telescope,  default to "chandra"
            inststrume  -- string of instrume,   default to "default"
            product     -- string of product,    default to None
            infile      -- string of input file for auto-params,  default to None

          Note:
          At API level,
    
          1> the instance of this method is a call of the named class
          that shall automatically launch the '__init__' method.
          For example, the action of
    
          c = Caldb('mytele', 'myinst', 'myprod')

          is to create an object, 'c', after activating ' __init__' internally.
          or
          c = Caldb("", "" , 'myprod', 'myinfile')
          will execute the querying with 'myprod' 
          via the embedded keywords from the header of the 'myinfile' file.

          2> ignore the first argument, 'self', of the function which is
          Python 'Class' internal property.
    
    """


    # Check environs first, raise KeyError if neither set
    try: 
      os.environ['CALDB']
      os.environ['CALDBCONFIG']
      os.environ['CALDBALIAS']
    except KeyError as err :
      raise caldbError("Error: failed to retrieve Env. variable {}.".format(str(err)))

    # Do initialization and avoiding confusions
    self.errmsg=""
    self.product=""

    if telescope == None or telescope == 0:
      telescope=""
    if instrume == None or instrume == 0:
      instrume=""
    if infile == None or infile == 0:
      infile=""
    if product == None or product == 0:
      product=""

    # Launch the base class
    stat = CDBlib.__init__( self, telescope, instrume, infile)
    if not stat:
      errmsg=self.errmsg
      self.errmsg=""
      raise caldbError(errmsg)


    # Launch the rest of them
    Calattr.__init__(self)   # user setting attributes

    # -------------------
    # initiate properties
    # -------------------
    if self.infile == "":           # self.infile assigned in CDBlib() level
      self._curtele  = telescope    # set telescope flag
      self._curinst  = instrume     # set instrument flag
      
    if product == b'' or product == "" :
      stat=1   # do none
    else :     # look up product
      stat=self._setProd(product)
      if not stat:
        errmsg=self.errmsg
        self.errmsg=""
        raise caldbError(errmsg)
  
    self.CALDB  = os.environ['CALDB'] 


  @property
  def close(self):
    """
    close - End the current Caldb interface session.
            
    """
 
    #clean the dictionary and revert the Python default prompt
    Clsneeds.clear()
    
    if not self.libcptr:
      return
    
    self._close()
    
  def _setProd(self, prod):
    """
    Set calibration product name.
    
         Argument:
         prod -- The product name in string.
    """

    try:
      stat=self._setProduct(prod)   # talk to libcaldb, any error would be raised 
      if not stat:
        return False
      stat=self._fillNeeded()       # fill the needed container, any error would be raised 
      if not stat:
        return False
 
      self._curprod=self.product       # set product flag

      #clean and update input-log dict
      #Clsattrs.clear()
      #Clsattrs.merge(self.attrdicts)
      Clsattrs=self.attrdicts      
      return True
    except caldbError as e:  #catch error, if any, raised from the above two
      self.errmsg=e.value

    return False

  def _updProd(self):
    # update product
    if not self.libcptr:
      return False

    stat=True;
    if self.product  and self._curprod != self.product:       
      stat=self._setProd(self.product)
    return stat


  def _fillNeeded(self, addtime=1):
    "Updated the current parameters to be queried"

    wpars=self._whichParams()

    Clsneeds.clear()
    if wpars==[]:
      if self.errmsg and self.errmsg.lower().find("error:") != -1:
        return 0
      else:
        return 1 
    
    for n,v,u in wpars:
      Clsneeds[n]=u

    #add the date/time attributes which ar not picked up in libcaldb
    if addtime:
      timeattr={'start':'','stop':''}
      #Clsneeds.merge(timeattr)
      Clsneeds['start'] = ''   
      Clsneeds['stop']  = ''


    return 1

  def _updNeeded(self):
    "Remove the name from Clsneeds list if it present in user' inputs"

    if len(Clsneeds)==0:
      return
    uinput={}
    uinput=self.attrdicts.copy()
    if len(uinput) == 0:
      return

    for name,val in uinput.items():
      uname=name.upper()

      if self._notsetPars(uname) == False:        
        if uname in Clsneeds.keys():
          del Clsneeds[uname] 
 

  def _notsetPars(self, name):
    name=name.lower()
    if name[:4] == "tele" or name[:4] == "inst"  or name[:4] == "prod" or name=="caldb" or name[:4] == "errm" or name=="infile":
      return True
    else:
      return False

  def _setParams(self):
    "Return True if params is pushed in to caldb-lib else False."
    
    if not self.libcptr:
      self.errmsg="Error: null caldb-lib pointer."
      return False

    uinput=self.attrdicts
         
    try:
      for name,value in uinput.items(): 
        if self._notUpdatable(name):
          continue
        name=name.lower()
        if not isinstance(value, tuple):
          value = value,""
        val,unit=value
        
        if self._notsetPars(name):
          continue
        elif name == "start":
          stat=self._setStartTime(val)
        elif name == "stop":
          stat=self._setStopTime(val)
        else:
          stat=self._setParam(name, val, unit)
        if not stat:
          raise caldbError (self.errmsg)

      return True
      
    except caldbError as e:
      self.errmsg=e.value
      return False
    except:
      self.errmsg ="Error: unexpected failure in setParams() ."
      return False
 
  @property  
  def search(self): 
    """
    search  - Retrieve calibration files on "object.name=value" queries.
              Return a list of the retrieved files otherwise an error message.

    """
    
    if not self.libcptr:
      if self.errmsg and isstring(self.errmsg) and self.errmsg.strip()!="":
        raise caldbError(self.errmsg)
      else :
        raise caldbError("Error: null caldb-lib pointer.")

  
    try:
      if  not self._curprod  and not self.product:
        raise caldbError("Error: not set 'product' yet.")
      elif self.product=="":
        raise caldbError("Error: empty 'product'.")

      if cmp_mydicts(Clsattrs, self.attrdicts): 
        self._setProd(self.product)
      else:
        self._updProd()


      stat=self._setParams()
      if not stat:
        raise caldbError(self.errmsg)
       
      nfiles = self._search()

      # catched an error
      if nfiles==-1 :  # -1==error
        raise caldbError(self.errmsg)   

      if nfiles==0 :  # 0==not-found
        return []

      if self.maphdr:
        stat=self._fillNeeded(0)
        if not stat:
          raise caldbError(self.errmsg)
   
      files=[]
      for ii in range(nfiles):
        filn=self._getFile(ii)
        if len(filn) != 0:
          files.append(filn)
 
      return files
    except caldbError as e:
      self.errmsg=""    # clean it
      raise caldbError(e.value)
    except:
      raise caldbError("Error: unexpected failure in search() .")

 
def calhelp():
  """
  Show how to get help for Caldb python module application.
  """
  
  print ("""
  ================================================
  Help for info of caldb4.py package
  >>> help('caldb4')
  
  Help for document of Caldb module class
  >>> help(Caldb)
  
  The Caldb Usage:
  >>> usage()


  ================================================
  Methods or Descriptors  defined in the Python Caldb:
  """)
  print (Caldb.__init__.__doc__ ,"\n",\
        Caldb.search.__doc__   ,"\n",\
        Caldb.close.__doc__    ,\
        """
   ================================================
   Syntax of Object Attribute Assignemnts
        
        c.attr=val       or
        c.attr=val,unit\n
     For example, an object 'c' defined as c=Caldb(<tele>,<inst>)
     to assign 'c' attribute 'start' with the string value.
        c.start='1999-02-12T06-09-35'
        \n
     to assign 'c' attrbuite 'fp_temp' with a float value and 'K' unit.
        c.fp_temp=148.0,'K'
        
        """)
     
  
#
# pycaldb4 usage 
# http://www.python.org/download/releases/2.2/descrintro/#staticmethods
#
def usage():
  """Show the usage with example for Caldb python module."""

  print ("\nUsage with example:\n")
  print ("""% Python
>>> from caldb4 import * 
>>> c = Caldb("chandra", "acis", "badpix")
>>> c.start='2000-01-29T20:00:00' 
>>> c.stop='2000-11-28' 
>>> c.fp_temp=158.0,'K' 
>>> f=c.search
>>> len(f)
10
>>> f[0]
'/data/regression_test/development/CALDB4/data/chandra/acis/badpix/acisD2000-11-28badpixN0003.fits[1]'
>>> f[1]
'/data/regression_test/development/CALDB4/data/chandra/acis/badpix/acisD2000-11-28badpixN0003.fits[2]'
>>> 
>>> c.ccd_id=3
>>> c

        ==================================================    
        The 'Caldb' configuration:
         telescope: 'chandra'
          instrume: 'acis'
             caldb: '/data/regression_test/development/CALDB4/'
        ==================================================

        ***** The currently queried *****
           product = 'badpix'
              stop = '2000-11-28'
            ccd_id = 3
             start = '2000-01-29T20:00:00'
           fp_temp = (158.0, 'K')

        ***** The to-be querrys *****
           GRATING = ''
          GRATTYPE = ''

>>> c.search
['/data/regression_test/development/CALDB4/data/chandra/acis/badpix/acisD2000-11-28badpixN0003.fits[4]']
>>>
>>> print (Caldb.search.__doc__

    search  - Retrieve cal. files on "object.name=value" queries.
              Return a list datatype of the retrieved cal. files.
              
    
>>> c.close
""")

def main():
  
  global Path_Infile

  Path_Infile = os.getenv("TESTIN")
  if Path_Infile:
      Path_Infile += "/calquiz"
  
  if len(sys.argv) == 1:
    # test stand-alone package 
    test1()              
    test2()              
  elif len(sys.argv) == 2:
    if sys.argv[1]=="-u" or  sys.argv[1]=="--usage":
      usage();
    elif sys.argv[1]=="-d" or  sys.argv[1]=="--doc":
      calhelp()  
    elif sys.argv[1]=="-h" or  sys.argv[1]=="--help":
      print ("\npython ",sys.argv[0])
      print ("python ",sys.argv[0]," --usage  | -u")
      print ("python ",sys.argv[0]," --doc    | -d")
      print ("python ",sys.argv[0]," --help   | -h")
      print ("python ",sys.argv[0]," --cxcsys | -c\n")
    elif sys.argv[1]=="-c" or  sys.argv[1]=="--cxcds":
      # test cxcds-dependent package 
      mkgrmf()
      mkarf_qe()
      mkarf_axe()
      mkarf_vig()
      mkarf_gre()
      ccd_simz()
      lookup_time()
      cdev_http()



def test1():
  print ("\n...... Querying Caldb(<telescop>, <instrume>, 'BADPIX') ......\n")
  c = Caldb("chandra", "acis", "badpix")
  #c.start='2000-01-29T20:00:00' 
  c.start='2000-11-28'
  c.fp_temp=158.0,'K'
  #print (c)
  f=c.search
  print (len(f), " files are found")
  print ("1> ", f[0])
  print ("2> ", f[1])
  print ("\nset ccd_id=3 and re-search, ")
  c.ccd_id=3
  #print (c)
  f=c.search
  print (len(f), " file is found")
  print ("1> ", f[0])
  c.close

def test2():
  print ("\n...... Querying Caldb(<telescop>, <instrume>, 'DET_GAIN') ......\n")

  c = Caldb("chandra","acis","det_gain")
  c.fp_temp=153.2
  c.cal_qual=':5'
  print (c)
  files=c.search
  c.close
  print (len(files), " files are found")
  ii=1
  for f in files:
    print (ii,"> ", f)
    ii+=1


def mkgrmf():
  print ("\n...... Querying Caldb("","", 'LSFPARM', <infile>) ......\n")
  file="obs_1921_evt1.fits"

  dir=Path_Infile #"/data/regression_test/development/indata/calquiz"
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return


  c = Caldb("","","lsfparm", infile)
  c.order=1
  c.rand_tg=0.5
  c.shell='1100'
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("acismeg1D1999-07-22lsfparmN0003.fits[1]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

def mkarf_qe():
  print ("\n...... Querying Caldb("","", 'QE', <infile>) ......\n")

  file="obs_3399_evt1.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","QE", infile)
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  print ("2> ", f[1])
  print ("3> ", f[2])
  if f[0].find("hrcsD1999-07-22qeN0009.fits[1]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

def mkarf_axe():
  print ("\n...... Querying Caldb('""','""', 'AXEFFA', <infile>) ......\n")

  file="obs_3399_evt1.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","AXEFFA", infile)
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("hrmaD1996-12-20axeffaN0007.fits[1]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

def mkarf_vig():
  print ("\n...... Querying Caldb("","", 'VIGNET', <infile>) ......\n")

  file="obs_3399_evt1.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","VIGNET", infile)
  c.grating='letg'
  c.shell='1000'
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("hrmaD1996-12-20vignetN0003.fits[2]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

 
def mkarf_gre():
  print ("\n...... Querying Caldb("","", 'GREFF', <infile>) ......\n")

  file="obs_3399_evt1.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","GREFF", infile)
  c.grating='letg'
  c.shell='0100'
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("letgD1996-11-01greffpr001N0005.fits[2]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

def ccd_simz():
  print ("\n...... Querying Caldb('','', 'BKGRND', <infile>) ......\n")

  file="evt2.ccd7.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","BKGRND", infile)
  c.ccd_id=7
  c.shell='0100'
  #print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("acis7sD2000-12-01bkgrnd_ctiN0001.fits[1]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

def lookup_time():
  print ("\n...... Querying Caldb('','', 'DET_GAIN', <infile>) ......\n")

  file="acism62566_000N000_bpix1.fits"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("","","det_gain",infile)
  #c.fp_temp=183.4
  f=c.search
  print (c)
  c.close

  print (len(f), " file is found")
  if len(f) != 0:
    print ("1> ", f[0])
    if f[0].find("acisD1999-08-13gainN0002.fits") != -1:
      print ("OK!")
    else:
      print ("not-OK!")
  else:
    print ("not-OK!")
     

def cdev_http():
  print ("\n...... Querying Caldb('CHANDRA','ACIS', 'DET_GAIN', infile) ......\n")

  file="acisf00635_000N004_evt1.fits.gz"
  dir=Path_Infile 
  infile= dir +'/'+file
  if not os.path.exists(infile):
             #skip the testing
    print ("\ninfile '",infile,"', does not exists. Skip the test.")
    return

  c = Caldb("chandra","acis","det_gain", infile)
  print (c)
  f=c.search
  c.close

  print (len(f), " file is found")
  print ("1> ", f[0])
  if f[0].find("acisD2000-01-29gain_ctiN0006.fits[1]") != -1:
    print ("OK!")
  else:
    print ("not-OK!")

if __name__ == "__main__":
  main()
