rcReportGen.py

Go to the documentation of this file.

#!/usr/local/bin/python
#
#                               Copyright 2005
#                                     by
#                        The Board of Trustees of the
#                     Leland Stanford Junior University.
#                            All rights reserved.
#

__facility__ = "Online"
__abstract__ = "Run report generator class for the ScriptEngine"
__author__   = "S. Tuvi <stuvi@SLAC.Stanford.edu> SLAC - GLAST LAT I&T/Online"
__date__     = "2005/07/23 00:08:27"
__updated__  = "$Date: 2006/04/28 01:41:21 $"
__version__  = "$Revision: 1.10 $"
__release__  = "$Name: HEAD $"
__credits__  = "SLAC"

import LICOS.copyright_SLAC

import os
import logging as log
import time
import calendar
import cStringIO
import MySQLdb

from xml.sax.saxutils  import XMLGenerator
from xml.parsers.expat import ParserCreate

UNITS_INT    = 0
UNITS_LONG   = 1
UNITS_FLOAT  = 2
UNITS_STRING = 3
UNITS_LIST   = 4
UNITS_DICT   = 5

UNIT_MAP = {
            'OperatorId'      : UNITS_LONG,
            'EventCount'      : UNITS_LONG,
            'BadEventCount'   : UNITS_LONG,
            'ErrorEventCount' : UNITS_LONG,
            'PauseCount'      : UNITS_INT,
            'ElapsedTime'     : UNITS_FLOAT,
            'CompletionStatus': UNITS_INT,
           }

# Shift Dictionary
SHIFT_DICT =  { 0: ("T00:00:00", "T08:00:00", "Owl"),
                1: ("T08:00:00", "T16:00:00", "Day"),
                2: ("T16:00:00", "T00:00:00", "Swing")
              }

class rcReportGen(object):
  """!\brief Run report generator class for the ScriptEngine.

  """
  def __init__(self, fileName="rcReport.out", prefs=None):
    """!\brief rcReportGen constructor.

    \param fileName Run report file name
    \param prefs    Preferences instance

    """
    self.__fileName  = fileName
    self.__prefs     = prefs
    if prefs is not None:
      self.__elognbl   = int(prefs["elognbl"])
    else:
      self.__elognbl   = 0


  def initialize(self, tstamp=None):
    """!\brief Initialize the run report generator.

    \param tstamp Timestamp of the run
    """
    self.__tstamp = tstamp
    self.__repFields = []
    self.__repData = []
    self.__repUnits = []
    if self.__elognbl:
      if 'ELOGBOOK_DB' in os.environ:
        dbname = os.environ['ELOGBOOK_DB']
      else:
        dbname = 'elogbook'
      if not self.connectDB(dbname):
        log.error("Error opening database %s"
                  " disabling Elogbook integration" % dbname)
        log.error(str(self.__db.lastError().text()))
        self.__elognbl = 0
      else:
        log.debug("Database %s opened successfully" % dbname)

  def connectDB(self, dbname):
    """!\brief Connect to the database.

    \return True if connect was successful, False otherwise.
    """
    if 'ELOGBOOK_HOST' in os.environ:
      host = os.environ['ELOGBOOK_HOST']
    else:
      host = 'localhost'

    try:
      self.__db = MySQLdb.connect(host   = host,
                                  db     = dbname,
                                  user   = 'elogbook',
                                  passwd = 'elogbook',
                                  port   = 3306)
    except MySQLdb.OperationalError, e:
      msg  = "Error encountered when attempting to connect to MySQL server on host: '%s'\n" % host
      msg += "  Actual exception was: %s" % e
      log.exception(MySQLdb.OperationalError(msg))
      return False
    return True

  def disconnectDB(self):
    """!\brief Disconnect from the database
    """
    self.__db.commit()
    return self.__db.close()

  def addField(self, field, data, units=UNITS_STRING):
    """!\brief Add \a field to the run report.

    \param field Name of the run report item
    \param data  Value of the run report item
    \param units Run report item unit
    """
    if field == 'TimeStamp':
      self.__tstamp = data
    else:
      self.__repFields.append(field)
      self.__repData.append(data)
      self.__repUnits.append(units)

  def writeReportCSV(self):
    """!\brief Write the run report as a comma delimited file.

    """
    if not os.path.exists(self.__fileName):
      f = file(self.__fileName, 'w+')
      i = 0
      for header in self.__repFields:
        f.write(header)
        i = i + 1
        if i < len(self.__repFields):
          f.write(',')
      f.write('\n')

    f = file(self.__fileName, 'a+')
    i = 0
    for data in self.__repData:
      f.write('"'+data+'"')
      i = i + 1
      if i < len(self.__repData):
        f.write(',')
    f.write('\n')
    f.close()

  def writeReportXML(self):
    """!\brief Write the run report as an XML.

    """
    if not os.path.exists(self.__fileName):
      f = file(self.__fileName, 'w+')
      newReportFile = 1
    else:
      f = file(self.__fileName, 'a+')
      newReportFile = 0

    result = cStringIO.StringIO()
    gen = XMLGenerator(result)
    if newReportFile:
      gen.startDocument()
    if self.__tstamp is None:
      self.__tstamp = time.gmtime()
    self.__tstamp = str(self.__tstamp)
    gen.startElement("testReport", {'timestamp': self.__tstamp})
    i = 0
    for header in self.__repFields:
      gen.startElement(header,{})
      gen.characters(self.__repData[i])
      gen.endElement(header)
      i = i + 1
    gen.endElement("testReport")
    if newReportFile:
      gen.endDocument()
    f.write(result.getvalue()+'\n')
    f.close()
    if self.__elognbl:
      if self.writeReportSQL() == 0:
        self.addShift(self.__tstamp)
      self.disconnectDB()

  def writeReportSQL(self):
    """!\brief Write the run report to a relational database.

    """
    fieldMap = {'RunId': 'RunID',
                'OperatorId': 'OperatorID'
               }

    dateFields = ['TimeStamp', 'StartTime', 'EndTime']

    self.__repFields.append('TimeStamp')
    self.__repData.append(self.__tstamp)
    table = "elogreport"
    recInfo = self.recordInfo(table)
    fields = [f.name for f in recInfo]
    cur = self.__db.cursor()
    additional = ""
    sqlFields = ()
    sqlValues = ()
    for i in range(len(self.__repFields)):
      field = self.__repFields[i]
      if field in fieldMap:
        field = fieldMap[field]
      if field in dateFields:
        t = eval(self.__repData[i])
        value = "%04d-%02d-%02d %02d:%02d:%02d" % (t[0], t[1], t[2], t[3], t[4], t[5])
      else:
        value = self.__repData[i]
      if field in fields:
        sqlFields += (field,)
        sqlValues += (value,)
      else:
        # Tue Mar 15 14:05:15 2005: RiC debug log commented out.
        #log.debug("Can not find field %s in table %s. Adding it to the 'Additional Fields' column."
        #         % (field, table))
        additional = additional + field + "???" + value + "!!!"
    if additional != "":
      sqlFields += ("AdditionFields",)
      sqlValues += (additional,)

    sqlFields = ", ".join([item for item in sqlFields])
    insertStr = "insert into %s(%s) values %s" % (table, sqlFields, str(sqlValues))
    if cur.execute(insertStr) != 1:
      log.error("Could not insert elogreport record, dumping field data:")
      for i in range(len(self.__repFields)):
        log.error("\t%s\t\t\t%s" % (self.__repFields[i], self.__repData[i]))
      log.error("SQL Error text: %s", str(cur.lastError().text()))
      result = -1
    else:
      result = 0
      self.__db.commit()
    cur.close()
    return result

  def recordInfo(self, table):
    recordInfo = ()
    c = self.__db.cursor()
    c.execute("show columns from " + table)
    while (True):
      row = c.fetchone()
      if row is None: break
      recordInfo += (FieldInfo(row),)
    c.close()
    return recordInfo

  def readReportXML(self):
    """!\brief Read and parse the XML run report.

    """
    # 3 handler functions
    def start_element(name, attrs):
      #print 'Start element:', str(name), attrs
      self.__data = ''
      if str(name) == 'testReport':
        self.__tstamp = str(attrs['timestamp'])
        self.__repFields.append('TimeStamp')
        self.__repData.append(self.__tstamp)
        self.__repUnits.append(UNITS_STRING)
      else:
        self.__repFields.append(str(name))
    def end_element(name):
      #print 'End element:', str(name)
      name = str(name)
      if name != 'testReport':
        self.__repData.append(self.__data)
        if name in UNIT_MAP:
          self.__repUnits.append(UNIT_MAP[name])
        else:
          self.__repUnits.append(UNITS_STRING)
    def char_data(data):
      #print 'Character data:', str(data)
      if self.__data is not None:
        self.__data += str(data)
    def external_entity_ref_handler(context,base,systemId,publicId):
      subParser = p.ExternalEntityParserCreate(context)
      subParser.StartElementHandler = start_element
      subParser.EndElementHandler = end_element
      subParser.CharacterDataHandler = char_data
      subParser.ParseFile(file(systemId))
      return 1
    self.__data = None
    p = ParserCreate()
    p.ExternalEntityRefHandler = external_entity_ref_handler
    p.Parse("""<!DOCTYPE report [<!ENTITY reportfile SYSTEM \""""+self.__fileName+"""\">
            ]><report>&reportfile;</report>""")

  def getReportFields(self):
    """!\brief Retrieve the run report item names.

    \return A list containing run report item names.
    """
    return self.__repFields

  def getReportData(self):
    """!\brief Retrieve the run report item values.

    \return A list containing run report item values.
    """
    return self.__repData

  def getReportUnits(self):
    """!\brief Retrieve the run report item units.

    \return A list containing run report item units.
    """
    return self.__repUnits

  def getTimestamp(self):
    """!\brief Retrieve the timestamp of the run.

    \return Timestamp of the run.
    """
    return self.__tstamp

  def addShift(self,tstamp):
    """!\brief Add a shift summary to the database if it doesn't already exist.

    This method finds the shift id that the run belongs to based on \a tstamp
    and creates a shift summary if it doesn't already exist. This was done
    because the web e-logbook does not display the shift if it doesn't contain
    a shift summary even if it contains runs that belong to that shift.

    \param tstamp Timestamp of the run.

    \return True if the shift is created, None otherwise.
    """
    (shiftID, startLocal, endLocal, type) = self.getShiftInfo(eval(tstamp))
    timeStamp = self.convertTimeTuple(time.gmtime())
    # If there are no records, create a new one
    cur = self.__db.cursor()
    table = "elogshiftsummary"
    selectStr = "select ShiftID from %s where ShiftID = '%s'" % (table, shiftID)
    cur.execute(selectStr)
    row = cur.fetchone()
    if row is None:
      sqlFields = ("ShiftID", "ShiftType", "StartTime", "EndTime", "LastUpdate")
      sqlValues = (shiftID, type, startLocal, endLocal, timeStamp)
      sqlFields = ", ".join([item for item in sqlFields])
      insertStr = "insert into %s(%s) values %s" % (table, sqlFields, str(sqlValues))
      if cur.execute(insertStr) == 1:
        log.debug("rcReportGen: Added new shift id: %s, type: %s, startLocal: %s, endLocal: %s, timestamp: %s for run timestamp: %s"
                  % (shiftID, type, startLocal, endLocal, timeStamp, tstamp))
        self.__db.commit()
        result = True
      else:
        log.warn("rcReportGen: Unable to insert elogshiftsummary record")
        result = False
      cur.close()
      return result
    else:
      #print "Shift for run: ", tstamp, " already exists in the database."
      pass


  def getShiftInfo(self,t):
    """!\brief Find the shift information based on the timestamp \a t.

    \param t GMT timestamp of the run

    \return A tuple containing the shift id, start time, end time and type.
    """
    #print "Timestamp=", t
    timeStampLocal = self.obtainLocalTime(t)
    secsTimeStamp = calendar.timegm(t)
    timeStamp = timeStampLocal.split('T')
    date = timeStamp[0]
    #Obtain End Date (Local)
    startDate = calendar.timegm(t)
    endQDate = startDate + 3600 * 24
    endDate = self.convertTimeTuple(time.localtime(endQDate)).split('T')[0]
    # Date tag for ShiftID
    dateList = date.split('-')
    dateTag = dateList[0] + dateList[1] + dateList[2]
    # If run timeStamp is within shift obtain ShiftID
    for shift in SHIFT_DICT:
      (startT, endT, type) = SHIFT_DICT[shift]
      startLocal = date + startT
      if shift == 2:
        endLocal = endDate + endT
      else:
        endLocal = date + endT
      secsStartTime = calendar.timegm(self.obtainGMTime(startLocal))
      secsEndTime   = calendar.timegm(self.obtainGMTime(endLocal))
      # The Core Code:
      # secsTo Returns the number of seconds from this datetime1 to (datetime2)
      # which is negative if dt is earlier than this datetime).
      # Example: datetime1.secsTo(datetime2) > 0 if datetime2 is later than datetime1
      secsToStartTime = secsTimeStamp - secsStartTime
      secsToEndTime   = secsTimeStamp - secsEndTime
      #print "Shift: ", shift, "Start: ", startLocal, " End: ", endLocal
      #print "Shift: ", shift, "Secs To Start: ", secsToStartTime, "Secs To End: ", secsToEndTime
      if secsToStartTime >= 0 and secsToEndTime <= 0:
        shiftID    = dateTag + str(shift)
        return (shiftID, startLocal, endLocal, type)

  def obtainLocalTime(self, dateTime):
    """!\brief Convert a GMT date/time to local date/time.

    \param dateTime GMT date/time tuple

    \return Local date/time in SQL format
    """
    localTimeTuple = time.localtime(calendar.timegm(dateTime))
    localTime = self.convertTimeTuple(localTimeTuple)
    return localTime

  def obtainGMTime(self, dateTime):
    """!\brief Convert a local date/time to a GMT date/time.

    \param dateTime Local date/time value in SQL format

    \return GMT date/time tuple
    """
    splitTime = dateTime.split('T')
    dateTime = splitTime[0] + splitTime[1]
    timeStamp = time.strptime(dateTime, "%Y-%m-%d%H:%M:%S")
    gmTimeTuple = time.gmtime(time.mktime(timeStamp))
    return gmTimeTuple

  def convertTimeTuple(self, timeTuple):
    """!\brief Convert a time tuple to an SQL format.

    \brief A date/time tuple.
    """
    timeStamp =  "%04d" % timeTuple[0] + "-" + \
                 "%02d" % timeTuple[1] + "-" + \
                 "%02d" % timeTuple[2] + "T" + \
                 "%02d" % timeTuple[3] + ":" + \
                 "%02d" % timeTuple[4] + ":" + \
                 "%02d" % timeTuple[5]
    return timeStamp

class FieldInfo(object):
  def __init__(self, row):
    self.name    = row[0]
    self.type    = row[1]
    self.null    = row[2]
    self.key     = row[3]
    self.default = row[4]
    self.extra   = row[5]

---