gVersions.py

Go to the documentation of this file.

#!/usr/local/bin/python
#
#                               Copyright 2005
#                                     by
#                        The Board of Trustees of the
#                     Leland Stanford Junior University.
#                            All rights reserved.
#

__facility__ = "Online"
__abstract__ = "Version retrieval utility"
__author__   = "Selim Tuvi <stuvi@slac.stanford.edu> SLAC - GLAST I&T/Online"
__date__     = "2005/07/23 00:08:27"
__updated__  = "$Date: 2006/03/22 01:34:28 $"
__version__  = "$Revision: 1.19 $"
__release__  = "$Name: HEAD $"
__credits__  = "SLAC"

import LICOS.copyright_SLAC

import sha
import marshal
import inspect
import os
import sys
import glob
import logging as log
import compiler

from LICOS.lib.LATconstants import *

class Gversions(object):
  """!\brief Version retrieval utility class.

  Contents:

    Provides accessor methods to retrieve version information from
    both the software and hardware.
    Displays version info from loaded libraries.
    Used for troubleshooting problems.

  Overview:

    Each driver on VxWorks side and each module on the client side
    provides its version number through functions. When the function
    cmd.getVersions() is called the command server packetizes this
    information and sends it over the socket to the command client
    which gets decoded into a tuple. The getModuleVersions() function
    converts this tuple to a dictionary and adds the python module
    versions to it.

  Usage:

    To retrieve version information one can write:

  \code
      python versions.py

      or

      import versions
      versions.Gversions.getModuleVersions()
  \endcode
  """

  def getModuleVersions(lcm=None):
    """!\brief Static class method which returns a dictionary of modules
       with their versions.

    \param lcm LCM instance

    \return Dictionary of modules with their versions
    """
    ignoreList =  [ 'scipy.gplt','ROOT'
                    #'__main__','_xmlplus','cPickle','pickle','sre','urllib',
                    #'xml','_xmlplus','xml.parsers.expat', 'xml.parsers.pyexpat',
                    #'pyexpat'
                  ]
    versions = {}
    if lcm is not None:
      # TODO: For now just collect SIU module info -ST
      if not lcm.modListPopulated():
        log.info("Retrieving FSW module versions from unit %s..."
                % FSW_UNIT_NAMES[FSW_UNIT_SIU])
      fswModList = lcm.cmxAsBuilt(FSW_UNIT_SIU)
      if fswModList is not None:
        unitModList = {}
        for (package, constituent, version) in fswModList:
          unitModList['%s[%s]' % (package.strip(), constituent.strip())] = version.strip()
        versions[FSW_UNIT_NAMES[FSW_UNIT_SIU]] = unitModList
      if not lcm.modListPopulated():
        log.info("FSW module dump complete.")
    mstrList = sys.modules.keys()
    mstrList.sort()
    for mStr in mstrList:
      if mStr in ignoreList: continue
      m = sys.modules[mStr]
      if hasattr(m, '__version__'):
        v = Gversions.extractVersion(str(getattr(m, '__version__')))
        if v.strip() != '' or Gversions.isModuleInReleases(m):
          versions[mStr] = v
      elif Gversions.isModuleInReleases(m):
        versions[mStr] = ''

    versions['Python'] = str(hex(sys.hexversion))
    try:
      import qt
      versions['Qt']     = qt.QT_VERSION_STR
    except: pass
    try:
      import sip
      if 'SIP_VERSION_STR' in dir(sip):
        versions['SIP'] = sip.SIP_VERSION_STR
      elif 'SIP_VERSION' in dir(sip):
        versions['SIP'] = hex(sip.SIP_VERSION)
    except: pass
    try:
      if 'PYQT_VERSION_STR' in dir(qt):
        versions['PyQt'] = qt.PYQT_VERSION_STR
      else:
        versions['PyQt']   = qt.PYQT_VERSION
    except: pass
    return versions
  getModuleVersions = staticmethod(getModuleVersions)

  def isModuleInReleases(m, rootDirs=None):
    """!\brief Method to determine if a loaded module is in the
    specified list of root directories.

    \param m Module object
    \param rootDirs List of root directories relative to $ONLINE_ROOT.

    \return Boolean that determines whether the module originated from the list of
            root directories.
    """
    if rootDirs is None:
      rootDirs = ROOT_DIRS
    if hasattr(m, '__file__'):
      srcFile = inspect.getsourcefile(m)
      if srcFile is not None:
        srcFile = os.path.normpath(os.path.normcase(srcFile))
        for rootDir in rootDirs:
          rootPath = os.path.normpath(os.path.normcase(os.path.join(ONLINE_ROOT, rootDir)))
          if srcFile.startswith(rootPath + os.sep):
            return True
    return False
  isModuleInReleases = staticmethod(isModuleInReleases)


  def extractVersion(verStr):
    """!\brief Extract the version string from the CVS keyword string.

    if that's what it appears to be.  If it doesn't, just
    return the input.

    \param verStr CVS keyword string

    \return Version string
    """
    colon  = verStr.find(':')
    if colon != -1:
      dollar = verStr.find('$', colon)
      if dollar != -1:
        return verStr[colon+2:dollar-1]
    return verStr
  extractVersion = staticmethod(extractVersion)

  def calcDigest(filePath):
    """!\brief Calculate the digest for a given file.

    \param filePath Full path of the file

    \return Digest data.
    """
    digest = sha.new()
    f = file(filePath, 'rb')
    data = f.read(65536)
    while len(data) != 0:
      digest.update(data)
      data = f.read(65536)
    f.close()
    #return digest.hexdigest()
    return digest.digest()
  calcDigest = staticmethod(calcDigest)


  def getModuleDigests():
    """!\brief Retrieve digests of all loaded modules.

    All modules which do not have __version__ or __file__
    attributes are ignored.

    \return A dictionary of digest data keyed by module name
    """
    ignoreList = ['scipy.gplt','ROOT']
    digests = {}
    mstrList = sys.modules.keys()
    mstrList.sort()
    for mStr in mstrList:
      if mStr in ignoreList: continue
      m = sys.modules[mStr]
      if hasattr(m, '__version__') and hasattr(m, '__file__'):
        srcFile = inspect.getsourcefile(m)
        digest_value = Gversions.calcDigest(srcFile)
        if digest_value is not None:
          digests[mStr] = digest_value
    return digests
  getModuleDigests = staticmethod(getModuleDigests)

  def getReleaseFromDigest(digestDataFile=None, rootDir='LICOS'):
    """!\brief Given a root directory extract the release tag.

    The root directory defaults to LICOS. This method can be used
    to extract release tag info from sub-system root directories where
    the digestData file is stored.

    \param digestDataFile The path pointing to the digestData file.
                          If none $ONLINE_ROOT/rootDir/digestData is assumed
    \param rootDir        Root directory relative to $ONLINE_ROOT

    \return Release tag
    """
    rootPath = os.path.join(ONLINE_ROOT, rootDir)
    digestDataFile = os.path.join(rootPath, 'digestData')
    if not os.path.exists(digestDataFile):
      raise RuntimeError, "Can not find the digest database file at the indicated location: %s" % rootPath
    f = file(digestDataFile,'rb')
    try:
      dc = marshal.load(f)
    except ValueError:
      log.exception("")
      Gversions.logBadMarshalDataMessage()
      return
    exec(dc)
    return _digestData._release
  getReleaseFromDigest = staticmethod(getReleaseFromDigest)

  def logBadMarshalDataMessage():
    """!\brief Message that is logged when an exception occurs during the
    marshalling of the digest data file.

    """
    log.error("If the above exception is a 'bad marshal data' error,")
    log.error("it is likely that you need to update one or more of the")
    log.error("digestData files pointed to by the root directory")
    log.error("variables. To create the digestData file, enter the")
    log.error("following command for each of the root directories that you have:")
    log.error(os.path.expandvars(
     "$ONLINE_ROOT\\LICOS\\setup\\createDigestData root_dir %s XXX_00_00_00"
     ) % sys.prefix)
  logBadMarshalDataMessage = staticmethod(logBadMarshalDataMessage)

  def verifyModuleDigests(digestDataFile=None, rootDir='LICOS', additionalFilesToVerify=[]):
    """!\brief Compare loaded modules and any additional files
    against the digest data.

    \param digestDataFile          The path pointing to the digestData file.
                                   If none $\a $ONLINE_ROOT/rootDir/digestData is assumed
    \param rootDir                 Root directory
    \param additionalFilesToVerify A list of file paths to check

    \return A tuple containing the release tag and a list of module names
            which failed the verification
    """
    ignoreList = ['scipy.gplt','ROOT']
    failedModules = []
    rootPath = os.path.join(ONLINE_ROOT, rootDir)
    if digestDataFile is None:
      digestDataFile = os.path.join(rootPath, 'digestData')
    if not os.path.exists(digestDataFile):
      raise RuntimeError, "Can not find the digest database file at the indicated location: %s" % rootPath
    f = file(digestDataFile,'rb')
    try:
      dc = marshal.load(f)
    except ValueError:
      log.exception("")
      Gversions.logBadMarshalDataMessage()
      return (None, [])
    exec(dc)
    mstrList = sys.modules.keys()
    mstrList.sort()
    for mStr in mstrList:
      if mStr in ignoreList: continue
      #raw_input("Entering %s}" % mStr)
      m = sys.modules[mStr]
      if hasattr(m, '__file__'):
        srcFile = inspect.getsourcefile(m)
        if srcFile is not None:
          srcFile = os.path.normpath(os.path.normcase(srcFile))
          if srcFile.startswith(os.path.normpath(os.path.normcase(rootPath)) + os.sep):
            found = False
            for (filename, digest) in _digestData._digests.items():
              if srcFile == os.path.normpath(os.path.normcase(filename)):
                found = True
                digest_value = Gversions.calcDigest(srcFile)
                if digest_value is None or digest != digest_value:
                  failedModules.append(mStr)
                break
            if not found:
              failedModules.append(mStr)
      #raw_input("Exiting %s}" % mStr)

    for fileSpec in additionalFilesToVerify:
      for filePath in glob.glob(fileSpec):
        normPath = os.path.normpath(os.path.normcase(filePath))
        if normPath.startswith(os.path.normpath(os.path.normcase(rootPath)) + os.sep):
          for (filename, digest) in _digestData._digests.items():
            if normPath == os.path.normpath(os.path.normcase(filename)):
              digest_value = Gversions.calcDigest(normPath)
              if digest_value is None or digest != digest_value:
                failedModules.append(os.path.split(filePath)[1])
              break

    return (_digestData._release, failedModules)
  verifyModuleDigests = staticmethod(verifyModuleDigests)

  def findVersionSymbol(filePath):
    """!\brief Find the __version__ symbol in the file.

    \param filePath Full path of the module source code

    \return Version number or None if not found.
    """
    found = False
    try:
      ast = compiler.parseFile(filePath)
      for node in ast.node.nodes:
        if node.__class__.__name__ == 'Assign':
          if sys.hexversion > 0x20305f0:
            if node.asList()[0].asList()[0] == '__version__':
              version = node.asList()[1].asList()[0]
            else:
              continue
          else:
            if node.asList()[0][0] == '__version__':
              version = node.asList()[1][0]
            else:
              continue
          if type(version) is str and version.startswith('$Revision:'):
            return Gversions.extractVersion(version)
          else:
            #print "*** %s: __version__ symbol found but does not begin with $Revision:" % filePath
            return None
    except:
      log.exception("Error in parsing file: %s" % filePath)

    if not found:
      #print "*** %s: Missing __version__" % filePath
      return None
  findVersionSymbol = staticmethod(findVersionSymbol)

  def verifyFileDigests(rootDir='LICOS', digestDataFile=None):
    """!\brief Verify the files under a root directory against the digest data file.

    \param rootDir                 Root directory
    \param digestDataFile          The path pointing to the digestData file.
                                   If none $ONLINE_ROOT/rootDir/digestData is assumed

    \return A tuple containing the release tag and a list of module names
            which failed the verification
    """
    skipVersionCheck = {'LICOS': ['sipconfig.py', 'pyqtconfig.py', 'testAppNoVersion.py']}
    failedFiles = []

    rootPath = os.path.join(ONLINE_ROOT, rootDir)
    if digestDataFile is None:
      digestDataFile = os.path.join(rootPath, 'digestData')
    if not os.path.exists(digestDataFile):
      raise RuntimeError, "Can not find the digest database file at the indicated location: %s" % rootPath
    f = file(digestDataFile,'rb')
    try:
      dc = marshal.load(f)
    except ValueError:
      log.exception("")
      Gversions.logBadMarshalDataMessage()
      return (None, [])
    exec(dc)
    for (filename, digest) in _digestData._digests.items():
      if os.path.exists(filename):
        fileDigest = Gversions.calcDigest(filename)
        if digest != fileDigest:
          failedFiles.append(filename)
        if os.path.splitext(filename)[1] == '.py':
          if rootDir in skipVersionCheck:
            if os.path.split(filename)[1] in skipVersionCheck[rootDir]:
              continue
          if Gversions.findVersionSymbol(filename) is None:
            failedFiles.append(filename)
      else:
        failedFiles.append(filename)
    return (_digestData._release, failedFiles)
  verifyFileDigests = staticmethod(verifyFileDigests)

  def collectDigests(path):
    """!\brief Given a path, collect digests from all files.

    \param path Root directory

    \return Dictionary of digests
    """
    def collectDigests2(digests, dir, files):
      """!\brief os.path.walk handler method.

      \param digests Dictionary of digests keyed by the full path name
      \param dir     Current directory being processed
      \param files   List of files in the current directory
      """
      ignoredDirs = ['CVS']
      validExtensions = ['.py',  # Python source
                         '.exe', # Windows executables
                         '.xml', # Schema and other XML files
                         '.vx',  # VxWorks scripts
                         '.o',   # C/C++ Object modules
                         '.cfg', # Configuration files
                         '.f',   # LHK configuration file binaries
                         '.bat', # Windows batch files
                         '.csh', # Shell scripts
                         '.sh',  # Shell scripts
                         '.txt', # Configuration files
                        ]
      l = len(files)
      for i in range(l-1,-1,-1):
        if files[i] in ignoredDirs:
          del files[i]
        else:
          fullFilename = os.path.normpath(os.path.join(dir, files[i]))
          if not os.path.isdir(fullFilename) and not os.path.islink(fullFilename):
            if os.path.splitext(files[i])[1].lower() in validExtensions:
              digest = Gversions.calcDigest(fullFilename)
              if digest is not None:
                if digests.has_key(fullFilename):
                  log.error("DUPLICATE ENTRY DETECTED: %s" % fullFilename)
                else:
                  digests[fullFilename] = digest
              else:
                log.error("NO DIGEST FOR %s" % files[i])
    digests = {}
    path = os.path.normpath(os.path.abspath(path))
    if os.path.isdir(path):
      os.path.walk(path, collectDigests2, digests)
    #~ for (file, digest) in digests.items():
      #~ print file #, repr(digest)
    #~ x = digests.keys()
    #~ x.sort()
    #~ for f in x:
      #~ print f
      return digests
    else:
      return None
  collectDigests = staticmethod(collectDigests)

  def printVersions():
    """!\brief Print module versions to the console.

    """
    print "\nCurrent Release: %s\n" % Gversions.getReleaseFromDigest()
    print "%-45s %30.30s" % ("Module", "Version")
    print "%-45s %30.30s" % ("--------", "-------")
    versions = Gversions.getModuleVersions()
    modList = versions.keys()
    modList.sort()
    for module in modList:
      print "%-45s %30.30s" % (module, versions[module])
  printVersions = staticmethod(printVersions)

if __name__ == '__main__':
  # Imports here are specified so that the versions for
  # all loaded modules can be retrieved. Add any additional
  # module imports here if they are to be reported in the
  # output of versions.bat
  # In order for all module versions to be reported, the __all__
  # variable in __init__.py of each package needs to be fully
  # populated.
  from LICOS.core.archive import *
  from LICOS.core.dispatch import *
  from LICOS.core.distributor import *
  from LICOS.core.export import *
  from LICOS.core.thread import *
  from LICOS.lib import *
  from LICOS.lib.cmdTlmDb import *
  from LICOS.lib.currValTable import *
  from LICOS.lib.scriptEngine import *
  from LICOS.lib.testReport import *
  from LICOS.science import *
  from LICOS.scriptEngine import *
  from LICOS.start import *
  from LICOS.telemetry import *
  from LICOS.tests.lib import *
  from LICOS.tests.scriptEngine import *
  from LICOS.tests.standalone import *
  from LICOS.tools.alarm import *
  from LICOS.tools.EnvDataXfer import *
  from LICOS.tools.LATc import *
  from LICOS.tools.logger import *
  from LICOS.tools.monitor import *
  from LICOS.tools.proc import *
  from LICOS.tools.proxy import *
  from LICOS.tools.teleCmdGUI import *
  from LICOS.tools.utcClock import *
  from LICOS.util import *

  import LDF
  import qt
  import Ft.Xml
  Gversions.printVersions()

---