FILE.py

Go to the documentation of this file.

#!/usr/local/bin/python
#
#                               Copyright 2005
#                                     by
#                        The Board of Trustees of the
#                     Leland Stanford Junior University.
#                            All rights reserved.
#
#
# References:  [FSW_GUIDE]    LAT-TD-07665-01    LAT FSW User Guide
#
#



__facility__ = "Online"
__abstract__ = "FILE related mnemonics"
__author__   = "Selim Tuvi <stuvi@slac.stanford.edu> SLAC - GLAST LAT I&T/Online"
__date__     = "2005/07/23 00:08:27"
__updated__  = "$Date: 2006/04/28 01:29:35 $"
__version__  = "$Revision: 1.18 $"
__release__  = "$Name: HEAD $"
__credits__  = "SLAC"

import LICOS.copyright_SLAC

import array
import logging as log
import os
import tempfile
import time

from cmdTlmDb import CmdPkg

from LICOS.lib.LATconstants            import *
from LICOS.lib.cmdTlmDb.TelemetryEvent import TelemetryEvent
from LICOS.lib.LATfunctions            import utcfromtimestamp

# Define our tempfile paths for consistency
if os.name == 'posix':
  FILETOOL_TMP_PATH = "/tmp"
else:   # gotta be windows...
  FILETOOL_TMP_PATH = os.path.normpath(os.path.join('C:/', 'TEMP'))

# executables to add FMX headers and compression to binary uploads
if os.name == 'posix':
  FILETOOL_HDR_ALLOW = True
  FSW_ADD_HEADER_CMD = 'file_hdr_prefix'     # flight/SVC/symlnk/linux-gcc/exe/zcompress
  FSW_COMPRESS_CMD   = 'zcompress'           # flight/SVC/symlnk/linux-gcc/exe/file_hdr_prefix
else:
  FILETOOL_HDR_ALLOW = False                 # do NOT allow addition of fsw headers from Windows
  FSW_ADD_HEADER_CMD = None
  FSW_COMPRESS_CMD   = None

# hook for keeping track of an arbitrary file number (last part of a FswFileID)
FILETOOL_ARBITRARY_FILENUMBER  = 0
# hook for keeping track of an arbitrary FMX key  (not to be used with real FMX)
FILETOOL_ARBITRARY_FMXID       = 0x20000000L

# packet, mnemonic and 'source' for primary boot FILESTAT state
FILE_PBC_FILESTAT = { FSW_UNIT_SIU : [ "LBTHKP",      "LPBCFILESTAT",  "" ],
                      FSW_UNIT_EPU0: [ "LBTEPU0HKP",  "LPBC0FILESTAT", "diag" ],
                      FSW_UNIT_EPU1: [ "LBTEPU1HKP",  "LPBC1FILESTAT", "diag" ],
                      FSW_UNIT_EPU2: [ "LBTEPU2HKP",  "LPBC2FILESTAT", "diag" ],
                      }


class FILE(CmdPkg):
  """!\brief FILE telecommand package class

  Provides access to FILE telecommands.
  """
  def __init__(self, db):
    """!\brief FILE constructor.

    \param db  Wrapper database object for the generated module
              containing command and telemetry database classes.
    """
    CmdPkg.__init__(self, db)
    self.__primaryBoot=False



  def downloadFile(self, unit, fileID):
    """!\brief Download a file from the LAT

    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object
    \return An FswFile object containing the data uploaded
    """
    class FileDump(object):
      def __init__(self, master):
        self.__master = master
        self.__cmdDb = master.getCmdDb()
        self.__fragments = {}
        self.__dumpSizes = {}
        self.__expectedID = None
        self.__dumpCTDBtelem = TelemetryEvent(self.__cmdDb.getVsc().getDiagHandler(),
                                              [self.__cmdDb.getVsc().getTlmDb().getApidFromName('LLFSDUMPCTDB')],
                                              self.dumpCTDB)
      def dumpCTDB(self, telemetry):
        # File chunks may be sent down out of order.
        # Cache each dump payload in a dictionary keyed on the file offset.
        try:
          pkt = self.__cmdDb.getVsc().getTlmDb().decodeTelemetry(telemetry)
          offset   = pkt.get_payload("LFSFDMPOFFSET")
          dumpSize = pkt.get_payload("LFSFDMPSIZE")

          # Check if this is part of the expected file
          dumpDev  = pkt.get_payload("LFSFDMPDEV")
          dumpDir  = pkt.get_payload("LFSFDMPDIR")
          dumpFile = pkt.get_payload("LFSFDMPFILE")

          if dumpDev  != self.__expectedID.device() or \
                 dumpDir  != self.__expectedID.directory() or \
                 dumpFile != self.__expectedID.fileNumber():
            msg = "CTDB dump resulted in unexpected file: %s, expected %s" % \
                  ( str(FswFileID(dumpDev, dumpDir, dumpFile)), self.__expectedID )
            raise RuntimeError(msg)

          dataMnems = ['LFSFDMPDATA%d' % i for i in xrange(390)]
          self.__fragments[offset] = ''.join(map(chr, map(pkt.get_payload, dataMnems)))
          self.__dumpSizes[offset] = dumpSize

        except Exception, e:
          print e
          raise

      def dump(self, unit, fileID):
        self.__expectedID = fileID
        self.__dumpCTDBtelem.enable()
        LFS = self.__cmdDb.LFS

        # need to know how much we expect
        expSize = None
        dir = self.__cmdDb.FILE.dumpDirectory(unit,fileID)
        files = dir.files()
        for f in files:
          if f.id.id() == fileID.id():
            expSize = f.size

        LFS.LFSFILEDUMPC( LFSFILEID=fileID.id(),
                          LFSUNIT=unit,
                          LFSPAD16=0 )

        # wait until we get the expected size
        timeWait = 0
        oldSize = 0
        downSize = sum(self.__dumpSizes.values())
        while downSize < expSize and timeWait < SHORT_TIMEOUT*5:
          # reset if we're still getting data
          if downSize > oldSize:
            timeWait = 0
          time.sleep(0.1)
          timeWait += 0.1
          downSize = sum(self.__dumpSizes.values())

        # reassemble
        offsets = self.__fragments.keys()
        offsets.sort()
        binary = ""
        for o in offsets:
          binary += self.__fragments[o][:self.__dumpSizes[o]]

        status = self.__master.checkLastCommandStatus()
        self.__dumpCTDBtelem.disable()

        return FswFileFromData(fileID, binary, uploaded=True)

    fd = FileDump(self.getCmdDb())

    fswFile = fd.dump(self.__LFSUNIT(unit), fileID)

    return fswFile


  def downloadFileToSSR(self, unit, fileID):
    """!\brief Download a file from the LAT

    \param app         A testAppBase object  (your user application...)
    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object
    \return status
    """
    self.getCmdDb().LFS.LFSFILEDUMPS( LFSFILEID=fileID.id(),
                                      LFSunit=self.__LFSUNIT(unit),
                                      LFSPAD16=0 )
    status = self.checkLastCommandStatus()
    return status

  def dumpDirectory(self, unit, directoryID, root=False):
    """!\brief Download a directory listing from the LAT

    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object with the correct directory set.
    \return An FswFile object containing the data uploaded
    """
    class DirDump(object):
      def __init__(self, master, root):
        self.__master = master
        self.__cmdDb  = master.getCmdDb()
        self.__root   = root
        self.__rep    = None
        self.__lastTime = time.time()
        self.__dumpSizes = {}
        if self.__root:
          wantedPacket = "LLFSROOTLIST"
        else:
          wantedPacket = "LLFSDIRLIST"


        self.__dumpDirTelem = TelemetryEvent(self.__cmdDb.getVsc().getDiagHandler(),
                                             [self.__cmdDb.getVsc().getTlmDb().getApidFromName(wantedPacket)],
                                             self.getPackets)
      def getPackets(self, telemetry):
        self.__lastTime = time.time()
        try:
          pkt = self.__cmdDb.getVsc().getTlmDb().decodeTelemetry(telemetry)
          if self.__root:
            prefix = 'LFSRDMPF'
          else:
            prefix = 'LFSDDMP'

          device    = pkt.get_payload( prefix + "DEV" )
          directory = pkt.get_payload( prefix + "DIR" )
          fileNum   = pkt.get_payload( prefix + "FILE" )
          # Deal with LCAT inconsistencies
          if self.__root:
            prefix = prefix[:-1]

          ro        = pkt.get_payload( prefix + "ROFLG")
          size      = pkt.get_payload( prefix + "SIZE")
          isdir     = pkt.get_payload( prefix + "DIRFLG")
          t         = pkt.get_payload( prefix + "TIME")
          u         = pkt.get_payload( prefix + "UNIT")
          blocks    = 0  # pkt.get_payload( prefix + "BLOCKS")  # bad?  why?


          id = FswFileID(device, directory, fileNum)
          item = FswDirectoryItem(id,
                                  ro, size, isdir, t, u, blocks)
          
          self.__rep.addItem(item)
          #print "getPackets:", str(id), str(self.__rep.id())

          # for real files in LLFSDIRLIST, we get the header!
          if not self.__root:
            dataMnems = [prefix+ 'FHDR%d' %i for i in xrange(32)]
            hdr = ''.join(map(chr, map(pkt.get_payload, dataMnems)))
            item.header = hdr
          
        except Exception, e:
          print e
          raise

      def dump(self, unit, dirID):
        self.__rep = FswDirectory( dirID=dirID )
        self.__dumpDirTelem.enable()
        self.__cmdDb.LFS.LFSDIRDUMP(LFSFILEID=dirID.id(),
                                    LFSUNIT=unit,
                                    LFSPAD16=0)
        # REVISIT:  need to wait for the entire directory to come down
        while True:
          if time.time() - self.__lastTime > 2.0:
            break
          else:
            time.sleep(0.1)
        status = self.__master.checkLastCommandStatus()
        
        time.sleep(1)

        self.__dumpDirTelem.disable()

        return self.__rep

    dd = DirDump(self.getCmdDb(), root)
    dirRep = dd.dump(self.__LFSUNIT(unit), directoryID)
    return dirRep

  def createDirectory(self, unit, directoryID):
    """!\brief create a directory on a LAT file system

    \param unit        LAT unit id
    \param directoryID An FswFileID object with the correct directory set.
    \return status     True if it worked, False if not.
    """
    if directoryID.directory() not in FSW_DIR_RANGE:
      log.error("%s attempted to create directory %d which is out of the allowed range" %\
                ("FILE.createDirectory", directoryID.directory()) )
      return False
    
    self.getCmdDb().LFS.LFSDIRCREATE(LFSUNIT=self.__LFSUNIT(unit),
                                     LFSFILEID=directoryID.id(),
                                     LFSPAD16=0)
    status = self.checkLastCommandStatus()
    return status

  def deleteDirectory(self, unit, directoryID, force=False):
    """!\brief Delete a directory on a LAT filesystem

    \param unit         LAT unit id to upload to
    \param directoryID  An FswFileID object with the correct directory set.
    \return status      True if success, False if failure
    """
    if directoryID.directory() not in FSW_DIR_RANGE:
      log.error("%s attempted to delete directory %d which is out of the allowed range" %\
                ("FILE.deleteDirectory", directoryID.directory()) )
      return False
    
    status = True
    if force:
      # Remove all files in this directory
      dirRep = self.dumpDirectory(unit, directoryID)
      for f in dirRep.files():
        status = self.deleteFile(unit, f.id)
        if not status:
          break
    if status:
      # log.debug("deleting directory %s" % directoryID)
      self.getCmdDb().LFS.LFSDIRDELETE(LFSUNIT=self.__LFSUNIT(unit),
                                       LFSFILEID=directoryID.id(),
                                       LFSPAD16=0)
      status = self.checkLastCommandStatus()

    return status


  def uploadFileFromDisk(self, fileName, unit, fileID):
    """!\brief Upload a disk resident file with an FMX header

    \param app         A testAppBase object  (your user application...)
    \param fileName    Fully qualified file name
    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object
    \return An FswFile object containing the data uploaded
    """
    uplFile = None
    try:
      uplFile = FswFileFromDisk(fileID, fileName)
    except:
      raise
    else:
      self.uploadFile(uplFile, unit)
    return uplFile

  def uploadFileFromData(self, fileData, unit, fileID):
    """!\brief Upload a memory resident file with an FMX header (Typically received from MOOT)

    \param app         A testAppBase object  (your user application...)
    \param fileData    Binary representation of an FMX file
    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object
    \return An FswFile object containing the data uploaded
    """
    uplFile = None
    try:
      uplFile = FswFileFromData(fileID, fileData)
    except:
      raise
    else:
      self.uploadFile(uplFile, unit)
    return uplFile

  def cancelUpload(self, unit):
    """!\brief cancel any in process uploads

    \param app         A testAppBase object  (your user application...)
    \param fswFile     FswFile object
    \param unit        LAT unit id to upload to
    \return status
    """
    self.LFILUPLCANCEL(LFILEUNIT=self.__LFSUNIT(unit))
    status = self.checkLastCommandStatus()
    return status

  def uploadFile(self, fswFile, unit):
    """!\brief Upload a memory resident fsw file object

    \param app         A testAppBase object  (your user application...)
    \param fswFile     FswFile object
    \param unit        LAT unit id to upload to
    \return status
    """

    # TODO:  JHP-  Check for uploaded?
    # if fswFile.uploaded():
    #   log.info("Fmx File 0x%x:%d has already been uploaded" % ( fswFile.fileId(), fswFile.fileId() )
    #   return True

    fileID = fswFile.fileID()
    uplStr = fswFile.binary()
    status = True

    # cancel any pending transfers
    # Difficulty with EPUs: cannot upload directly.
    #   Must upload to SIU and copy across...
    #   Thus, must cancel upload on SIU too.
    epuUnits = ( FSW_UNIT_EPU0, FSW_UNIT_EPU1, FSW_UNIT_EPU2, FSW_UNIT_EPU3 )
    status = self.cancelUpload(unit)
    if unit in epuUnits:
      # We're in primary boot in EPU, but not in SIU.  
      cachePB = self.__primaryBoot
      self.__primaryBoot = False
      status &= self.cancelUpload(FSW_UNIT_SIU)
      self.__primaryBoot = cachePB
    if not status:
      print status
      raise RuntimeError("Recieved bad status from cancelUpload.  Check previous warning.")

    # start file transfer command
    # fiddly code: 1: FSW uploads packets in multiples of 48 bytes.
    #              2: filuplstart must tell system *exactly* how much data to expect
    #              3: Round up to next 48 byte boundary
    dumpSize = len(fswFile.binary())
    d,m = divmod(dumpSize, 48)
    if m != 0:
      d += 1
    self.LFILUPLSTART(LFILESIZE=d*48)
    status = self.checkLastCommandStatus()
    if not status:
      raise RuntimeError("Recieved bad status from upload start.  Check previous warning.")

    # loop over 48 byte chunks
    nDataBytes = 48
    chunk = 0
    offset = 0
    lastpacket = dumpSize // nDataBytes - 1
    while True:
      offset = chunk*nDataBytes
      if offset >= dumpSize:
        break
      if lastpacket > 0:
        if chunk == 0:
          seqFlags = 0x1 # first packet
        elif chunk == lastpacket:
          seqFlags = 0x2 # last packet
        else:
          seqFlags = 0x0 # middle packet
      else:
        seqFlags = 0x3     # not a sequence
      self.getCmdObj('LFILUPLDATA').seqFlags = seqFlags
      self.__uploadChunk(uplStr[offset:offset+nDataBytes], offset, nDataBytes)
      chunk += 1
    
    # It takes a while for an upload to commit, due to 1553 speed.
    # set max timeout to number of upload chunks * 1/8 second plus a fudge factor
    uploadTimeout = max(STANDARD_TIMEOUT, chunk*(1./8.)+STANDARD_TIMEOUT)

    # Difficulty with EPUs: cannot upload directly.
    #   Must commit to SIU and copy across sith LFILUPLEPU
    if unit in epuUnits:
      log.debug("Uploading to %s.  Sending to SIU RAM first" % FSW_UNIT_NAMES[unit])
      
      tempSiuID = FswFileID( device = FSW_DEVICE_RAM,
                         directory = 15,
                         fileNumber = FswFileID.MASK_FILENUM)
      # We're in primary boot in EPU, but not in SIU.  
      cachePB = self.__primaryBoot
      self.__primaryBoot = False
      # commit to SIU
      self.LFILUPLCOMMIT( LFILEUNIT=self.__LFSUNIT(FSW_UNIT_SIU),
                          LFILEFLAGS=0,
                          LFILEID=tempSiuID.id())
      status = self.checkLastCommandStatus()
      # do the copy
      self.LFILUPLEPU(LFILEUNIT=self.__LFSUNIT(unit),
                      LFILEPAD16 = 0,
                      LFILEID = tempSiuID.id())
      status = self.checkLastCommandStatus()
      # put primaryBoot flag back.
      self.__primaryBoot = cachePB

    # execute commit.  EPU difficulty has already been handled
    log.debug("committing to unit:%s, flags:%s, id:%s.  This may take up to %d seconds." % ( unit, 0, str(fileID), uploadTimeout))
    self.LFILUPLCOMMIT(LFILEUNIT=self.__LFSUNIT(unit),
                       LFILEFLAGS=0,
                       LFILEID=fileID.id())
    
    # when in primary boot, must examine boot hsk for file state.
    #   "Primary boot" doesn't just mean "siu primary boot"
    #   If SIU is in secondary and EPUx is in primary and we are uploading
    #   to EPU, that counts as primary.
    if self.__primaryBoot:
      pkt, tlm, handler = FILE_PBC_FILESTAT[unit]
      self.__fileState = -9999999     # watch variable
      vsc = self.getCmdDb().getVsc()
      def inlinePBCHandler(telemetry):# Inline handler to get PBC file stat.
        try:
          pkt = self.getCmdDb().getVsc().getTlmDb().decodeTelemetry(telemetry)
          self.__fileState = pkt.get_payload(tlm)
        except Exception, e:
          self.__fileState = -9999999

      # SIU comes in telem line.  EPUx comes in diag line.
      if handler == "diag":
        handler = vsc.getDiagHandler()
      else:
        handler = vsc.getTelemHandler()
      pbtTelem = TelemetryEvent(handler,
                                [vsc.getTlmDb().getApidFromName(pkt),],
                                inlinePBCHandler)
      # enable async handler and loop until watch variable changes to state 0,
      #  or until STANDARD_TIMEOUT.
      totWait = 0
      pbtTelem.enable()
      time.sleep(1)
      while self.__fileState != 0:
        totWait += 1
        time.sleep(1)
        if totWait > uploadTimeout:
          status = False
          break
      pbtTelem.disable()
    else:
      # in Secondary boot, wait for commit message to come back.
      status = self.checkLastCommandStatus(timeout = uploadTimeout)
      if not status:
        raise RuntimeError("Received bad status from File Upload Commit.  Check previous warning")

    # don't forget to tell the file that it has been uploaded
    if status: status = fswFile.setUploaded(True)
    return status

  def copyFile(self, unit, source, dest, destUnit=None):
    """!\brief  Copy a file on any LAT unit

    \param app         A testAppBase object  (your user application...)
    \param unit        LAT unit id to upload to
    \param source      An FswFileID object corresponding to the source file
    \param dest        An FswFileID object corresponding to the destination file
    \param destUnit    Unit to copy to (Default = None ==> same unit)
    \return status     True if success, False if failure
    """
    status = True
    LFS  = self.getCmdDb().LFS
    FILE = self.getCmdDb().FILE
    if destUnit is None:
      LFS.LFSFILECOPY(LFSUNIT=self.__LFSUNIT(unit),
                      LFSSRCFILEID=source.id(),
                      LFSDESTFILEID=dest.id(),
                      LFSPAD16=0)
      status &= self.checkLastCommandStatus()
    else:
      # copy from SIU to EPU
      self.cancelUpload(destUnit)
      
      self.LFILUPLEPU(LFILEUNIT=self.__LFSUNIT(destUnit),
                      LFILEPAD16 = 0,
                      LFILEID = source.id())
      status &= self.checkLastCommandStatus()
      if not status:
        log.warning("bad status in copyFile")
        return status
      time.sleep(5)
      self.LFILUPLCOMMIT( LFILEUNIT=self.__LFSUNIT(destUnit),
                          LFILEFLAGS = 0,
                          LFILEID = dest.id())
      status &= self.checkLastCommandStatus()
      

    return status

  def deleteFile(self, unit, fileID):
    """!\brief Delete a file on a LAT filesystem

    \param app         A testAppBase object  (your user application...)
    \param unit        LAT unit id to upload to
    \param fileID      An FswFileID object
    \return status     True if success, False if failure
    """
    self.getCmdDb().LFS.LFSFILEDELETE(LFSUNIT=self.__LFSUNIT(unit),
                                      LFSFILEID=fileID.id(),
                                      LFSPAD16=0)
    status = self.checkLastCommandStatus()
    return status

  def __LFSUNIT(self, realUnit, tranID=0):
    # Actual upload is 4 bits of unit and 12 bits of opaque transaction ID.
    return ( (realUnit & 0xf) << 12 | (tranID & 0xfff) )

  def __uploadChunk(self, uplChunk, offset, nDataBytes):
    lChunk = len(uplChunk)
    if lChunk > nDataBytes:
      raise ValueError, "Length of a chunk can't be greater than %d bytes" % nDataBytes
    # Pad to nDataBytes bytes
    if (nDataBytes-lChunk):
      uplChunk = uplChunk + ( chr(0) * (nDataBytes-lChunk) )
    l = array.array('B',uplChunk).tolist()
    self.LFILUPLDATA(offset, *l)

    # from binascii import hexlify
    # print "chunk", offset, hexlify(uplChunk)
    # We do not get command confirmation for LFILUPLDATA telecommands

  def checkLastCommandStatus(self, timeout=STANDARD_TIMEOUT):
    # boilerplate code that waits for completion on a command
    #   and checks the completion status.  If bad, log a warning
    status = True
    if self.__primaryBoot:
      print "WARNING: command completion disabled"
      import time
      time.sleep(1)
      return status
    app = self.getCmdDb()
    app.waitForConfirmation(timeout)
    statusMsg = app.getConfirmationMsg()
    if not app.getVsc().getTlmDb().msgIsSuccess(statusMsg):
      log.warning(app.getVsc().getTlmDb().msgByValue(statusMsg))
      status=False
    return status

  def setPrimaryBoot(self, bool):
    self.__primaryBoot = bool








######################################################
class FswFile(object):
  """!\brief Functional interface to a generic file in FSW
  """
  def __init__(self, fid, bin, uploaded=False):
    """!\brief Constructor

    \param  fid       FswFileID for this file
    \param  bin       Raw binary representation of file
    \param  uploaded  True if file has been uploaded
    """
    self.__fid      = fid
    self.__bin      = bin
    self.__uploaded = uploaded

  def fileID(self):
    """!\brief retrieve file ID

    \return a FswFileID object
    """
    return self.__fid

  def binary(self):
    """!\brief retrieve file data

    \return a raw binary string
    """
    return self.__bin

  def uploaded(self):
    """!\brief retrieve file upload status

    \return True if uploaded
    """
    return self.__uploaded

  def setUploaded(self, bool):
    """!\brief Set file upload status

    \param  bool True if file is uploaded
    \return True if this succeeds
    """
    self.__uploaded = bool
    return True

  def write(self, stream):
    """!\brief write file to stream  (file descriptor)

    \param  stream  Any writable file descriptor
    """
    stream.write(self.__bin)
    stream.flush()    # because of buffering, string may not show up until flush() or close()

class FswFileFromData(FswFile):
  """!\brief A memory resident FSW file interface
  """
  def __init__(self, fileID, fileDat, uploaded=False):
    """!\brief Constructor

    \param fileID    FswFileID for this file
    \param fileDat   The file's raw data
    \param uploaded  True if file has been uploaded
    """
    try:
      FswFile.__init__(self, fileID, fileDat, uploaded)
    except:
      raise

class FswFileFromDisk(FswFile):
  """!\brief A disk resident FSW file interface
  """
  def __init__(self, fileID, fileName, uploaded=False):
    """!\brief Constructor

    \param fileID    FswFileID for this file
    \param fileName  A fully qualified file name to open
    \param uploaded  True if file has been uploaded
    """
    self.name = fileName
    try:
      self.file = open(fileName, 'rb')
      bin = self.file.read()
      FswFile.__init__(self, fileID, bin, uploaded)
    except Exception, e:
      raise



class FswFileID(object):
  """!\brief Define a Flight Software File ID

  Defines the FSW file id, as specified in the FILE FSW traveler document
  """
  MASK_DEVICE    = 0x7
  MASK_DIRECTORY = 0x7F
  MASK_FILENUM   = 0x3FFFFF
  SHFT_DEVICE    = 29
  SHFT_DIRECTORY = 22
  SHFT_FILENUM   =  0
  def __init__(self, *argv, **argd):
    """!\brief Polymorphic constructor

    Takes 1 or 3 arguments {fileID} or {device,directory,fileNumber}
    Arguments can be either named or not.  A mix of both named and unnamed
    args is verboten.

    \param fileID    A LAT file ID (32 bit packed)

    \param device      Device to store file to [0..7]
    \param directory   Directory to store file to [0..0x7F]
    \param fileNumber  File Number [0..0x3FFFFF]
    """
    if len(argv) == 1:
      self.__fid = argv[0]
    elif len(argv) == 3:
      self.__fid = self.__computeFileID(argv[0], argv[1], argv[2])
    elif len(argd) == 1:
      self.__fid = argd['fileID']
    elif len(argd) == 3:
      self.__fid = self.__computeFileID( argd['device'], argd['directory'], argd['fileNumber'] )
    else:
      raise SyntaxError( "Incorrect arguments for FswFileID constructor: argv=%s, argd=%d" % (str(argv), str(argd)) )

  def __computeFileID(self, device, directory, filNum):
    """!\brief Compute the 32 bit FSW destination from component parts
    """
    ### Should we also check the following masks and ranges for __fid specified constructors?

    if device & self.MASK_DEVICE != device:
      raise "Selected device number 0x%x is greater than the maximum of 0x7" % device
    if directory & self.MASK_DIRECTORY != directory:
      raise "Selected directory number 0x%x is greater than the maximum of 0x7F" % directory
    if directory not in FSW_DIR_RANGE + [ FSW_DIR_ROOT ]:
      raise "Selected directory number 0x%x not in allowed directory range of [0..6f, 7f]" % directory
    if filNum & self.MASK_FILENUM != filNum:
      raise "Selected file number 0x%x is greater than the maximum of 0x3fffff" % filNum

    dest = (device    << self.SHFT_DEVICE) + \
           (directory << self.SHFT_DIRECTORY) + \
           (filNum    << self.SHFT_FILENUM)
    return dest

  def id(self):
    """!\brief retrieve file ID

    \return 32 bit file ID
    """
    return self.__fid

  def device(self):
    """!\brief retrieve device

    \return 3 bit device
    """
    return (self.__fid >> self.SHFT_DEVICE ) & self.MASK_DEVICE

  def directory(self):
    """!\brief retrieve file ID

    \return 7 bit directory
    """
    return (self.__fid >> self.SHFT_DIRECTORY ) & self.MASK_DIRECTORY

  def fileNumber(self):
    """!\brief retrieve file ID

    \return 22 bit file Number
    """
    return (self.__fid >> self.SHFT_FILENUM ) & self.MASK_FILENUM

  def __str__(self):
    return '/%s/d%03d/f%07d' % \
           ( FSW_DEVICE_NAMES[self.device()], self.directory(), self.fileNumber())


class FswDirectoryItem(object):
  """!\brief Representation of an item in an FSW directory

  REVISIT:  Make the members private and member functions?
  """
  __isdirTxt = [ '-', 'd' ]
  __roTxt    = [ 'w', '-' ]
  def __init__(self, fileID, ro, size, isdir, time, unit, blocks):
    """!\ Constructor

    \param fileID  A FswFileID object
    \param ro      file is ReadOnly flag (True/False)
    \param size    File size in bytes
    \param isdir   file is a directory flag  (True/False)
    \param time    File modification time in LAT epoch
    \param unit    File unit (i.e. FSW_UNIT_SIU)
    \param blocks  REVISIT::  file blocks, maybe unused?
    """
    self.id     = fileID
    self.ro     = ro
    self.size   = size
    self.isdir  = isdir
    self.time   = time
    self.unit   = unit
    self.blocks = blocks
    self.header = None

  def dump(self, stream):
    """!\brief dump a directoryItem to a file descriptor.

    \param stream  Any file descriptor  (ie. sys.stdout)
    """
    msg = ""
    msg += self.__isdirTxt[self.isdir]
    msg += 'r'                            # always readable
    msg += self.__roTxt[self.ro]
    msg += ' %4s' % FSW_UNIT_NAMES[self.unit]
    msg += ' %10d' % self.size
    try:
      msg += ' %s' % utcfromtimestamp(int(self.time), 0).ctime()
    except:
      msg += ' %s' % utcfromtimestamp(0, 0).ctime()
    strid = str(self.id)
    if self.isdir: strid = strid[:-8]
    msg += ' %s' % strid
    print >> stream, msg


class FswDirectory(object):
  """!\brief Representation of a FSW directory

  """
  def __init__(self, dirID):
    """!\ Constructor

    \param dirID  A FswFileID object
    """
    self.__id = dirID
    self.__files = []

  def addItem(self, f):
    """!\brief Add a file to this representation

    \param f  An FswDirectoryItem
    """
    if f not in self.__files:
      self.__files.append(f)

  def files(self):
    """!\brief return a list of all files in this directory rep
    """
    return self.__files

  def id(self):
    """!\brief retrieve file ID

    \return a FswFileID object
    """
    return self.__id

  def setId(self, id):
    """!\brief set the ID for this directory

    \param id   A FswFileID object
    """
    self.__id = id

  def dump(self, stream):
    """!\brief dump a directory to a file descriptor

    \param stream  Any file descriptor  (ie. sys.stdout)
    """
    msg = '/%s/d%03d' % ( FSW_DEVICE_NAMES[self.id().device()], self.id().directory())
    print >> stream, msg
    for item in self.__files:
      item.dump(stream)

def addFmxHeader(inFile, doCompress=False, preCompressed=False, name=None,
                 type=None, key=None, timestamp=None):
  """!\brief Add an fmx header to a file.  compressing on the fly if requested

  \param  inFile         An FswFile object
  \param  doCompress     True/False  to compress on fly/not
  \param  preCompressed  True/False  if already compressed
  \param  name
  \param  type
  \param  key
  \param  timestamp

  \return An FswFile object with proper headers for upload
  """
  if not FILETOOL_HDR_ALLOW:
    raise RuntimeError('Adding an FMX header is not possible on non-unix systems')

  hdrCmd  = FSW_ADD_HEADER_CMD
  if hasattr(inFile, 'name'):  # passing a disk resident file
    rawFile = inFile
  else:
    rawFile  = tempfile.NamedTemporaryFile(mode='w+b')
    inFile.write(rawFile.file)
  hdrFile  = tempfile.NamedTemporaryFile(mode='w+b')


  if doCompress:
    # compress in tmpspace
    cmpFile  = tempfile.NamedTemporaryFile(mode='w+b')

    cmd  = FSW_COMPRESS_CMD
    cmd += ' %s' % rawFile.name
    cmd += ' %s' % cmpFile.name
    rc = os.system(cmd)
    rawFile = cmpFile
  if doCompress or preCompressed:
    hdrCmd += ' -c'

  # Various options for the header adding command
  if name is not None:
    hdrCmd += ' -n=%s' % name
  if type is not None:
    hdrCmd += ' -t=%s' % str(type)
  if key is not None:
    hdrCmd += ' -k=%s' % str(key)
  if timestamp is not None:
    hdrCmd += ' -s=%s' % str(timestamp)

  hdrCmd += ' %s' % rawFile.name
  hdrCmd += ' %s' % hdrFile.name
  rc = os.system(hdrCmd)

  retFile = FswFileFromDisk(inFile.fileID(), hdrFile.name)

  return retFile

---