Source Code for Module TEES.Detectors.ToolChain

  1  import sys, os, copy, types 
  2  from Detector import Detector 
  3  from Core.Model import Model 
  4  import Utils.ElementTreeUtils as ETUtils 
  5  import Utils.Parameters as Parameters 
  6  import Utils.ElementTreeUtils as ETUtils 
  7   
  8  NOTHING = object() 
  9   
 10 -class ToolChain(Detector): 
 11 -    def __init__(self): 
 12          Detector.__init__(self) 
 13          # Settings 
 14          self.STATE_TOOLCHAIN = "PROCESS" 
 15          self.steps = [] 
 16          for step in self.getDefaultSteps(): 
 17              self.addStep(*step) 
 18          self.intermediateFilesAtSource = False 
 19          self.compressIntermediateFiles = True 
 20          self.intermediateFileTag = "temp" 
 21          self.modelParameterStringName = None 
 22       
 23 -    def getDefaultSteps(self): 
 24          return [] 
 25       
 26 -    def getDefaultParameters(self, defaults=None, defaultValue=None): 
 27          if defaults == None: 
 28              defaults = {"omitSteps":None, "intermediateFiles":None} 
 29          for step in self.getDefaultSteps(): 
 30              for argName in sorted(step[2].keys()): 
 31                  parameterName = step[0] + "." + argName 
 32                  if defaultValue == NOTHING: 
 33                      defaults[parameterName] = NOTHING 
 34                  else: 
 35                      defaults[parameterName] = defaultValue 
 36          return defaults 
 37   
 38 -    def getParameters(self, parameters=None, model=None, defaultValue=None, modelParameterStringName=None): 
 39          if modelParameterStringName == None: 
 40              modelParameterStringName = self.modelParameterStringName 
 41          if parameters == None and model != None: 
 42              model = self.openModel(model, "r") 
 43              parameters = model.getStr(modelParameterStringName, defaultIfNotExist=None) 
 44          defaultStepNames = [x[0] for x in self.getDefaultSteps()] 
 45          valueLimits={"omitSteps":defaultStepNames, "intermediateFiles":defaultStepNames + [True]} 
 46          defaults = self.getDefaultParameters(defaultValue=defaultValue) 
 47          return Parameters.get(parameters, defaults, valueLimits=valueLimits) 
 48       
 49 -    def applyParameters(self, parameters): 
 50          self.select.markOmitSteps(parameters["omitSteps"]) 
 51          for step in self.steps: 
 52              for argName in sorted(step[2].keys()): 
 53                  parameterName = step[0] + "." + argName 
 54                  if parameters[parameterName] != NOTHING: 
 55                      step[2][argName] = parameters[parameterName] 
 56              if parameters["intermediateFiles"] != None: 
 57                  if parameters["intermediateFiles"] != True and step in parameters["intermediateFiles"]: 
 58                      self.setIntermediateFile(step[0], step[3]) 
 59                  else: 
 60                      self.setIntermediateFile(step[0], None) 
 61       
 62 -    def addStep(self, name, function, argDict, intermediateFile=None, ioArgNames={"input":"input", "output":"output"}): 
 63          assert name not in [x[0] for x in self.steps], (name, steps) 
 64          self.steps.append([name, function, argDict, intermediateFile, ioArgNames]) 
 65       
 66 -    def setArgForAllSteps(self, argument, value, argMustExist=True): 
 67          for step in self.steps: 
 68              if argMustExist and argument not in step[2]: 
 69                  continue 
 70              step[2][argument] = value 
 71       
 72 -    def stepArgs(self, step): 
 73          for s in self.steps: 
 74              if step == s[0]: 
 75                  return s[2] 
 76          assert False 
 77           
 78 -    def setIntermediateFile(self, stepName, filename): 
 79          for s in self.steps: 
 80              if stepName == s[0]: 
 81                  if filename == True: 
 82                      filename = s[3] 
 83                  elif filename in [False, "None", None]: 
 84                      filename = None 
 85                  s[3] = filename 
 86                  return 
 87          assert False, (stepName, filename) 
 88       
 89 -    def setIntermediateFiles(self, stepToFilename): 
 90          for key in sorted(stepToFilename.keys()): 
 91              self.setIntermediateFile(key, stepToFilename[key]) 
 92       
 93 -    def setNoIntermediateFiles(self): 
 94          for step in self.steps: 
 95              self.setIntermediateFile(step[0], None) 
 96       
 97 -    def getIntermediateFilePath(self, step): 
 98          if step[3] != None: 
 99              if self.intermediateFilesAtSource: 
100                  if type(self.source) in types.StringTypes: 
101                      firstSource = self.source.split(",") # this may be a list of directories 
102                      if os.path.isfile(firstSource): 
103                          rv = firstSource + "-" + step[3] 
104                      else: # is a directory 
105                          rv = os.path.join(firstSource, step[3]) 
106                  else: 
107                      rv = filename 
108              else: 
109                  rv = os.path.join(self.outDir, self.intermediateFileTag + "-" + step[3]) 
110              if self.compressIntermediateFiles and not rv.endswith(".gz"): 
111                  rv += ".gz"  
112              return rv 
113          else: 
114              return None 
115       
116 -    def process(self, input, output, parameters=None, model=None, fromStep=None, toStep=None, omitSteps=None): 
117          self.initVariables(source=input, xml=input, outDir=os.path.dirname(output)) 
118          if os.path.basename(output) != "": 
119              self.intermediateFileTag = os.path.basename(output) 
120          else: 
121              self.intermediateFileTag = "" 
122          self.enterState(self.STATE_TOOLCHAIN, [x[0] for x in self.steps], fromStep, toStep, omitSteps) 
123          parameters = self.getParameters(parameters, model, defaultValue=NOTHING) 
124          self.applyParameters(parameters) 
125          # Run the tools 
126          print >> sys.stderr, "Tool chain parameters:", Parameters.toString(parameters, skipKeysWithValues=[NOTHING], skipDefaults=self.getDefaultParameters()) 
127          if os.path.exists(output) and not os.path.isdir(output): 
128              print >> sys.stderr, "Removing existing preprocessor output file", output 
129              os.remove(output) 
130          savedIntermediate = None # Output from a previous step if "fromStep" is used 
131          for step in self.steps: 
132              if self.checkStep(step[0]): 
133                  if savedIntermediate != None: # A previous run of the program saved an intermediate file 
134                      print >> sys.stderr, "Reading input from saved intermediate file", savedIntermediate 
135                      self.xml = ETUtils.ETFromObj(savedIntermediate) 
136                      savedIntermediate = None 
137                  stepArgs = copy.copy(step[2]) # make a copy of the arguments to which i/o can be added 
138                  stepArgs[step[4]["input"]] = self.xml # the input 
139                  if self.getIntermediateFilePath(step) != None: # this step should save an intermediate file 
140                      stepArgs[step[4]["output"]] = self.getIntermediateFilePath(step) 
141                  print >> sys.stderr, "Running step", step[0], "with arguments", stepArgs 
142                  step[1](**stepArgs) # call the tool 
143              elif self.getStepStatus(step[0]) == "BEFORE": # this step was run earlier 
144                  savedIntermediate = self.getIntermediateFilePath(step) 
145          # End state and return 
146          xml = self.xml # state-specific member variable self.xml will be removed when exiting state 
147          self.exitState() 
148          if self.state == None: # if the whole toolchain has finished, return the final product 
149              if not os.path.isdir(output): # if output is a directory, it was given only for storing intermediate files ... 
150                  ETUtils.write(xml, output) # ... otherwise, save the final output 
151              return xml 
152          else: 
153              return None 
154