from AZutilities import dataUtilities
import unittest
import os,sys
from trainingMethods import AZorngPLS
from trainingMethods import AZorngCvSVM
from trainingMethods import AZorngRF
from trainingMethods import AZorngCvANN
from AZutilities import evalUtilities
from AZutilities import miscUtilities
import AZOrangeConfig as AZOC
import AZorngTestUtil
import AZLearnersParamsConfig
import commands
from AZutilities import paramOptUtilities
class optimizerTest(AZorngTestUtil.AZorngTestUtil):
def setUp(self):
"""Creates the training and testing data set attributes. """
# The original templateProfile created at installation is needed in order to call the runScript of optimizer from
# appspack with proper environment.
self.assert_(os.path.isfile(os.path.join(os.environ["AZORANGEHOME"], "templateProfile")),"Missing file: "+os.path.join(os.environ["AZORANGEHOME"], "templateProfile"))
self.contTestDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/Reg_No_metas_Test.tab")
self.contTrainDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/Reg_No_metas_Train.tab")
self.discTrainDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/BinClass_No_metas_Train.tab")
self.discTestDataPath = os.path.join(AZOC.AZORANGEHOME,"tests/source/data/BinClass_No_metas_Test.tab")
# Read in the data
self.discTrain = dataUtilities.DataTable(self.discTrainDataPath)
self.discTest = dataUtilities.DataTable(self.discTestDataPath)
self.contTrain = dataUtilities.DataTable(self.contTrainDataPath)
self.contTest = dataUtilities.DataTable(self.contTestDataPath)
def testRF_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.612] #opencv1.1: 0.90480000000000005
optimizer = paramOptUtilities.Appspack()
learner = AZorngRF.RFLearner()
learnerName = "RFLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["nActVars"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Set the NumThreads
pars.setParameter("NumThreads","optimize",False)
# Change the default
pars.setParameter("NumThreads","default","1")
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_RF_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
useDefaultPoint = False,\
findMin=fMin,\
runPath = runPath,\
useStd = False,\
verbose = 0,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
# Alternatively, we can set machinefile=0 to us also all available cores
machinefile =0)
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
print "Number of cores used: ",optimizer.np
verbTunedPars = optimizer.getTunedParameters()
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
#Check if the number of processors used are all the core available
notUsed, out = commands.getstatusoutput("cat /proc/cpuinfo | grep processor")
self.assertEqual(optimizer.np, len(out.split("\n")))
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assert_(len(verbTunedPars["optParam"]) in [8,9,10])
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:" + str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=3) # Must be > 2
miscUtilities.removeDir(runPath)
def testCvANN_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.585]
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvANN.CvANNLearner()
learnerName = "CvANNLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["maxIter","nHidden"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvANN_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile =["localhost:2","localhost:2"])
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:",verbTunedPars["ResIdx"]
print "Optimizer runPath:",runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 14)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:" + str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=5) # Must be > 2
miscUtilities.removeDir(runPath)
def testCvSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.585] #Should be the result of the default point
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C"]#, "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
pars.setParameter("priors","default",{"POS":50, "NEG":4}) # These priors are to ensure the default point will be the best!
pars.setParameter("gamma","default",0.001) # This is a bad value to ensure the default point will be the best!
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_CvSVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "Best result index from intRes file:",verbTunedPars["ResIdx"]
print "Optimizer runPath:",runPath
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:"+str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=5)
#Check Priors
self.assertEqual( tunedPars[1]["priors"],"None")
learner.priors = {'NEG':4,'POS':2}
classifier = learner(self.discTest)
classifier.write(os.path.join(runPath,"CvSVMModel"))
file = open(os.path.join(runPath,"CvSVMModel/model.svm"),"r")
lines = file.readlines()
file.close()
priors = [round(x,2) for x in eval((lines[18].strip()).replace("data:",""))]
self.assertEqual(len(priors),2)
self.assertEqual(priors[self.discTest.domain.classVar.values.index("POS")],2.0*float(tunedPars[1]["C"]))
self.assertEqual(priors[self.discTest.domain.classVar.values.index("NEG")],4.0*float(tunedPars[1]["C"]))
miscUtilities.removeDir(runPath)
def disable_testSVM_MPI(self):
"""
Tests changing the default range of the optimizer.
Use MPI versio0n of appspack
"""
# Classification accuracy:
ExpectedCA = [0.6] #New at orange2.0
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
# Calculate the optimal parameters. This can take a long period of time!
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
useParameters = pars.getParametersDict(),\
findMin=fMin,\
runPath = runPath,\
verbose = 0,\
useStd = False,\
#advancedMPIoptions = "-all-local -allcpus") # to use this the
# file "<MPICHDIR>/share/machines.LINUX must be properly configured"
np = 4,\
machinefile = os.path.realpath(os.path.join(os.environ["AZORANGEHOME"], "tests/source/APPS_machines")))
verbTunedPars = optimizer.getTunedParameters()
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check that the learner was optimized
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
# Check the number of optimized parameters
self.assertEqual(len(verbTunedPars["optParam"]), 12)
# Check the accuracy
self.assert_(round(verbTunedPars["bestRes"],3) in [round(x,3) for x in ExpectedCA],"Got:"+str(verbTunedPars["bestRes"]))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig: 14) Must be > 2
miscUtilities.removeDir(runPath)
def testSVM_MPI_2(self):
###################################################################
# Test other way of setting appspack
###################################################################
# Classification accuracy:
ExpectedCA = 0.6
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI_2")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useParameters = pars.getParametersDict(),\
verbose = 0,
useStd = False,\
advancedMPIoptions = None,
np = 4,
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
self.assertEqual(round(tunedPars[0],3),round(ExpectedCA,3))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig 14) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
def disable_testSVM_MPI_3(self):
###################################################################
# Test other way of setting appspack
###################################################################
# Classification accuracy:
ExpectedCA = 0.6
optimizer = paramOptUtilities.Appspack()
learner = AZorngCvSVM.CvSVMLearner()
learnerName = "CvSVMLearner"
# Create an interface for setting optimizer parameters
pars = AZLearnersParamsConfig.API(learnerName)
# Set all parameters to not be optimized
pars.setOptimizeAllParameters(False)
parameterList = ["C", "gamma"]
# Set the parameters in parameterList to be optimized
for parameter in parameterList:
pars.setParameter(parameter,"optimize",True)
# Change the range
pars.setParameter("C","range",miscUtilities.power2Range(-5,2,1))
trainFile=self.discTrainDataPath
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_SVM_MPI_3")
evalM = "AZutilities.evalUtilities.CA"
fMin = False
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = optimizer(learner=learner,\
dataSet=trainFile,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useParameters = pars.getParametersDict(),\
verbose = 0,\
useStd = False,\
advancedMPIoptions = "-v -np 4",\
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
# Check if the MPI version was used
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(optimizer.usedMPI, True)
self.assertEqual(round(tunedPars[0],3),round(ExpectedCA,3))
self.assert_(len(dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt")))>=12) # (orig 14) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
def test_PLS_MPI(self):
"""Test of optimizer with advanced configuration
"""
# Classification accuracy:
ExpectedCA = [0.567049808429, 0.567049808429]
ExpectedCAwithTest = [0.6]
#Create the appspack instance
opt=paramOptUtilities.Appspack()
#Learner to be optimized
learner=AZorngPLS.PLSLearner()
#dataset to use in the parameters optimization (Discrete class in this example)
dataSet=self.discTrainDataPath
# Define the objective function. This requires:
# defining the extreme to find (the min or max): findMin=True or findMin=False
fMin=False
# defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults):
# evaluateMethod="AZutilities.evalUtilities.CA"
evalM="AZutilities.evalUtilities.CA"
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_PLS_MPI")
# Load the optimization parameters from the default configuration (AZLearnersParamsConfig.py)
parameters = AZLearnersParamsConfig.API("PLSLearner")
parameters.setParameter("method","default",'pls1')
# change the optimization parameters
parameters.setParameter("method","default",'pls1') # make the method fixed (do not optimize) to be pls1
parameters.setParameter("method","optimize",False)
parameters.setParameter("method","rangeType","values") # assure that the keyword for the values range type is
#set correctly for values instead of interval
parameters.setParameter("k","range",[1 , 3 , 5 , 6 , 10]) # make the method fixed (do not optimize) to be pls1
parameters.setParameter("k","optimize",True)
parameters.setParameter("k","rangeType","values") # assure that the keyword for the values range type is
# set correctly for values instead of interval
# Run the appspack which will configure the input learner and aditionaly return
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = opt(learner=learner,\
dataSet=dataSet,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useStd = False,\
useParameters = parameters.getParametersDict(),\
# The 'useParameters' is mandatory, even placing a file with the new configurations in the
# running directory, that we pass to the optimizer the correct parameters to use.
# The parameters placed on the running directory are for appspack usage, and the
# optimizer needs to know what parameters appspack will use, otherwise, it will
# load the default ones
verbose = 0,
np = 6,
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(opt.usedMPI, True)
self.assert_(round(tunedPars[0],3) in [round(x,3) for x in ExpectedCAwithTest],"Got:" + str(tunedPars[0]))
#The learner is now with its optimized parameters already set, so we can now make a classifier out of it
classifier = learner(self.discTrain)
CA = evalUtilities.getClassificationAccuracy(self.discTest,classifier)
self.assert_(round(CA,3) in [round(x,3) for x in ExpectedCA], "Got: "+str(CA))
resData1 = dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt"))
self.assert_(len(resData1)>=4) # (orig: 5) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
def test_PLS_MPI_2(self):
###################################################################
# Test other way of setting appspack
###################################################################i
# Classification accuracy:
ExpectedCA = [0.567049808429, 0.593869731801]
ExpectedCAwithTest = [0.6, 0.579] #New at orange2.0
#Create the appspack instance
opt=paramOptUtilities.Appspack()
#Learner to be optimized
learner=AZorngPLS.PLSLearner()
#dataset to use in the parameters optimization (Discrete class in this example)
dataSet=self.discTrainDataPath
# Define the objective function. This requires:
# defining the extreme to find (the min or max): findMin=True or findMin=False
fMin=False
# defining the method for evaluation (must be a method that accepts as input an orngTest.ExperimentResults):
# evaluateMethod="AZutilities.evalUtilities.CA"
evalM="AZutilities.evalUtilities.CA"
# Create a directory for running the appspack (if not defined it will use the present working directory)
runPath = miscUtilities.createScratchDir(desc="ParamOptTest_PLS_MPI_2")
# Load the optimization parameters from the default configuration (AZLearnersParamsConfig.py)
parameters = AZLearnersParamsConfig.API("PLSLearner")
parameters.setParameter("method","default",'pls1')
# change the optimization parameters
parameters.setParameter("method","default",'pls1') # make the method fixed (do not optimize) to be pls1
parameters.setParameter("method","optimize",False)
parameters.setParameter("method","rangeType","values") # assure that the keyword for the values range type is
#set correctly for values instead of interval
parameters.setParameter("k","range",[1 , 3 , 5 , 6 , 10]) # make the method fixed (do not optimize) to be pls1
parameters.setParameter("k","optimize",True)
parameters.setParameter("k","rangeType","values") # assure that the keyword for the values range type is
# set correctly for values instead of interval
#[<minimum of objective function found>, <optimized parameters>]
tunedPars = opt(learner=learner,\
dataSet=dataSet,\
evaluateMethod = evalM,\
findMin=fMin,\
runPath = runPath,\
useParameters = parameters.getParametersDict(),\
verbose = 0,
useStd = False,\
advancedMPIoptions = None,
np = 4,
machinefile = ["localhost:2","localhost:2"])
print "Returned: ", tunedPars
print "====================== optimization Done ==========================="
print "Learner optimized flag = ", learner.optimized
print "Tuned parameters = ", tunedPars[1]
print "Best optimization result = ", tunedPars[0]
print "check the file intRes.txt to see the intermediate results of optimizer!"
self.assertEqual(learner.optimized,True)
# Check if the MPI version was used
self.assertEqual(opt.usedMPI, True)
self.assert_(round(tunedPars[0],3) in [round(x,3) for x in ExpectedCAwithTest],"Got:" + str(tunedPars[0]))
#The learner is now with its optimized parameters already set, so we can now make a classifier out of it
classifier = learner(self.discTrain)
CA = evalUtilities.getClassificationAccuracy(self.discTest,classifier)
self.assert_(round(CA,3) in [round(x,3) for x in ExpectedCA], "Got: "+str(CA))
resData2 = dataUtilities.DataTable(os.path.join(runPath,"optimizationLog.txt"))
self.assert_(len(resData2)>=4) # (orig 5) Must be > 2
#print runPath
miscUtilities.removeDir(runPath)
if __name__ == "__main__":
##scPA Test first if the service is available from this machine. If not, report it and make no tests
if not os.path.isfile(os.path.join(os.environ["AZORANGEHOME"],"orangeDependencies/bin/appspack_mpi")):
print "SKIP (No appspack_mpi installed)"
sys.exit(1)
#Test SSH connection to localhost
# import AutoConfig
# report = AutoConfig.config()()
# if report != "":
# print "AutoConfig report:\n"+report
# print "SKIP (Unable to configure properly interfaceless SSH)"
# sys.exit(1)
##ecPA
suite = unittest.TestLoader().loadTestsFromTestCase(optimizerTest)
unittest.TextTestRunner(verbosity=2).run(suite)
