package us.hall.weka; import java.util.Arrays; import java.util.Random; import java.util.Vector; import java.util.HashMap; import java.io.File; import java.io.PrintStream; import java.io.Serializable; import java.util.concurrent.TimeUnit; import weka.classifiers.Evaluation; import weka.core.Instances; import weka.core.OptionHandler; import weka.core.RevisionHandler; import weka.core.RevisionUtils; import weka.classifiers.meta.CVParameterSelection; import weka.core.Utils; public class CVStepper extends CVParameterSelection implements OptionHandler { /** number of iterations to try and improve */ static final int TRIES = 3; // This could be made a command-line parm /** Are we being run with outer CV? We might need to differentiate split as well and not just cv or no-cv? */ boolean m_cv = true; /** If cv which call are we on */ private static int cvFoldNum; /** If number of folds option was given */ private static int numFolds; /** We use our own debug flag */ boolean m_Debug = false; boolean stepperRun = false; double m_StepperBestPerformance = 0d; String[] m_StepperBestClassifierOptions; /** * cross-validation calls involve multiple invocations to new instances * in order to accumulate benchmark information the fields need to be * static. This makes this not suitable for multi-threaded. */ /** elapsed time for stepper run */ long elapsedStepper; /** elapsed time for exhaustive run */ long elapsedExhaustive; /** cv elapsed time for stepper run */ static long cvElapsedStepper; /** cv elapsed exhaustive times */ static long cvElapsedExhaustive; /** total of elapsed stepper times */ // static long totalStepperElapsed; /** total of elapsed exhaustive times */ // static long totalExhaustiveElapsed; /** Stepper searches */ HashMap searches = new HashMap(); /** Initially found optimal options hashcode */ static double originalOptimalHash; /** CV found alternate optimal options */ static HashMap optimals = new HashMap(); static HashMap optimalOptions = new HashMap(); /** Did stepper search include the configuration considered exhaustive best? */ boolean optimalSearch = false; /** stepper evaluations */ int stepperEvaluations; /** cv stepper evaluations */ static int cvStepperEvaluations; /** total number of stepper evaluations */ // static int totalStepperEvaluations; /** exhaustive evaluations */ int exhaustiveEvaluations; /** cv exhaustive evaluations */ static int cvExhaustiveEvaluations; /** total number of exhaustive evaluations */ // static int totalExhaustiveEvaluations; /** total number of steps */ // static int totalSteps; /** Changed by the (E)xhaustive commandline parameter */ boolean stepper = true; /** Changed by the (B)enchmark commandline parameter */ boolean benchmark = false; /** Benchmark information for stepper search */ int numSteps; /** cv number of steps */ static int cvNumSteps; /** The mimimum improvement */ protected double epsilon = 0.001d; /** The set of parameters to cross-validate over */ protected Vector m_CVParams = new Vector(); /** * A data structure to hold values associated with a single * cross-validation search parameter */ class StepperCVParameter implements Serializable, RevisionHandler { /** for serialization */ static final long serialVersionUID = -4668812017709421953L; /** Char used to identify the option of interest */ private String m_ParamChar; /** Lower bound for the CV search */ private double m_Lower; /** Upper bound for the CV search */ private double m_Upper; /** Number of steps during the search */ private double m_Steps; /** The parameter value with the best performance */ private double m_ParamValue; /** True if the parameter should be added at the end of the argument list */ private boolean m_AddAtEnd; /** True if the parameter should be rounded to an integer */ private boolean m_RoundParam; /** Stepper saves increment */ private double m_increment; /** Stepper allows multiplication increment */ private char m_operator; /** * Constructs a CVParameter. * * @param param the parameter definition * @throws Exception if construction of CVParameter fails */ public StepperCVParameter(String param) throws Exception { String[] parts = param.split(" "); if (parts.length < 4 || parts.length > 5) { throw new Exception("CVParameter " + param + ": four or five components expected!"); } try { Double.parseDouble(parts[0]); throw new Exception("CVParameter " + param + ": Character parameter identifier expected"); } catch (NumberFormatException n) { m_ParamChar = parts[0]; } try { m_Lower = Double.parseDouble(parts[1]); } catch (NumberFormatException n) { throw new Exception("CVParameter " + param + ": Numeric lower bound expected"); } if (parts[2].equals("A")) { m_Upper = m_Lower - 1; } else if (parts[2].equals("I")) { m_Upper = m_Lower - 2; } else { try { m_Upper = Double.parseDouble(parts[2]); if (m_Upper < m_Lower) { throw new Exception("CVParameter " + param + ": Upper bound is less than lower bound"); } } catch (NumberFormatException n) { throw new Exception("CVParameter " + param + ": Upper bound must be numeric, or 'A' or 'N'"); } } try { m_Steps = Double.parseDouble(parts[3]); } catch (NumberFormatException n) { throw new Exception("CVParameter " + param + ": Numeric number of steps expected"); } if (parts.length == 5 && parts[4].equals("R")) { m_RoundParam = true; } else if (parts.length == 5 && parts[4].equals("*")) { m_operator = '*'; } else if (parts.length == 6 && parts[5].equals("*")) { m_operator = '*'; } else m_operator = '+'; } /** * Returns a CVParameter as a string. * * @return the CVParameter as string */ public String toString() { String result = m_ParamChar + " " + m_Lower + " "; switch ((int)(m_Lower - m_Upper + 0.5)) { case 1: result += "A"; break; case 2: result += "I"; break; default: result += m_Upper; break; } result += " " + m_Steps; if (m_RoundParam) { result += " R"; } return result; } /** * Returns the revision string. * * @return the revision */ public String getRevision() { return RevisionUtils.extract("$Revision: 10141 $"); } } /** * Create the options array to pass to the classifier. The parameter * values and positions are taken from m_ClassifierOptions and * m_CVParams. * * @return the options array */ protected String [] createOptions() { String [] options = new String [m_ClassifierOptions.length + 2 * m_CVParams.size()]; int start = 0, end = options.length; // Add the cross-validation parameters and their values for (int i = 0; i < m_CVParams.size(); i++) { StepperCVParameter cvParam = (StepperCVParameter)m_CVParams.elementAt(i); double paramValue = cvParam.m_ParamValue; if (cvParam.m_RoundParam) { // paramValue = (double)((int) (paramValue + 0.5)); paramValue = Math.rint(paramValue); } boolean isInt = ((paramValue - (int)paramValue) == 0); if (cvParam.m_AddAtEnd) { options[--end] = "" + ((cvParam.m_RoundParam || isInt) ? Utils.doubleToString(paramValue,4) : cvParam.m_ParamValue); //Utils.doubleToString(paramValue,4); options[--end] = "-" + cvParam.m_ParamChar; } else { options[start++] = "-" + cvParam.m_ParamChar; options[start++] = "" + ((cvParam.m_RoundParam || isInt) ? Utils.doubleToString(paramValue,4) : cvParam.m_ParamValue); //+ Utils.doubleToString(paramValue,4); } } // Add the static parameters System.arraycopy(m_ClassifierOptions, 0, options, start, m_ClassifierOptions.length); return options; } /** * Finds the best parameter combination. (recursive for each parameter * being optimised). * * @param depth the index of the parameter to be optimised at this level * @param trainData the data the search is based on * @param random a random number generator * @throws Exception if an error occurs */ protected void findParamsByCrossValidation(int depth, Instances trainData, Random random) throws Exception { if (benchmark) { long start = System.currentTimeMillis(); // ensure exhaustive has unmodified parameters Vector parmsBkup = (Vector)m_CVParams.clone(); if (m_Debug) System.out.println("Benchmark Stepper search..."); m_BestPerformance = -99; stepperByCrossValidation(trainData,random); m_CVParams = parmsBkup; if (m_cv) cvElapsedStepper += System.currentTimeMillis() - start; else { elapsedStepper = System.currentTimeMillis() - start; if (benchmark) { m_StepperBestPerformance = m_BestPerformance; m_StepperBestClassifierOptions = m_BestClassifierOptions.clone(); } } m_BestPerformance = -99; start = System.currentTimeMillis(); if (m_Debug) System.out.println("Benchmark exhaustive search..."); paramsByCrossValidation(depth,trainData,random); if (m_cv) cvElapsedExhaustive += System.currentTimeMillis() - start; else elapsedExhaustive = System.currentTimeMillis() - start; if (!m_cv) { originalOptimalHash = Arrays.deepHashCode(m_BestClassifierOptions); } else { int optimalHash = Arrays.deepHashCode(m_BestClassifierOptions); if (optimalHash != originalOptimalHash && !optimals.containsKey(optimalHash)) { optimals.put(optimalHash,m_BestPerformance); optimalOptions.put(optimalHash,m_BestClassifierOptions.clone()); } } } else { if (stepper) { if (m_Debug) System.out.println("Stepper search..."); stepperByCrossValidation(trainData,random); } else { if (m_Debug) System.out.println("Exhaustive search..."); paramsByCrossValidation(depth,trainData,random); } } } private void stepperByCrossValidation(Instances trainData,Random random) throws Exception { /* Initialize parameter values to lower bounds */ int attemptsLeft = TRIES; for (int i = 0; i < m_CVParams.size(); i++) { StepperCVParameter cvParam = (StepperCVParameter)m_CVParams.elementAt(i); cvParam.m_ParamValue = cvParam.m_Lower; /* Set parameter increment */ double upper; switch ((int)(cvParam.m_Lower - cvParam.m_Upper + 0.5)) { case 1: upper = m_NumAttributes; break; case 2: upper = m_TrainFoldSize; break; default: upper = cvParam.m_Upper; break; } if (cvParam.m_operator == '+') cvParam.m_increment = (upper - cvParam.m_Lower) / (cvParam.m_Steps - 1); else if (cvParam.m_operator == '*') { cvParam.m_increment = cvParam.m_Steps; } m_CVParams.set(i,cvParam); } /* Do evaluation to establish initial state */ double currentError = evaluateStep(trainData); m_BestPerformance = currentError; m_BestClassifierOptions = createOptions(); if (benchmark) searches.put(Arrays.deepHashCode(m_StepperBestClassifierOptions),m_StepperBestPerformance); double improvement = Double.MAX_VALUE; double increment = 0d; StepperCVParameter lastCVParam = null; int lastCVParmIdx = -1; int lastResortIdx = 0; /** * Should keep attemptsLeft for each index so if we hit one in a section * Section: normal, best idx, last improved, last resort idx * in a section that is a already tried config we can decrement it's unique * attemptsLeft so we correctly figure next increment and don't loop? */ while (improvement > epsilon || attemptsLeft > 0) { double bestImp = 0d; int bestImpIdx = -1; StepperCVParameter cvParam = null; for (int i = 0;i < m_CVParams.size(); i++) { cvParam = (StepperCVParameter)m_CVParams.elementAt(i); if (cvParam.m_ParamValue < cvParam.m_Upper) { if (cvParam.m_operator == '+') cvParam.m_ParamValue += cvParam.m_increment; else { cvParam.m_ParamValue *= cvParam.m_increment; } } else continue; m_CVParams.set(i,cvParam); double error = evaluateStep(trainData); if (false) { System.err.print("Checking options for " + m_Classifier.getClass().getName() + ":"); String[] options = createOptions(); for (int j = 0; j < options.length; j++) { System.err.print(" " + options[j]); } System.err.println(" " + Utils.doubleToString(error,6,4)); } improvement = currentError - error; if (currentError <= error || improvement <= epsilon) { if (cvParam.m_operator == '+') cvParam.m_ParamValue -= cvParam.m_increment; else { cvParam.m_ParamValue /= cvParam.m_increment; } m_CVParams.set(i,cvParam); if (improvement > bestImp) { bestImp = improvement; bestImpIdx = i; } } else { currentError = error; if (benchmark) searches.put(Arrays.deepHashCode(createOptions()),currentError); lastCVParam = cvParam; lastCVParmIdx = i; if (m_cv) cvNumSteps++; else numSteps++; } } if (improvement <= epsilon) { /* * This should do less searching than exhaustive. * The biggest concern I think would be that it does too little * searching. * I am considering augmenting that in two ways... * 1) Up to TRIES times attempt steps in the direction (attribute * increment) that had best improvement (although less than epsilon). * 2) If none have shown improvement, try a step (increment) of the last * attribute that was previously successfully incremented. * (TRIES should still apply). * 3) If still none just try incrementing the first parameter */ boolean success = false; double error = 0d; if (bestImp > 0) { // was there any improvement at all? cvParam = (StepperCVParameter)m_CVParams.elementAt(bestImpIdx); double inc; if (cvParam.m_operator == '+') { inc = cvParam.m_increment; if (cvParam.m_ParamValue + inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue += inc; } else { inc = cvParam.m_increment * Math.pow(cvParam.m_increment,TRIES-attemptsLeft+1); if (cvParam.m_ParamValue * inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue *= inc; } m_CVParams.set(bestImpIdx,cvParam); currentError -= bestImp; error = evaluateStep(trainData); improvement = currentError - error; if (currentError <= error || improvement <= epsilon) { if (cvParam.m_operator == '+') cvParam.m_ParamValue -= inc; else { cvParam.m_ParamValue /= inc; } m_CVParams.set(bestImpIdx,cvParam); attemptsLeft--; } else { currentError = error; if (benchmark) searches.put(Arrays.deepHashCode(createOptions()),currentError); lastCVParam = cvParam; lastCVParmIdx = bestImpIdx; if (m_cv) cvNumSteps++; else numSteps++; } } else if (lastCVParam != null) { // attempt step in last successful direction while (attemptsLeft > 0 && !success) { cvParam = m_CVParams.get(lastCVParmIdx); double inc; if (cvParam.m_operator == '+') { inc = cvParam.m_increment; if (cvParam.m_ParamValue + inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue += inc; } else { inc = cvParam.m_increment * Math.pow(cvParam.m_increment,TRIES-attemptsLeft+1); if (cvParam.m_ParamValue * inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue *= inc; } m_CVParams.set(lastCVParmIdx,cvParam); error = evaluateStep(trainData); improvement = currentError - error; if (currentError <= error || improvement <= epsilon) { if (cvParam.m_operator == '+') cvParam.m_ParamValue -= inc; else { cvParam.m_ParamValue /= inc; } m_CVParams.set(lastCVParmIdx,cvParam); attemptsLeft--; } else success = true; } if (success) { currentError = error; if (benchmark) searches.put(Arrays.deepHashCode(createOptions()),currentError); if (m_cv) cvNumSteps++; else numSteps++; if (improvement > bestImp) { bestImp = improvement; bestImpIdx = lastCVParmIdx; } } } else { /** * This could maybe go all the way through each param * trying each increment? A little less arbitrary than * just index 0. Although might need to get away from the * TRIES idea? * * We need to keep looping this until we give up or reach success */ while (attemptsLeft > 0 && !success) { cvParam = (StepperCVParameter)m_CVParams.elementAt(0); double inc; if (cvParam.m_operator == '+') { inc = cvParam.m_increment; if (cvParam.m_ParamValue + inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue += inc; } else { inc = cvParam.m_increment * Math.pow(cvParam.m_increment,TRIES-attemptsLeft+1); if (cvParam.m_ParamValue * inc > cvParam.m_Upper) { attemptsLeft = 0; lastCVParam = null; continue; } cvParam.m_ParamValue *= inc; } m_CVParams.set(0,cvParam); error = evaluateStep(trainData); improvement = currentError - error; if (currentError <= error || improvement <= epsilon) { if (cvParam.m_operator == '+') cvParam.m_ParamValue -= inc; else { cvParam.m_ParamValue /= inc; } m_CVParams.set(0,cvParam); attemptsLeft--; } else success = true; } if (success) { // Success! managed to get some traction currentError = error; bestImp = improvement; bestImpIdx = 0; if (benchmark) searches.put(Arrays.deepHashCode(createOptions()),currentError); if (m_cv) cvNumSteps++; else numSteps++; } } } } if (m_Debug) { System.err.println("Cross-validated error rate: " + Utils.doubleToString(currentError, 6, 4)); } if ((m_BestPerformance == -99) || (currentError < m_BestPerformance)) { m_BestPerformance = currentError; m_BestClassifierOptions = createOptions(); } // System.out.println("number of steps " + numSteps + " evaluations " + stepperEvaluations); } private double evaluateStep(Instances trainData) throws Exception { if (m_cv) cvStepperEvaluations++; else stepperEvaluations++; Evaluation evaluation = new Evaluation(trainData); String [] options = createOptions(); if (m_Debug) { System.err.print("Setting options for " + m_Classifier.getClass().getName() + ":"); for (int i = 0; i < options.length; i++) { System.err.print(" " + options[i]); } // System.err.println(""); } ((OptionHandler)m_Classifier).setOptions(options); for (int j = 0; j < m_NumFolds; j++) { // We want to randomize the data the same way for every // learning scheme. Instances train = trainData.trainCV(m_NumFolds, j, new Random(1)); Instances test = trainData.testCV(m_NumFolds, j); m_Classifier.buildClassifier(train); evaluation.setPriors(train); evaluation.evaluateModel(m_Classifier, test); } if (m_Debug) { System.err.println(" " + Utils.doubleToString(evaluation.errorRate(),6,4)); } return evaluation.errorRate(); } private void paramsByCrossValidation(int depth, Instances trainData, Random random) throws Exception { if (depth < m_CVParams.size()) { StepperCVParameter cvParam = (StepperCVParameter)m_CVParams.elementAt(depth); double upper; switch ((int)(cvParam.m_Lower - cvParam.m_Upper + 0.5)) { case 1: upper = m_NumAttributes; break; case 2: upper = m_TrainFoldSize; break; default: upper = cvParam.m_Upper; break; } double increment = 0d; if (cvParam.m_operator == '+') increment = (upper - cvParam.m_Lower) / (cvParam.m_Steps - 1); else { increment = cvParam.m_Steps; } if (cvParam.m_operator == '+') { for(cvParam.m_ParamValue = cvParam.m_Lower; cvParam.m_ParamValue <= upper; cvParam.m_ParamValue += increment) { paramsByCrossValidation(depth + 1, trainData, random); } } else { for(cvParam.m_ParamValue = cvParam.m_Lower; cvParam.m_ParamValue <= upper; cvParam.m_ParamValue *= increment) { paramsByCrossValidation(depth + 1, trainData, random); } } } else { if (m_cv) cvExhaustiveEvaluations++; else exhaustiveEvaluations++; Evaluation evaluation = new Evaluation(trainData); // Set the classifier options String [] options = createOptions(); if (m_Debug) { System.err.print("Setting options for " + m_Classifier.getClass().getName() + ":"); for (int i = 0; i < options.length; i++) { System.err.print(" " + options[i]); } // System.err.println(""); } ((OptionHandler)m_Classifier).setOptions(options); for (int j = 0; j < m_NumFolds; j++) { // We want to randomize the data the same way for every // learning scheme. Instances train = trainData.trainCV(m_NumFolds, j, new Random(1)); Instances test = trainData.testCV(m_NumFolds, j); m_Classifier.buildClassifier(train); evaluation.setPriors(train); evaluation.evaluateModel(m_Classifier, test); } double error = evaluation.errorRate(); if (m_Debug) { // System.err.println("Cross-validated error rate: " // + Utils.doubleToString(error, 6, 4)); System.err.println(" " + Utils.doubleToString(error,6,4)); } if ((m_BestPerformance == -99) || (error < m_BestPerformance)) { m_BestPerformance = error; m_BestClassifierOptions = createOptions(); } } } /* From StackOverflow */ private static String formatInterval(final long l) { final long hr = TimeUnit.MILLISECONDS.toHours(l); final long min = TimeUnit.MILLISECONDS.toMinutes(l - TimeUnit.HOURS.toMillis(hr)); final long sec = TimeUnit.MILLISECONDS.toSeconds(l - TimeUnit.HOURS.toMillis(hr) - TimeUnit.MINUTES.toMillis(min)); final long ms = TimeUnit.MILLISECONDS.toMillis(l - TimeUnit.HOURS.toMillis(hr) - TimeUnit.MINUTES.toMillis(min) - TimeUnit.SECONDS.toMillis(sec)); return String.format("%02d:%02d:%02d.%03d", hr, min, sec, ms); } /** * Generates the classifier. * * @param instances set of instances serving as training data * @throws Exception if the classifier has not been generated successfully */ public void buildClassifier(Instances instances) throws Exception { if (cvFoldNum == 0) { m_cv = isCrossValidated(); if (m_cv) cvFoldNum++; } else { m_cv = true; cvFoldNum++; // is CV bump the fold number static } // can classifier handle the data? getCapabilities().testWithFail(instances); // remove instances with missing class Instances trainData = new Instances(instances); trainData.deleteWithMissingClass(); if (!(m_Classifier instanceof OptionHandler)) { throw new IllegalArgumentException("Base classifier should be OptionHandler."); } m_InitOptions = ((OptionHandler)m_Classifier).getOptions(); // m_BestPerformance = -99; m_NumAttributes = trainData.numAttributes(); Random random = new Random(m_Seed); trainData.randomize(random); m_TrainFoldSize = trainData.trainCV(m_NumFolds, 0).numInstances(); // Check whether there are any parameters to optimize if (m_CVParams.size() == 0) { m_Classifier.buildClassifier(trainData); m_BestClassifierOptions = m_InitOptions; return; } if (trainData.classAttribute().isNominal()) { trainData.stratify(m_NumFolds); } m_BestClassifierOptions = null; // Set up m_ClassifierOptions -- take getOptions() and remove // those being optimised. m_ClassifierOptions = ((OptionHandler)m_Classifier).getOptions(); for (int i = 0; i < m_CVParams.size(); i++) { Utils.getOption(((StepperCVParameter)m_CVParams.elementAt(i)).m_ParamChar, m_ClassifierOptions); } findParamsByCrossValidation(0, trainData, random); String[] options = (String [])m_BestClassifierOptions.clone(); ((OptionHandler)m_Classifier).setOptions(options); m_Classifier.buildClassifier(trainData); if (benchmark) { if (m_cv && cvFoldNum == numFolds) { System.out.println("== Benchmark Information (total CV) =="); // totalSteps += numSteps; System.out.println("--STEPPER--"); System.out.println("Elapsed: " + formatInterval(cvElapsedStepper)); System.out.println("CV number of evaluations: " + cvStepperEvaluations); System.out.println("CV Number of steps taken: " + cvNumSteps); // System.out.println("Result: " + Utils.doubleToString(m_StepperBestPerformance,6,4)); // System.out.print("Options: "); // for (int i = 0; i < m_StepperBestClassifierOptions.length; i++) { // System.out.print(" " + m_BestClassifierOptions[i]); // } // System.out.println(""); System.out.println("--EXHAUSTIVE--"); System.out.println("CV Elapsed: " + formatInterval(cvElapsedExhaustive)); System.out.println("CV Number of evaluations: " + cvExhaustiveEvaluations); System.out.println("CV Result: " + Utils.doubleToString(m_BestPerformance,6,4)); System.out.print("Options: "); for (int i = 0; i < m_BestClassifierOptions.length; i++) { System.out.print(" " + m_BestClassifierOptions[i]); } System.out.println(""); System.out.println("Alternate optimal: " + optimals.size()); for (Integer optKey : optimalOptions.keySet()) { String[] optOption = optimalOptions.get(optKey); System.out.print(" "); for (int i = 0; i < optOption.length; i++) { System.out.print(" " + optOption[i]); } System.out.println(" (" + Utils.doubleToString(optimals.get(optKey),6,4) + ")"); } System.out.println(""); } else if (!m_cv) { optimalSearch = searches.containsKey(Arrays.deepHashCode(m_BestClassifierOptions)); System.out.println("== Benchmark Information (non-CV) =="); System.out.println("--STEPPER--"); System.out.println("Elapsed: " + formatInterval(elapsedStepper)); System.out.println("Number of evaluations: " + stepperEvaluations); System.out.println("Total Number of steps taken: " + numSteps); System.out.println("Result: " + Utils.doubleToString(m_StepperBestPerformance,6,4)); System.out.print("Options: "); for (int i = 0; i < m_BestClassifierOptions.length; i++) { System.out.print(" " + m_StepperBestClassifierOptions[i]); } System.out.println(""); System.out.println("Optimal Search: " + optimalSearch); System.out.println("--EXHAUSTIVE--"); System.out.println("Elapsed: " + formatInterval(elapsedExhaustive)); System.out.println("Number of evaluations: " + exhaustiveEvaluations); System.out.println("Result: " + Utils.doubleToString(m_BestPerformance,6,4)); System.out.print("Options: "); for (int i = 0; i < m_BestClassifierOptions.length; i++) { System.out.print(" " + m_BestClassifierOptions[i]); } System.out.println(""); } } } /** * Adds a scheme parameter to the list of parameters to be set * by cross-validation * * @param cvParam the string representation of a scheme parameter. The * format is:
* param_char lower_bound upper_bound number_of_steps
* eg to search a parameter -P from 1 to 10 by increments of 1:
* P 1 10 11
* @throws Exception if the parameter specifier is of the wrong format */ public void addCVParameter(String cvParam) throws Exception { StepperCVParameter newCV = new StepperCVParameter(cvParam); m_CVParams.addElement(newCV); } /** * Parses a given list of options.

* * Valid options are:

* * * 

 -E
   *  If set, the normal exhaustive CVParameterSelection search is done 
   *  may be used in conjunction with -B for benchmarking
* * 
 -B
   *  If set, benchmarking information is output (Elapsed, number of evals).
   * 
   
   *
   * Options after -- are passed to the designated sub-classifier. 

   *
   * @param options the list of options as an array of strings
   * @throws Exception if an option is not supported
   */
  public void setOptions(String[] options) throws Exception {
    for (int i=0;i();
     optimalOptions = new HashMap();
     cvElapsedStepper = 0;
     cvElapsedExhaustive = 0;
     cvStepperEvaluations = 0;
     cvExhaustiveEvaluations = 0;
     cvNumSteps = 0;
  }
                
  /**
   * Main method for testing this class.
   *
   * @param argv the options
   */
  public static void main(String [] argv) {
    cleanCV();
    runClassifier(new CVStepper(), argv);
  }	
}