Mercurial > repos > bgruening > sklearn_model_validation

--- a/main_macros.xml	Tue Jul 10 03:13:16 2018 -0400
+++ b/main_macros.xml	Fri Jul 13 03:56:45 2018 -0400
@@ -35,7 +35,8 @@
     if not options['threshold'] or options['threshold'] == 'None':
       options['threshold'] = None
       if 'extra_estimator' in inputs and inputs['extra_estimator']['has_estimator'] == 'no_load':
-        fitted_estimator = pickle.load(open("inputs['extra_estimator']['fitted_estimator']", 'r'))
+        with open("inputs['extra_estimator']['fitted_estimator']", 'rb') as model_handler:
+          fitted_estimator = pickle.load(model_handler)
         new_selector = selector(fitted_estimator, prefit=True, **options)
       else:
         estimator=inputs["estimator"]
@@ -83,7 +84,7 @@
       parse_dates=True
     )
   else:
-    X = mmread(open(file1, 'r'))
+    X = mmread(file1)

   header = 'infer' if params["selected_tasks"]["selected_algorithms"]["input_options"]["header2"] else None
   column_option = params["selected_tasks"]["selected_algorithms"]["input_options"]["column_selector_options_2"]["selected_column_selector_option2"]
@@ -432,19 +433,6 @@


   <!--Data interface-->
-  <xml name="tabular_input">
-    <param name="infile" type="data" format="tabular" label="Data file with numeric values"/>
-    <param name="start_column" type="data_column" data_ref="infile" optional="True" label="Select a subset of data. Start column:" />
-    <param name="end_column" type="data_column" data_ref="infile" optional="True" label="End column:" />
-  </xml>
-
-  <xml name="sample_cols" token_label1="File containing true class labels:" token_label2="File containing predicted class labels:" token_multiple1="False" token_multiple2="False" token_format1="tabular" token_format2="tabular" token_help1="" token_help2="">
-    <param name="infile1" type="data" format="@FORMAT1@" label="@LABEL1@" help="@HELP1@"/>
-    <param name="col1" multiple="@MULTIPLE1@" type="data_column" data_ref="infile1" label="Select target column(s):"/>
-    <param name="infile2" type="data" format="@FORMAT2@" label="@LABEL2@" help="@HELP2@"/>
-    <param name="col2" multiple="@MULTIPLE2@" type="data_column" data_ref="infile2" label="Select target column(s):"/>
-    <yield/>
-  </xml>

   <xml name="samples_tabular" token_multiple1="false" token_multiple2="false">
     <param name="infile1" type="data" format="tabular" label="Training samples dataset:"/>
@@ -472,13 +460,13 @@
       <param name="@COL_NAME@" multiple="@MULTIPLE@" type="data_column" data_ref="@INFILE@" label="Select target column(s):"/>
     </when>
     <when value="by_header_name">
-      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="String seperate by colon. For example: target1,target2"/>
+      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="Comma-separated string. For example: target1,target2"/>
     </when>
     <when value="all_but_by_index_number">
       <param name="@COL_NAME@" multiple="@MULTIPLE@" type="data_column" data_ref="@INFILE@" label="Select target column(s):"/>
     </when>
     <when value="all_but_by_header_name">
-      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="String seperate by colon. For example: target1,target2"/>
+      <param name="@COL_NAME@" type="text" value="" label="Type header name(s):" help="Comma-separated string. For example: target1,target2"/>
     </when>
     <when value="all_columns">
     </when>
@@ -553,11 +541,6 @@
     </conditional>
   </xml>

-  <xml name="multitype_input" token_format="tabular" token_help="All datasets with tabular format are supporetd.">
-    <param name="infile_transform" type="data" format="@FORMAT@" label="Select a dataset to transform:" help="@HELP@"/>
-  </xml>
-
-
   <!--Advanced options-->
   <xml name="nn_advanced_options">
     <section name="options" title="Advanced Options" expanded="False">
@@ -822,9 +805,17 @@
     </param>
   </xml>

+  <xml name="sparse_preprocessors_ext">
+    <expand macro="sparse_preprocessors">
+      <option value="KernelCenterer">Kernel Centerer (Centers a kernel matrix)</option>
+      <option value="MinMaxScaler">Minmax Scaler (Scales features to a range)</option>
+      <option value="PolynomialFeatures">Polynomial Features (Generates polynomial and interaction features)</option>
+      <option value="RobustScaler">Robust Scaler (Scales features using outlier-invariance statistics)</option>
+    </expand>
+  </xml>
+
   <xml name="sparse_preprocessor_options">
     <when value="Binarizer">
-        <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
         <section name="options" title="Advanced Options" expanded="False">
             <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
                 label="Use a copy of data for precomputing binarization" help=" "/>
@@ -834,7 +825,6 @@
         </section>
     </when>
     <when value="Imputer">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
           <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for precomputing imputation" help=" "/>
@@ -854,7 +844,6 @@
       </section>
     </when>
     <when value="StandardScaler">
-      <expand macro="multitype_input"/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for performing inplace scaling" help=" "/>
@@ -865,14 +854,12 @@
       </section>
     </when>
     <when value="MaxAbsScaler">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="true"
             label="Use a copy of data for precomputing scaling" help=" "/>
       </section>
     </when>
     <when value="Normalizer">
-      <expand macro="multitype_input" format="tabular,txt" help="Tabular and sparse datasets are supporetd."/>
       <section name="options" title="Advanced Options" expanded="False">
         <param argument="norm" type="select" optional="true" label="The norm to use to normalize non zero samples" help=" ">
           <option value="l1" selected="true">l1</option>
@@ -885,6 +872,41 @@
     </when>
     <yield/>
   </xml>
+
+  <xml name="sparse_preprocessor_options_ext">
+    <expand macro="sparse_preprocessor_options">
+      <when value="KernelCenterer">
+        <section name="options" title="Advanced Options" expanded="False">
+        </section>
+      </when>
+      <when value="MinMaxScaler">
+          <section name="options" title="Advanced Options" expanded="False">
+              <!--feature_range-->
+              <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Use a copy of data for precomputing normalization" help=" "/>
+          </section>
+      </when>
+      <when value="PolynomialFeatures">
+          <section name="options" title="Advanced Options" expanded="False">
+              <param argument="degree" type="integer" optional="true" value="2" label="The degree of the polynomial features " help=""/>
+              <param argument="interaction_only" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="false" label="Produce interaction features only" help="(Features that are products of at most degree distinct input features) "/>
+              <param argument="include_bias" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true" label="Include a bias column" help="Feature in which all polynomial powers are zero "/>
+          </section>
+      </when>
+      <when value="RobustScaler">
+          <section name="options" title="Advanced Options" expanded="False">
+              <!--=True, =True, copy=True-->
+              <param argument="with_centering" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Center the data before scaling" help=" "/>
+              <param argument="with_scaling" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Scale the data to interquartile range" help=" "/>
+              <param argument="copy" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolflase" checked="true"
+                  label="Use a copy of data for inplace scaling" help=" "/>
+          </section>
+      </when>
+    </expand>
+  </xml>
+
   <xml name="estimator_input_no_fit">
     <expand macro="feature_selection_estimator" />
     <conditional name="extra_estimator">
@@ -892,6 +914,7 @@
       <expand macro="feature_selection_estimator_choices" />
     </conditional>
   </xml>
+
   <xml name="feature_selection_all">
     <conditional name="feature_selection_algorithms">
       <param name="selected_algorithm" type="select" label="Select a feature selection algorithm">
@@ -1014,6 +1037,7 @@
       </when-->
     </conditional>
   </xml>
+
   <xml name="feature_selection_score_function">
     <param argument="score_func" type="select" label="Select a score function">
       <option value="chi2">chi2 - Compute chi-squared stats between each non-negative feature and class</option>
@@ -1023,6 +1047,7 @@
       <option value="mutual_info_regression">mutual_info_regression - Estimate mutual information for a continuous target variable</option>
     </param>
   </xml>
+
   <xml name="feature_selection_estimator">
     <param argument="estimator" type="select" label="Select an estimator" help="The base estimator from which the transformer is built.">
       <option value="svm.SVR(kernel=&quot;linear&quot;)">svm.SVR(kernel=&quot;linear&quot;)</option>
@@ -1032,6 +1057,7 @@
       <option value="ensemble.RandomForestRegressor(n_estimators = 1000, random_state = 42)">ensemble.RandomForestRegressor(n_estimators = 1000, random_state = 42)</option>
     </param>
   </xml>
+
   <xml name="feature_selection_extra_estimator">
       <param name="has_estimator" type="select" label="Does your estimator on the list above?">
         <option value="yes">Yes, my estimator is on the list</option>
@@ -1039,6 +1065,7 @@
         <yield/>
       </param>
   </xml>
+
   <xml name="feature_selection_estimator_choices">
     <when value="yes">
     </when>
@@ -1047,6 +1074,7 @@
     </when>
     <yield/>
   </xml>
+
   <xml name="feature_selection_methods">
     <conditional name="select_methods">
       <param name="selected_method" type="select" label="Select an operation">
--- a/model_validation.xml	Tue Jul 10 03:13:16 2018 -0400
+++ b/model_validation.xml	Fri Jul 13 03:56:45 2018 -0400
@@ -22,7 +22,7 @@
 import pickle
 import numpy as np
 import sklearn.model_selection
-from sklearn import svm, linear_model, ensemble
+from sklearn import svm, linear_model, ensemble, preprocessing
 from sklearn.pipeline import Pipeline

 @COLUMNS_FUNCTION@
@@ -30,7 +30,8 @@
 @FEATURE_SELECTOR_FUNCTION@

 input_json_path = sys.argv[1]
-params = json.load(open(input_json_path, "r"))
+with open(input_json_path, "r") as param_handler:
+    params = json.load(param_handler)

 input_type = params["input_options"]["selected_input"]
 if input_type=="tabular":
@@ -49,7 +50,7 @@
             parse_dates=True
     )
 else:
-    X = mmread(open("$input_options.infile1", 'r'))
+    X = mmread("$input_options.infile1")

 header = 'infer' if params["input_options"]["header2"] else None
 column_option = params["input_options"]["column_selector_options_2"]["selected_column_selector_option2"]
@@ -75,10 +76,17 @@

 pipeline_steps = []

+## Set up pre_processor and add to pipeline steps.
+if params['pre_processing']['do_pre_processing'] == 'Yes':
+    preprocessor = params["pre_processing"]["pre_processors"]["selected_pre_processor"]
+    pre_processor_options = params["pre_processing"]["pre_processors"]["options"]
+    my_class = getattr(preprocessing, preprocessor)
+    pipeline_steps.append( ('pre_processor', my_class(**pre_processor_options)) )
+
 ## Set up feature selector and add to pipeline steps.
 if params['feature_selection']['do_feature_selection'] == 'Yes':
     feature_selector = feature_selector(params['feature_selection']['feature_selection_algorithms'])
-    pipeline_steps.append( ('feature_selector', feature_selector))
+    pipeline_steps.append( ('feature_selector', feature_selector) )

 ## Set up estimator and add to pipeline.
 estimator=params["model_validation_functions"]["estimator"]
@@ -138,6 +146,19 @@
         </configfile>
     </configfiles>
     <inputs>
+        <conditional name="pre_processing">
+            <param name="do_pre_processing" type="select" label="Do pre_processing?">
+                <option value="No" selected="true"/>
+                <option value="Yes"/>
+            </param>
+            <when value="No"/>
+            <when value="Yes">
+                <conditional name="pre_processors">
+                    <expand macro="sparse_preprocessors_ext" />
+                    <expand macro="sparse_preprocessor_options_ext" />
+                </conditional>
+            </when>
+        </conditional>
         <conditional name="feature_selection">
             <param name="do_feature_selection" type="select" label="Do feature selection?">
                 <option value="No" selected="true"/>
@@ -352,7 +373,54 @@
             <param name="infile2" value="regression_y.tabular" ftype="tabular"/>
             <param name="header2" value="true" />
             <param name="selected_column_selector_option2" value="all_columns"/>
-            <output name="outfile" file="mv_result07.tabular"/>
+            <output name="outfile" >
+                <assert_contents>
+                    <has_line line="0.7824428015300172" />
+                </assert_contents>
+            </output>
+        </test>
+        <test>
+            <param name="do_pre_processing" value="Yes"/>
+            <param name="selected_pre_processor" value="RobustScaler"/>
+            <param name="do_feature_selection" value="Yes"/>
+            <param name="selected_algorithm" value="SelectKBest"/>
+            <param name="score_func" value="f_classif"/>
+            <param name="selected_function" value="GridSearchCV"/>
+            <param name="estimator" value="svm.SVR(kernel=&quot;linear&quot;)"/>
+            <param name="has_estimator" value="yes"/>
+            <param name="param_grid" value="[{'feature_selector__k': [3, 5, 7, 9], 'estimator__C': [1, 10, 100, 1000]}]"/>
+            <param name="return_type" value="best_score_"/>
+            <param name="infile1" value="regression_X.tabular" ftype="tabular"/>
+            <param name="header1" value="true" />
+            <param name="selected_column_selector_option" value="all_columns"/>
+            <param name="infile2" value="regression_y.tabular" ftype="tabular"/>
+            <param name="header2" value="true" />
+            <param name="selected_column_selector_option2" value="all_columns"/>
+            <output name="outfile" >
+                <assert_contents>
+                    <has_line line="0.7938837807353147" />
+                </assert_contents>
+            </output>
+        </test>
+         <test>
+            <param name="do_pre_processing" value="Yes"/>
+            <param name="selected_pre_processor" value="RobustScaler"/>
+            <param name="selected_function" value="GridSearchCV"/>
+            <param name="estimator" value="svm.SVR(kernel=&quot;linear&quot;)"/>
+            <param name="has_estimator" value="yes"/>
+            <param name="param_grid" value="[{'estimator__C': [1, 10, 100, 1000]}]"/>
+            <param name="return_type" value="best_score_"/>
+            <param name="infile1" value="regression_X.tabular" ftype="tabular"/>
+            <param name="header1" value="true" />
+            <param name="selected_column_selector_option" value="all_columns"/>
+            <param name="infile2" value="regression_y.tabular" ftype="tabular"/>
+            <param name="header2" value="true" />
+            <param name="selected_column_selector_option2" value="all_columns"/>
+            <output name="outfile" >
+                <assert_contents>
+                    <has_line line="0.7904476204861263" />
+                </assert_contents>
+            </output>
         </test>
     </tests>
     <help>
--- a/test-data/mv_result07.tabular	Tue Jul 10 03:13:16 2018 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-0.7824428015300172