| Previous changeset 25:68afcd163b3d (2019-07-09) Next changeset 27:5a21a631d2eb (2019-09-13) |
|
Commit message:
planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/sklearn commit 60f0fbc0eafd7c11bc60fb6c77f2937782efd8a9-dirty |
|
modified:
clf_metrics.xml main_macros.xml search_model_validation.py stacking_ensembles.py test-data/RandomForestClassifier.zip test-data/StackingCVRegressor01.zip test-data/pipeline01 test-data/pipeline02 test-data/pipeline03 test-data/pipeline04 test-data/pipeline05 test-data/pipeline06 test-data/pipeline07 test-data/pipeline08 test-data/pipeline09 test-data/pipeline10 test-data/pipeline11 test-data/pipeline12 test-data/pipeline14 test-data/pipeline15 test-data/prp_model01 test-data/prp_model02 test-data/prp_model04 test-data/prp_model05 test-data/prp_model07 test-data/prp_model08 test-data/prp_model09 test-data/searchCV01 test-data/searchCV02 |
|
added:
keras_deep_learning.py keras_macros.xml model_prediction.py test-data/StackingRegressor02.zip test-data/StackingVoting03.zip test-data/deepsear_1feature.json test-data/fitted_keras_g_regressor01.zip test-data/keras01.json test-data/keras02.json test-data/keras03.json test-data/keras04.json test-data/keras_batch_model01 test-data/keras_batch_model02 test-data/keras_batch_model03 test-data/keras_batch_params01.tabular test-data/keras_model01 test-data/keras_model02 test-data/keras_model04 test-data/keras_params04.tabular test-data/keras_prefitted01.zip test-data/keras_save_weights01.h5 test-data/model_pred01.tabular test-data/model_pred02.tabular test-data/pipeline16 test-data/prp_result10 test-data/regression_groups.tabular test-data/train_test_eval01.tabular test-data/train_test_eval03.tabular test-data/train_test_eval_model01 test-data/train_test_eval_weights01.h5 test-data/train_test_eval_weights02.h5 train_test_eval.py |
|
removed:
feature_selectors.py iraps_classifier.py model_validations.py pk_whitelist.json preprocessors.py test-data/pipeline13 utils.py |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb clf_metrics.xml --- a/clf_metrics.xml Tue Jul 09 19:36:00 2019 -0400 +++ b/clf_metrics.xml Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -21,9 +21,8 @@ import numpy as np import sys from sklearn import metrics +from galaxy_ml.utils import read_columns -sys.path.insert(0, '$__tool_directory__') -from utils import read_columns input_json_path = sys.argv[1] with open(input_json_path, "r") as param_handler: |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb feature_selectors.py --- a/feature_selectors.py Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,357 +0,0 @@\n-"""\n-DyRFE\n-DyRFECV\n-MyPipeline\n-MyimbPipeline\n-check_feature_importances\n-"""\n-import numpy as np\n-\n-from imblearn import under_sampling, over_sampling, combine\n-from imblearn.pipeline import Pipeline as imbPipeline\n-from sklearn import (cluster, compose, decomposition, ensemble,\n- feature_extraction, feature_selection,\n- gaussian_process, kernel_approximation,\n- metrics, model_selection, naive_bayes,\n- neighbors, pipeline, preprocessing,\n- svm, linear_model, tree, discriminant_analysis)\n-\n-from sklearn.base import BaseEstimator\n-from sklearn.base import MetaEstimatorMixin, clone, is_classifier\n-from sklearn.feature_selection.rfe import _rfe_single_fit, RFE, RFECV\n-from sklearn.model_selection import check_cv\n-from sklearn.metrics.scorer import check_scoring\n-from sklearn.utils import check_X_y, safe_indexing, safe_sqr\n-from sklearn.utils._joblib import Parallel, delayed, effective_n_jobs\n-\n-\n-class DyRFE(RFE):\n- """\n- Mainly used with DyRFECV\n-\n- Parameters\n- ----------\n- estimator : object\n- A supervised learning estimator with a ``fit`` method that provides\n- information about feature importance either through a ``coef_``\n- attribute or through a ``feature_importances_`` attribute.\n- n_features_to_select : int or None (default=None)\n- The number of features to select. If `None`, half of the features\n- are selected.\n- step : int, float or list, optional (default=1)\n- If greater than or equal to 1, then ``step`` corresponds to the\n- (integer) number of features to remove at each iteration.\n- If within (0.0, 1.0), then ``step`` corresponds to the percentage\n- (rounded down) of features to remove at each iteration.\n- If list, a series of steps of features to remove at each iteration.\n- Iterations stops when steps finish\n- verbose : int, (default=0)\n- Controls verbosity of output.\n-\n- """\n- def __init__(self, estimator, n_features_to_select=None, step=1,\n- verbose=0):\n- super(DyRFE, self).__init__(estimator, n_features_to_select,\n- step, verbose)\n-\n- def _fit(self, X, y, step_score=None):\n-\n- if type(self.step) is not list:\n- return super(DyRFE, self)._fit(X, y, step_score)\n-\n- # dynamic step\n- X, y = check_X_y(X, y, "csc")\n- # Initialization\n- n_features = X.shape[1]\n- if self.n_features_to_select is None:\n- n_features_to_select = n_features // 2\n- else:\n- n_features_to_select = self.n_features_to_select\n-\n- step = []\n- for s in self.step:\n- if 0.0 < s < 1.0:\n- step.append(int(max(1, s * n_features)))\n- else:\n- step.append(int(s))\n- if s <= 0:\n- raise ValueError("Step must be >0")\n-\n- support_ = np.ones(n_features, dtype=np.bool)\n- ranking_ = np.ones(n_features, dtype=np.int)\n-\n- if step_score:\n- self.scores_ = []\n-\n- step_i = 0\n- # Elimination\n- while np.sum(support_) > n_features_to_select and step_i < len(step):\n-\n- # if last step is 1, will keep loop\n- if step_i == len(step) - 1 and step[step_i] != 0:\n- step.append(step[step_i])\n-\n- # Remaining features\n- features = np.arange(n_features)[support_]\n-\n- # Rank the remaining features\n- estimator = clone(self.estimator)\n- if self.verbose > 0:\n- print("Fitting estimator with %d features." % np.sum(support_))\n-\n- estimator.fit(X[:, features], y)\n-\n- # Get coefs\n- if hasattr(estimator, \'coef_\'):\n- coefs = estimator.coef_\n- else:\n- coefs = getattr(estimator, \'feature_importances_\', None)\n- '..b' # Note that joblib raises a non-picklable error for bound methods\n- # even if n_jobs is set to 1 with the default multiprocessing\n- # backend.\n- # This branching is done so that to\n- # make sure that user code that sets n_jobs to 1\n- # and provides bound methods as scorers is not broken with the\n- # addition of n_jobs parameter in version 0.18.\n-\n- if effective_n_jobs(self.n_jobs) == 1:\n- parallel, func = list, _rfe_single_fit\n- else:\n- parallel = Parallel(n_jobs=self.n_jobs)\n- func = delayed(_rfe_single_fit)\n-\n- scores = parallel(\n- func(rfe, self.estimator, X, y, train, test, scorer)\n- for train, test in cv.split(X, y, groups))\n-\n- scores = np.sum(scores, axis=0)\n- diff = int(scores.shape[0]) - len(step)\n- if diff > 0:\n- step = np.r_[step, [step[-1]] * diff]\n- scores_rev = scores[::-1]\n- argmax_idx = len(scores) - np.argmax(scores_rev) - 1\n- n_features_to_select = max(\n- n_features - sum(step[:argmax_idx]),\n- self.min_features_to_select)\n-\n- # Re-execute an elimination with best_k over the whole set\n- rfe = DyRFE(estimator=self.estimator,\n- n_features_to_select=n_features_to_select, step=self.step,\n- verbose=self.verbose)\n-\n- rfe.fit(X, y)\n-\n- # Set final attributes\n- self.support_ = rfe.support_\n- self.n_features_ = rfe.n_features_\n- self.ranking_ = rfe.ranking_\n- self.estimator_ = clone(self.estimator)\n- self.estimator_.fit(self.transform(X), y)\n-\n- # Fixing a normalization error, n is equal to get_n_splits(X, y) - 1\n- # here, the scores are normalized by get_n_splits(X, y)\n- self.grid_scores_ = scores[::-1] / cv.get_n_splits(X, y, groups)\n- return self\n-\n-\n-class MyPipeline(pipeline.Pipeline):\n- """\n- Extend pipeline object to have feature_importances_ attribute\n- """\n- def fit(self, X, y=None, **fit_params):\n- super(MyPipeline, self).fit(X, y, **fit_params)\n- estimator = self.steps[-1][-1]\n- if hasattr(estimator, \'coef_\'):\n- coefs = estimator.coef_\n- else:\n- coefs = getattr(estimator, \'feature_importances_\', None)\n- if coefs is None:\n- raise RuntimeError(\'The estimator in the pipeline does not expose \'\n- \'"coef_" or "feature_importances_" \'\n- \'attributes\')\n- self.feature_importances_ = coefs\n- return self\n-\n-\n-class MyimbPipeline(imbPipeline):\n- """\n- Extend imblance pipeline object to have feature_importances_ attribute\n- """\n- def fit(self, X, y=None, **fit_params):\n- super(MyimbPipeline, self).fit(X, y, **fit_params)\n- estimator = self.steps[-1][-1]\n- if hasattr(estimator, \'coef_\'):\n- coefs = estimator.coef_\n- else:\n- coefs = getattr(estimator, \'feature_importances_\', None)\n- if coefs is None:\n- raise RuntimeError(\'The estimator in the pipeline does not expose \'\n- \'"coef_" or "feature_importances_" \'\n- \'attributes\')\n- self.feature_importances_ = coefs\n- return self\n-\n-\n-def check_feature_importances(estimator):\n- """\n- For pipeline object which has no feature_importances_ property,\n- this function returns the same comfigured pipeline object with\n- attached the last estimator\'s feature_importances_.\n- """\n- if estimator.__class__.__module__ == \'sklearn.pipeline\':\n- pipeline_steps = estimator.get_params()[\'steps\']\n- estimator = MyPipeline(pipeline_steps)\n- elif estimator.__class__.__module__ == \'imblearn.pipeline\':\n- pipeline_steps = estimator.get_params()[\'steps\']\n- estimator = MyimbPipeline(pipeline_steps)\n- else:\n- return estimator\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb iraps_classifier.py --- a/iraps_classifier.py Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,569 +0,0 @@\n-"""\n-class IRAPSCore\n-class IRAPSClassifier\n-class BinarizeTargetClassifier\n-class BinarizeTargetRegressor\n-class _BinarizeTargetScorer\n-class _BinarizeTargetProbaScorer\n-\n-binarize_auc_scorer\n-binarize_average_precision_scorer\n-\n-binarize_accuracy_scorer\n-binarize_balanced_accuracy_scorer\n-binarize_precision_scorer\n-binarize_recall_scorer\n-"""\n-\n-\n-import numpy as np\n-import random\n-import warnings\n-\n-from abc import ABCMeta\n-from scipy.stats import ttest_ind\n-from sklearn import metrics\n-from sklearn.base import BaseEstimator, clone, RegressorMixin\n-from sklearn.externals import six\n-from sklearn.feature_selection.univariate_selection import _BaseFilter\n-from sklearn.metrics.scorer import _BaseScorer\n-from sklearn.pipeline import Pipeline\n-from sklearn.utils import as_float_array, check_X_y\n-from sklearn.utils._joblib import Parallel, delayed\n-from sklearn.utils.validation import (check_array, check_is_fitted,\n- check_memory, column_or_1d)\n-\n-\n-VERSION = \'0.1.1\'\n-\n-\n-class IRAPSCore(six.with_metaclass(ABCMeta, BaseEstimator)):\n- """\n- Base class of IRAPSClassifier\n- From sklearn BaseEstimator:\n- get_params()\n- set_params()\n-\n- Parameters\n- ----------\n- n_iter : int\n- sample count\n-\n- positive_thres : float\n- z_score shreshold to discretize positive target values\n-\n- negative_thres : float\n- z_score threshold to discretize negative target values\n-\n- verbose : int\n- 0 or geater, if not 0, print progress\n-\n- n_jobs : int, default=1\n- The number of CPUs to use to do the computation.\n-\n- pre_dispatch : int, or string.\n- Controls the number of jobs that get dispatched during parallel\n- execution. Reducing this number can be useful to avoid an\n- explosion of memory consumption when more jobs get dispatched\n- than CPUs can process. This parameter can be:\n- - None, in which case all the jobs are immediately\n- created and spawned. Use this for lightweight and\n- fast-running jobs, to avoid delays due to on-demand\n- spawning of the jobs\n- - An int, giving the exact number of total jobs that are\n- spawned\n- - A string, giving an expression as a function of n_jobs,\n- as in \'2*n_jobs\'\n-\n- random_state : int or None\n- """\n-\n- def __init__(self, n_iter=1000, positive_thres=-1, negative_thres=0,\n- verbose=0, n_jobs=1, pre_dispatch=\'2*n_jobs\',\n- random_state=None):\n- """\n- IRAPS turns towwards general Anomaly Detection\n- It comapares positive_thres with negative_thres,\n- and decide which portion is the positive target.\n- e.g.:\n- (positive_thres=-1, negative_thres=0)\n- => positive = Z_score of target < -1\n- (positive_thres=1, negative_thres=0)\n- => positive = Z_score of target > 1\n-\n- Note: The positive targets here is always the\n- abnormal minority group.\n- """\n- self.n_iter = n_iter\n- self.positive_thres = positive_thres\n- self.negative_thres = negative_thres\n- self.verbose = verbose\n- self.n_jobs = n_jobs\n- self.pre_dispatch = pre_dispatch\n- self.random_state = random_state\n-\n- def fit(self, X, y):\n- """\n- X: array-like (n_samples x n_features)\n- y: 1-d array-like (n_samples)\n- """\n- X, y = check_X_y(X, y, [\'csr\', \'csc\'], multi_output=False)\n-\n- def _stochastic_sampling(X, y, random_state=None, positive_thres=-1,\n- negative_thres=0):\n- # each iteration select a random number of random subset of\n- # training samples. this is somewhat different from the original\n- # IRAPS method, but effect is almost the same.\n- SAMPLE_SIZE = [0.25, 0.75]\n- n_samples = X.shape[0'..b'lue = main_estimator.discretize_value\n- less_is_positive = main_estimator.less_is_positive\n-\n- if less_is_positive:\n- y_trans = y < discretize_value\n- else:\n- y_trans = y > discretize_value\n-\n- y_pred = clf.predict(X)\n- if sample_weight is not None:\n- return self._sign * self._score_func(y_trans, y_pred,\n- sample_weight=sample_weight,\n- **self._kwargs)\n- else:\n- return self._sign * self._score_func(y_trans, y_pred,\n- **self._kwargs)\n-\n-\n-# roc_auc\n-binarize_auc_scorer =\\\n- _BinarizeTargetProbaScorer(metrics.roc_auc_score, 1, {})\n-\n-# average_precision_scorer\n-binarize_average_precision_scorer =\\\n- _BinarizeTargetProbaScorer(metrics.average_precision_score, 1, {})\n-\n-# roc_auc_scorer\n-iraps_auc_scorer = binarize_auc_scorer\n-\n-# average_precision_scorer\n-iraps_average_precision_scorer = binarize_average_precision_scorer\n-\n-\n-class BinarizeTargetRegressor(BaseEstimator, RegressorMixin):\n- """\n- Extend regression estimator to have discretize_value\n-\n- Parameters\n- ----------\n- regressor: object\n- Estimator object such as derived from sklearn `RegressionMixin`.\n-\n- z_score: float, default=-1.0\n- Threshold value based on z_score. Will be ignored when\n- fixed_value is set\n-\n- value: float, default=None\n- Threshold value\n-\n- less_is_positive: boolean, default=True\n- When target is less the threshold value, it will be converted\n- to True, False otherwise.\n-\n- Attributes\n- ----------\n- regressor_: object\n- Fitted regressor\n-\n- discretize_value: float\n- The threshold value used to discretize True and False targets\n- """\n-\n- def __init__(self, regressor, z_score=-1, value=None,\n- less_is_positive=True):\n- self.regressor = regressor\n- self.z_score = z_score\n- self.value = value\n- self.less_is_positive = less_is_positive\n-\n- def fit(self, X, y, sample_weight=None):\n- """\n- Calculate the discretize_value fit the regressor with traning data\n-\n- Returns\n- ------\n- self: object\n- """\n- y = check_array(y, accept_sparse=False, force_all_finite=True,\n- ensure_2d=False, dtype=\'numeric\')\n- y = column_or_1d(y)\n-\n- if self.value is None:\n- discretize_value = y.mean() + y.std() * self.z_score\n- else:\n- discretize_value = self.Value\n- self.discretize_value = discretize_value\n-\n- self.regressor_ = clone(self.regressor)\n-\n- if sample_weight is not None:\n- self.regressor_.fit(X, y, sample_weight=sample_weight)\n- else:\n- self.regressor_.fit(X, y)\n-\n- # attach classifier attributes\n- if hasattr(self.regressor_, \'feature_importances_\'):\n- self.feature_importances_ = self.regressor_.feature_importances_\n- if hasattr(self.regressor_, \'coef_\'):\n- self.coef_ = self.regressor_.coef_\n- if hasattr(self.regressor_, \'n_outputs_\'):\n- self.n_outputs_ = self.regressor_.n_outputs_\n- if hasattr(self.regressor_, \'n_features_\'):\n- self.n_features_ = self.regressor_.n_features_\n-\n- return self\n-\n- def predict(self, X):\n- """Predict target value of X\n- """\n- check_is_fitted(self, \'regressor_\')\n- y_pred = self.regressor_.predict(X)\n- if not np.all((y_pred >= 0) & (y_pred <= 1)):\n- y_pred = (y_pred - y_pred.min()) / (y_pred.max() - y_pred.min())\n- if self.less_is_positive:\n- y_pred = 1 - y_pred\n- return y_pred\n-\n-\n-# roc_auc_scorer\n-regression_auc_scorer = binarize_auc_scorer\n-\n-# average_precision_scorer\n-regression_average_precision_scorer = binarize_average_precision_scorer\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb keras_deep_learning.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/keras_deep_learning.py Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| b'@@ -0,0 +1,359 @@\n+import argparse\n+import json\n+import keras\n+import pandas as pd\n+import pickle\n+import six\n+import warnings\n+\n+from ast import literal_eval\n+from keras.models import Sequential, Model\n+from galaxy_ml.utils import try_get_attr, get_search_params\n+\n+\n+def _handle_shape(literal):\n+ """Eval integer or list/tuple of integers from string\n+\n+ Parameters:\n+ -----------\n+ literal : str.\n+ """\n+ literal = literal.strip()\n+ if not literal:\n+ return None\n+ try:\n+ return literal_eval(literal)\n+ except NameError as e:\n+ print(e)\n+ return literal\n+\n+\n+def _handle_regularizer(literal):\n+ """Construct regularizer from string literal\n+\n+ Parameters\n+ ----------\n+ literal : str. E.g. \'(0.1, 0)\'\n+ """\n+ literal = literal.strip()\n+ if not literal:\n+ return None\n+\n+ l1, l2 = literal_eval(literal)\n+\n+ if not l1 and not l2:\n+ return None\n+\n+ if l1 is None:\n+ l1 = 0.\n+ if l2 is None:\n+ l2 = 0.\n+\n+ return keras.regularizers.l1_l2(l1=l1, l2=l2)\n+\n+\n+def _handle_constraint(config):\n+ """Construct constraint from galaxy tool parameters.\n+ Suppose correct dictionary format\n+\n+ Parameters\n+ ----------\n+ config : dict. E.g.\n+ "bias_constraint":\n+ {"constraint_options":\n+ {"max_value":1.0,\n+ "min_value":0.0,\n+ "axis":"[0, 1, 2]"\n+ },\n+ "constraint_type":\n+ "MinMaxNorm"\n+ }\n+ """\n+ constraint_type = config[\'constraint_type\']\n+ if constraint_type == \'None\':\n+ return None\n+\n+ klass = getattr(keras.constraints, constraint_type)\n+ options = config.get(\'constraint_options\', {})\n+ if \'axis\' in options:\n+ options[\'axis\'] = literal_eval(options[\'axis\'])\n+\n+ return klass(**options)\n+\n+\n+def _handle_lambda(literal):\n+ return None\n+\n+\n+def _handle_layer_parameters(params):\n+ """Access to handle all kinds of parameters\n+ """\n+ for key, value in six.iteritems(params):\n+ if value == \'None\':\n+ params[key] = None\n+ continue\n+\n+ if type(value) in [int, float, bool]\\\n+ or (type(value) is str and value.isalpha()):\n+ continue\n+\n+ if key in [\'input_shape\', \'noise_shape\', \'shape\', \'batch_shape\',\n+ \'target_shape\', \'dims\', \'kernel_size\', \'strides\',\n+ \'dilation_rate\', \'output_padding\', \'cropping\', \'size\',\n+ \'padding\', \'pool_size\', \'axis\', \'shared_axes\']:\n+ params[key] = _handle_shape(value)\n+\n+ elif key.endswith(\'_regularizer\'):\n+ params[key] = _handle_regularizer(value)\n+\n+ elif key.endswith(\'_constraint\'):\n+ params[key] = _handle_constraint(value)\n+\n+ elif key == \'function\': # No support for lambda/function eval\n+ params.pop(key)\n+\n+ return params\n+\n+\n+def get_sequential_model(config):\n+ """Construct keras Sequential model from Galaxy tool parameters\n+\n+ Parameters:\n+ -----------\n+ config : dictionary, galaxy tool parameters loaded by JSON\n+ """\n+ model = Sequential()\n+ input_shape = _handle_shape(config[\'input_shape\'])\n+ layers = config[\'layers\']\n+ for layer in layers:\n+ options = layer[\'layer_selection\']\n+ layer_type = options.pop(\'layer_type\')\n+ klass = getattr(keras.layers, layer_type)\n+ other_options = options.pop(\'layer_options\', {})\n+ options.update(other_options)\n+\n+ # parameters needs special care\n+ options = _handle_layer_parameters(options)\n+\n+ # add input_shape to the first layer only\n+ if not getattr(model, \'_layers\') and input_shape is not None:\n+ options[\'input_shape\'] = input_shape\n+\n+ model.add(klass(**options))\n+\n+ return model\n+\n+\n+def get_functional_model(config):\n+ """Construct keras functional model from Galaxy tool parameters\n+\n+ Parameters\n+ -'..b'\'Sequential\':\n+ options[\'model_type\'] = \'sequential\'\n+ klass = Sequential\n+ elif json_model[\'class_name\'] == \'Model\':\n+ options[\'model_type\'] = \'functional\'\n+ klass = Model\n+ else:\n+ raise ValueError("Unknow Keras model class: %s"\n+ % json_model[\'class_name\'])\n+\n+ # load prefitted model\n+ if inputs[\'mode_selection\'][\'mode_type\'] == \'prefitted\':\n+ estimator = klass.from_config(config)\n+ estimator.load_weights(infile_weights)\n+ # build train model\n+ else:\n+ cls_name = inputs[\'mode_selection\'][\'learning_type\']\n+ klass = try_get_attr(\'galaxy_ml.keras_galaxy_models\', cls_name)\n+\n+ options[\'loss\'] = (inputs[\'mode_selection\']\n+ [\'compile_params\'][\'loss\'])\n+ options[\'optimizer\'] =\\\n+ (inputs[\'mode_selection\'][\'compile_params\']\n+ [\'optimizer_selection\'][\'optimizer_type\']).lower()\n+\n+ options.update((inputs[\'mode_selection\'][\'compile_params\']\n+ [\'optimizer_selection\'][\'optimizer_options\']))\n+\n+ train_metrics = (inputs[\'mode_selection\'][\'compile_params\']\n+ [\'metrics\']).split(\',\')\n+ if train_metrics[-1] == \'none\':\n+ train_metrics = train_metrics[:-1]\n+ options[\'metrics\'] = train_metrics\n+\n+ options.update(inputs[\'mode_selection\'][\'fit_params\'])\n+ options[\'seed\'] = inputs[\'mode_selection\'][\'random_seed\']\n+\n+ if batch_mode:\n+ generator = get_batch_generator(inputs[\'mode_selection\']\n+ [\'generator_selection\'])\n+ options[\'data_batch_generator\'] = generator\n+ options[\'prediction_steps\'] = \\\n+ inputs[\'mode_selection\'][\'prediction_steps\']\n+ options[\'class_positive_factor\'] = \\\n+ inputs[\'mode_selection\'][\'class_positive_factor\']\n+ estimator = klass(config, **options)\n+ if outfile_params:\n+ hyper_params = get_search_params(estimator)\n+ # TODO: remove this after making `verbose` tunable\n+ for h_param in hyper_params:\n+ if h_param[1].endswith(\'verbose\'):\n+ h_param[0] = \'@\'\n+ df = pd.DataFrame(hyper_params, columns=[\'\', \'Parameter\', \'Value\'])\n+ df.to_csv(outfile_params, sep=\'\\t\', index=False)\n+\n+ print(repr(estimator))\n+ # save model by pickle\n+ with open(outfile, \'wb\') as f:\n+ pickle.dump(estimator, f, pickle.HIGHEST_PROTOCOL)\n+\n+\n+if __name__ == \'__main__\':\n+ warnings.simplefilter(\'ignore\')\n+\n+ aparser = argparse.ArgumentParser()\n+ aparser.add_argument("-i", "--inputs", dest="inputs", required=True)\n+ aparser.add_argument("-m", "--model_json", dest="model_json")\n+ aparser.add_argument("-t", "--tool_id", dest="tool_id")\n+ aparser.add_argument("-w", "--infile_weights", dest="infile_weights")\n+ aparser.add_argument("-o", "--outfile", dest="outfile")\n+ aparser.add_argument("-p", "--outfile_params", dest="outfile_params")\n+ args = aparser.parse_args()\n+\n+ input_json_path = args.inputs\n+ with open(input_json_path, \'r\') as param_handler:\n+ inputs = json.load(param_handler)\n+\n+ tool_id = args.tool_id\n+ outfile = args.outfile\n+ outfile_params = args.outfile_params\n+ model_json = args.model_json\n+ infile_weights = args.infile_weights\n+\n+ # for keras_model_config tool\n+ if tool_id == \'keras_model_config\':\n+ config_keras_model(inputs, outfile)\n+\n+ # for keras_model_builder tool\n+ else:\n+ batch_mode = False\n+ if tool_id == \'keras_batch_models\':\n+ batch_mode = True\n+\n+ build_keras_model(inputs=inputs,\n+ model_json=model_json,\n+ infile_weights=infile_weights,\n+ batch_mode=batch_mode,\n+ outfile=outfile,\n+ outfile_params=outfile_params)\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb keras_macros.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/keras_macros.xml Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| b'@@ -0,0 +1,1228 @@\n+<macros>\n+ <token name="@KERAS_VERSION@">0.4.0</token>\n+\n+ <xml name="macro_stdio">\n+ <stdio>\n+ <exit_code range="1:" level="fatal" description="Error occurred. Please check Tool Standard Error"/>\n+ </stdio>\n+ </xml>\n+\n+ <xml name="keras_optimizer_common" token_lr="0.01">\n+ <section name="optimizer_options" title="Optimizer Advanced Options" expanded="false">\n+ <param argument="lr" type="float" value="@LR@" optional="true" label="Learning rate" help="float >= 0"/>\n+ <yield/>\n+ <!--param argument="clipnorm" type="float" value="" optional="true" label="clipnorm" help="float >= 0"/-->\n+ <!--param argument="clipvalue" type="float" value="" optional="true" label="clipvalue" help="float >= 0"/-->\n+ </section>\n+ </xml>\n+\n+ <xml name="keras_optimizer_common_more" token_lr="0.001">\n+ <expand macro="keras_optimizer_common" lr="@LR@">\n+ <param argument="epsilon" type="float" value="" label="epsilon" optional="true" help="Fuzz factor. If `None`, defaults to `K.epsilon()`"/>\n+ <param argument="decay" type="float" value="0" optional="true" label="decay" help="Learning rate decay over each update."/>\n+ <yield/>\n+ </expand>\n+ </xml>\n+\n+ <xml name="keras_activations">\n+ <param argument="activation" type="select" label="Activation function">\n+ <option value="linear" selected="true">None / linear (default)</option>\n+ <option value="softmax">softmax</option>\n+ <option value="elu">elu</option>\n+ <option value="selu">selu</option>\n+ <option value="softplus">softplus</option>\n+ <option value="softsign">softsign</option>\n+ <option value="relu">relu</option>\n+ <option value="tanh">tanh</option>\n+ <option value="sigmoid">sigmoid</option>\n+ <option value="hard_sigmoid">hard_sigmoid</option>\n+ <option value="exponential">tanh</option>\n+ </param>\n+ </xml>\n+\n+ <xml name="keras_initializers" token_argument="kernel_initializer" token_default_kernel="false" token_default_bias="false" token_default_embeddings="false">\n+ <param argument="@ARGUMENT@" type="select" label="@ARGUMENT@">\n+ <option value="zeros" selected="@DEFAULT_BIAS@">zero / zeros / Zeros</option>\n+ <option value="ones">one / ones / Ones</option>\n+ <option value="constant">constant / Constant</option>\n+ <option value="random_normal">normal / random_normal / RandomNormal</option>\n+ <option value="random_uniform" selected="@DEFAULT_EMBEDDINGS@">uniform / random_uniform / RandomUniform</option>\n+ <option value="truncated_normal">truncated_normal / TruncatedNormal</option>\n+ <option value="orthogonal">orthogonal / Orthogonal</option>\n+ <option value="identity">identity / Identity</option>\n+ <option value="glorot_normal">glorot_normal</option>\n+ <option value="glorot_uniform" selected="@DEFAULT_KERNEL@">glorot_uniform</option>\n+ <option value="he_normal">he_normal</option>\n+ <option value="he_uniform">he_uniform</option>\n+ <option value="lecun_normal">lecun_normal</option>\n+ <option value="lecun_uniform">lecun_uniform</option>\n+ </param>\n+ </xml>\n+\n+ <xml name="keras_regularizers" token_argument="kernel_regularizer">\n+ <param argument="@ARGUMENT@" type="text" value="(0. , 0.)" optional="true" label="@ARGUMENT@"\n+ help="(l1, l2). l1/l2: float; L1/l2 regularization factor. (0., 0.) is equivalent to `None`"/>\n+ </xml>\n+\n+ <xml name="keras_constraints_options">\n+ <section name="constraint_options" title="Constraint Advanced Options" expanded="false">\n+ <yield/>\n+ <param argument="axis" type="text" value="0" help="Integer or list of integers. axis along which to calculate weight norms">\n+ <sanitizer>\n+ <valid initial="default">\n+ <add value="["/>\n+ <add value="]"/>\n+ </valid>\n+ </sanitizer>\n+ </param>\n+ </section>\n+ </xml>\n+\n+ <xml name="keras_constraints" token_argument="kernel_constraint"'..b'ple="true" label="Select metrics">\n+ <option value="acc" selected="true">acc / accruracy</option>\n+ <option value="binary_accuracy">binary_accuracy</option>\n+ <option value="categorical_accuracy">categorical_accuracy</option>\n+ <option value="sparse_categorical_accuracy">sparse_categorical_accuracy</option>\n+ <option value="mse">mse / MSE / mean_squared_error</option>\n+ <option value="mae">mae / MAE / mean_absolute_error</option>\n+ <option value="mae">mape / MAPE / mean_absolute_percentage_error</option>\n+ <option value="cosine_proximity">cosine_proximity</option>\n+ <option value="cosine">cosine</option>\n+ <option value="none">none</option>\n+ </param>\n+ </section>\n+ </xml>\n+\n+ <xml name="keras_fit_params_section">\n+ <section name="fit_params" title="Fit Parameters" expanded="true">\n+ <param name="epochs" type="integer" value="1" min="1" label="epochs"/>\n+ <param name="batch_size" type="integer" value="32" optional="true" label="batch_size" help="Integer or blank for 32"/>\n+ <param name="steps_per_epoch" type="integer" value="" optional="true" label="steps_per_epoch" help="The number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. The default None is equal to the number of samples in your dataset divided by the batch size, or 1 if that cannot be determined."/>\n+ <param name="validation_steps" type="integer" value="" optional="true" label="validation_steps" help="Default None. Total number of steps (batches of samples) to validate before stopping." />\n+ <!--`validation_freq` will be available in next keras version-->\n+ <!--param name="validation_freq" type="integer" value="1" optional="true" label="validation_freq" help="Integer only at current moment. If an integer, specifies how many training epochs to run before a new validation run is performed."/-->\n+ <expand macro="keras_callbacks"/>\n+ </section>\n+ </xml>\n+\n+ <!--Citation-->\n+ <xml name="keras_citation">\n+ <citation type="bibtex">\n+ @misc{chollet2015keras,\n+ title={Keras},\n+ url={https://keras.io},\n+ author={Chollet, Fran\\c{c}ois and others},\n+ year={2015},\n+ howpublished={https://keras.io},\n+ }\n+ </citation>\n+ </xml>\n+\n+ <xml name="tensorflow_citation">\n+ <citation type="bibtex">\n+ @misc{tensorflow2015-whitepaper,\n+ title={ {TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems},\n+ url={https://www.tensorflow.org/},\n+ note={Software available from tensorflow.org},\n+ author={\n+ Mart\\\'{\\i}n~Abadi and\n+ Ashish~Agarwal and\n+ Paul~Barham and\n+ Eugene~Brevdo and\n+ Zhifeng~Chen and\n+ Craig~Citro and\n+ Greg~S.~Corrado and\n+ Andy~Davis and\n+ Jeffrey~Dean and\n+ Matthieu~Devin and\n+ Sanjay~Ghemawat and\n+ Ian~Goodfellow and\n+ Andrew~Harp and\n+ Geoffrey~Irving and\n+ Michael~Isard and\n+ Yangqing Jia and\n+ Rafal~Jozefowicz and\n+ Lukasz~Kaiser and\n+ Manjunath~Kudlur and\n+ Josh~Levenberg and\n+ Dandelion~Man\\\'{e} and\n+ Rajat~Monga and\n+ Sherry~Moore and\n+ Derek~Murray and\n+ Chris~Olah and\n+ Mike~Schuster and\n+ Jonathon~Shlens and\n+ Benoit~Steiner and\n+ Ilya~Sutskever and\n+ Kunal~Talwar and\n+ Paul~Tucker and\n+ Vincent~Vanhoucke and\n+ Vijay~Vasudevan and\n+ Fernanda~Vi\\\'{e}gas and\n+ Oriol~Vinyals and\n+ Pete~Warden and\n+ Martin~Wattenberg and\n+ Martin~Wicke and\n+ Yuan~Yu and\n+ Xiaoqiang~Zheng},\n+ year={2015},\n+ }\n+ </citation>\n+ </xml>\n+\n+</macros>\n\\ No newline at end of file\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb main_macros.xml --- a/main_macros.xml Tue Jul 09 19:36:00 2019 -0400 +++ b/main_macros.xml Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| b'@@ -1,16 +1,12 @@\n <macros>\n- <token name="@VERSION@">1.0.0.4</token>\n+ <token name="@VERSION@">1.0.7.10</token>\n+\n+ <token name="@ENSEMBLE_VERSION@">0.2.0</token>\n \n <xml name="python_requirements">\n <requirements>\n <requirement type="package" version="3.6">python</requirement>\n- <requirement type="package" version="0.20.3">scikit-learn</requirement>\n- <requirement type="package" version="0.24.2">pandas</requirement>\n- <requirement type="package" version="0.80">xgboost</requirement>\n- <requirement type="package" version="0.9.13">asteval</requirement>\n- <requirement type="package" version="0.6">skrebate</requirement>\n- <requirement type="package" version="0.4.2">imbalanced-learn</requirement>\n- <requirement type="package" version="0.16.0">mlxtend</requirement>\n+ <requirement type="package" version="0.7.10">Galaxy-ML</requirement>\n <yield/>\n </requirements>\n </xml>\n@@ -420,8 +416,7 @@\n \n <xml name="sparse_target" token_label1="Select a sparse matrix:" token_label2="Select the tabular containing true labels:" token_multiple="False" token_format1="txt" token_format2="tabular" token_help1="" token_help2="">\n <param name="infile1" type="data" format="@FORMAT1@" label="@LABEL1@" help="@HELP1@"/>\n- <param name="infile2" type="data" format="@FORMAT2@" label="@LABEL2@" help="@HELP2@"/>\n- <param name="col2" multiple="@MULTIPLE@" type="data_column" data_ref="infile2" label="Select target column(s):"/>\n+ <expand macro="input_tabular_target"/>\n </xml>\n \n <xml name="sl_mixed_input">\n@@ -429,6 +424,8 @@\n <param name="selected_input" type="select" label="Select input type:">\n <option value="tabular" selected="true">tabular data</option>\n <option value="sparse">sparse matrix</option>\n+ <option value="seq_fasta">sequnences in a fasta file</option>\n+ <option value="refseq_and_interval">reference genome and intervals</option>\n </param>\n <when value="tabular">\n <expand macro="samples_tabular" multiple1="true" multiple2="false"/>\n@@ -436,6 +433,36 @@\n <when value="sparse">\n <expand macro="sparse_target"/>\n </when>\n+ <when value="seq_fasta">\n+ <expand macro="inputs_seq_fasta"/>\n+ </when>\n+ <when value="refseq_and_interval">\n+ <expand macro="inputs_refseq_and_interval"/>\n+ </when>\n+ </conditional>\n+ </xml>\n+\n+ <xml name="input_tabular_target">\n+ <param name="infile2" type="data" format="tabular" label="Dataset containing class labels or target values:"/>\n+ <param name="header2" type="boolean" optional="true" truevalue="booltrue" falsevalue="boolfalse" checked="false" label="Does the dataset contain header:" />\n+ <conditional name="column_selector_options_2">\n+ <expand macro="samples_column_selector_options" column_option="selected_column_selector_option2" col_name="col2" multiple="false" infile="infile2"/>\n+ </conditional>\n+ </xml>\n+\n+ <xml name="inputs_seq_fasta">\n+ <param name="fasta_path" type="data" format="fasta" label="Dataset containing fasta genomic/protein sequences" help="Sequences will be one-hot encoded to arrays."/>\n+ <expand macro="input_tabular_target"/>\n+ </xml>\n+\n+ <xml name="inputs_refseq_and_interval">\n+ <param name="ref_genome_file" type="data" format="fasta" label="Dataset containing reference genomic sequence"/>\n+ <param name="interval_file" type="data" format="interval" label="Dataset containing sequence intervals for training" help="interval. Sequences will be retrieved from the reference genome and one-hot encoded to training arrays."/>\n+ <param name="target_file" type="data" format="bed" label="Dataset containing positions and features for target values." help="bed. The file will be compressed with `bgzip` and then indexed using `tabix`."/>\n+ <param name="infile2" type="data" format="tabular" label="Dataset containing the feature'..b'ine/estimator object"/>\n+ <section name="hyperparams_swapping" title="Hyperparameter Swapping" expanded="false">\n+ <param name="infile_params" type="data" format="tabular" optional="true" label="Choose the dataset containing hyperparameters for the pipeline/estimator above" help="This dataset could be the output of `get_params` in the `Estimator Attributes` tool."/>\n+ <repeat name="param_set" min="1" max="30" title="New hyperparameter setting">\n+ <param name="sp_name" type="select" optional="true" label="Choose a parameter name (with current value)">\n+ <options from_dataset="infile_params" startswith="@">\n+ <column name="name" index="2"/>\n+ <column name="value" index="1"/>\n+ <filter type="unique_value" name="unique_param" column="1"/>\n+ </options>\n+ </param>\n+ <param name="sp_value" type="text" value="" optional="true" label="New value" help="Supports int, float, boolean, single quoted string, and selected object constructor. Similar to the `Parameter settings for search` section in `searchcv` tool except that only single value is expected here.">\n+ <sanitizer>\n+ <valid initial="default">\n+ <add value="'"/>\n+ <add value="""/>\n+ </valid>\n+ </sanitizer>\n+ </param>\n+ </repeat>\n+ </section>\n+ </xml>\n+\n <xml name="search_cv_options">\n <expand macro="scoring_selection"/>\n <expand macro="model_validation_common_options"/>\n@@ -1750,6 +1814,40 @@\n </conditional>\n </xml>\n \n+ <xml name="stacking_voting_weights">\n+ <section name="options" title="Advanced Options" expanded="false">\n+ <param argument="weights" type="text" value="[]" optional="true" help="Sequence of weights (float or int). Uses uniform weights if None (`[]`).">\n+ <sanitizer>\n+ <valid initial="default">\n+ <add value="["/>\n+ <add value="]"/>\n+ </valid>\n+ </sanitizer>\n+ </param>\n+ <yield/>\n+ </section>\n+ </xml>\n+\n+ <xml name="preprocessors_sequence_encoders">\n+ <conditional name="encoder_selection">\n+ <param name="encoder_type" type="select" label="Choose the sequence encoder class">\n+ <option value="GenomeOneHotEncoder">GenomeOneHotEncoder</option>\n+ <option value="ProteinOneHotEncoder">ProteinOneHotEncoder</option>\n+ </param>\n+ <when value="GenomeOneHotEncoder">\n+ <expand macro="preprocessors_sequence_encoder_arguments"/>\n+ </when>\n+ <when value="ProteinOneHotEncoder">\n+ <expand macro="preprocessors_sequence_encoder_arguments"/>\n+ </when>\n+ </conditional>\n+ </xml>\n+\n+ <xml name="preprocessors_sequence_encoder_arguments">\n+ <param argument="seq_length" type="integer" value="" min="0" optional="true" help="Integer. Sequence length"/>\n+ <param argument="padding" type="boolean" truevalue="booltrue" falsevalue="boolfalse" checked="true" help="Whether to pad or truncate sequence to meet the sequence length."/>\n+ </xml>\n+\n <!-- Outputs -->\n \n <xml name="output">\n@@ -1847,7 +1945,7 @@\n </citation>\n </xml>\n \n- <xml name="imblearn_citation">\n+ <xml name="imblearn_citation">\n <citation type="bibtex">\n @article{JMLR:v18:16-365,\n author = {Guillaume Lema{{\\^i}}tre and Fernando Nogueira and Christos K. Aridas},\n@@ -1862,4 +1960,19 @@\n </citation>\n </xml>\n \n+ <xml name="selene_citation">\n+ <citation type="bibtex">\n+ @article{chen2019selene,\n+ title={Selene: a PyTorch-based deep learning library for sequence data},\n+ author={Chen, Kathleen M and Cofer, Evan M and Zhou, Jian and Troyanskaya, Olga G},\n+ journal={Nature methods},\n+ volume={16},\n+ number={4},\n+ pages={315},\n+ year={2019},\n+ publisher={Nature Publishing Group}\n+ }\n+ </citation>\n+ </xml>\n+\n </macros>\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb model_prediction.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/model_prediction.py Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,205 @@ +import argparse +import json +import numpy as np +import pandas as pd +import warnings + +from scipy.io import mmread +from sklearn.pipeline import Pipeline + +from galaxy_ml.utils import (load_model, read_columns, + get_module, try_get_attr) + + +N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1)) + + +def main(inputs, infile_estimator, outfile_predict, + infile_weights=None, infile1=None, + fasta_path=None, ref_seq=None, + vcf_path=None): + """ + Parameter + --------- + inputs : str + File path to galaxy tool parameter + + infile_estimator : strgit + File path to trained estimator input + + outfile_predict : str + File path to save the prediction results, tabular + + infile_weights : str + File path to weights input + + infile1 : str + File path to dataset containing features + + fasta_path : str + File path to dataset containing fasta file + + ref_seq : str + File path to dataset containing the reference genome sequence. + + vcf_path : str + File path to dataset containing variants info. + """ + warnings.filterwarnings('ignore') + + with open(inputs, 'r') as param_handler: + params = json.load(param_handler) + + # load model + with open(infile_estimator, 'rb') as est_handler: + estimator = load_model(est_handler) + + main_est = estimator + if isinstance(estimator, Pipeline): + main_est = estimator.steps[-1][-1] + if hasattr(main_est, 'config') and hasattr(main_est, 'load_weights'): + if not infile_weights or infile_weights == 'None': + raise ValueError("The selected model skeleton asks for weights, " + "but dataset for weights wan not selected!") + main_est.load_weights(infile_weights) + + # handle data input + input_type = params['input_options']['selected_input'] + # tabular input + if input_type == 'tabular': + header = 'infer' if params['input_options']['header1'] else None + column_option = (params['input_options'] + ['column_selector_options_1'] + ['selected_column_selector_option']) + if column_option in ['by_index_number', 'all_but_by_index_number', + 'by_header_name', 'all_but_by_header_name']: + c = params['input_options']['column_selector_options_1']['col1'] + else: + c = None + + df = pd.read_csv(infile1, sep='\t', header=header, parse_dates=True) + + X = read_columns(df, c=c, c_option=column_option).astype(float) + + if params['method'] == 'predict': + preds = estimator.predict(X) + else: + preds = estimator.predict_proba(X) + + # sparse input + elif input_type == 'sparse': + X = mmread(open(infile1, 'r')) + if params['method'] == 'predict': + preds = estimator.predict(X) + else: + preds = estimator.predict_proba(X) + + # fasta input + elif input_type == 'seq_fasta': + if not hasattr(estimator, 'data_batch_generator'): + raise ValueError( + "To do prediction on sequences in fasta input, " + "the estimator must be a `KerasGBatchClassifier`" + "equipped with data_batch_generator!") + pyfaidx = get_module('pyfaidx') + sequences = pyfaidx.Fasta(fasta_path) + n_seqs = len(sequences.keys()) + X = np.arange(n_seqs)[:, np.newaxis] + seq_length = estimator.data_batch_generator.seq_length + batch_size = getattr(estimator, 'batch_size', 32) + steps = (n_seqs + batch_size - 1) // batch_size + + seq_type = params['input_options']['seq_type'] + klass = try_get_attr( + 'galaxy_ml.preprocessors', seq_type) + + pred_data_generator = klass( + fasta_path, seq_length=seq_length) + + if params['method'] == 'predict': + preds = estimator.predict( + X, data_generator=pred_data_generator, steps=steps) + else: + preds = estimator.predict_proba( + X, data_generator=pred_data_generator, steps=steps) + + # vcf input + elif input_type == 'variant_effect': + klass = try_get_attr('galaxy_ml.preprocessors', + 'GenomicVariantBatchGenerator') + + options = params['input_options'] + options.pop('selected_input') + if options['blacklist_regions'] == 'none': + options['blacklist_regions'] = None + + pred_data_generator = klass( + ref_genome_path=ref_seq, vcf_path=vcf_path, **options) + + pred_data_generator.fit() + + preds = estimator.model_.predict_generator( + pred_data_generator.flow(batch_size=32), + workers=N_JOBS, + use_multiprocessing=True) + + if preds.min() < 0. or preds.max() > 1.: + warnings.warn('Network returning invalid probability values. ' + 'The last layer might not normalize predictions ' + 'into probabilities ' + '(like softmax or sigmoid would).') + + if params['method'] == 'predict_proba' and preds.shape[1] == 1: + # first column is probability of class 0 and second is of class 1 + preds = np.hstack([1 - preds, preds]) + + elif params['method'] == 'predict': + if preds.shape[-1] > 1: + # if the last activation is `softmax`, the sum of all + # probibilities will 1, the classification is considered as + # multi-class problem, otherwise, we take it as multi-label. + act = getattr(estimator.model_.layers[-1], 'activation', None) + if act and act.__name__ == 'softmax': + classes = preds.argmax(axis=-1) + else: + preds = (preds > 0.5).astype('int32') + else: + classes = (preds > 0.5).astype('int32') + + preds = estimator.classes_[classes] + # end input + + # output + if input_type == 'variant_effect': # TODO: save in batchs + rval = pd.DataFrame(preds) + meta = pd.DataFrame( + pred_data_generator.variants, + columns=['chrom', 'pos', 'name', 'ref', 'alt', 'strand']) + + rval = pd.concat([meta, rval], axis=1) + + elif len(preds.shape) == 1: + rval = pd.DataFrame(preds, columns=['Predicted']) + else: + rval = pd.DataFrame(preds) + + rval.to_csv(outfile_predict, sep='\t', + header=True, index=False) + + +if __name__ == '__main__': + aparser = argparse.ArgumentParser() + aparser.add_argument("-i", "--inputs", dest="inputs", required=True) + aparser.add_argument("-e", "--infile_estimator", dest="infile_estimator") + aparser.add_argument("-w", "--infile_weights", dest="infile_weights") + aparser.add_argument("-X", "--infile1", dest="infile1") + aparser.add_argument("-O", "--outfile_predict", dest="outfile_predict") + aparser.add_argument("-f", "--fasta_path", dest="fasta_path") + aparser.add_argument("-r", "--ref_seq", dest="ref_seq") + aparser.add_argument("-v", "--vcf_path", dest="vcf_path") + args = aparser.parse_args() + + main(args.inputs, args.infile_estimator, args.outfile_predict, + infile_weights=args.infile_weights, infile1=args.infile1, + fasta_path=args.fasta_path, ref_seq=args.ref_seq, + vcf_path=args.vcf_path) |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb model_validations.py --- a/model_validations.py Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,252 +0,0 @@\n-"""\n-class\n------\n-OrderedKFold\n-RepeatedOrderedKold\n-\n-\n-function\n---------\n-train_test_split\n-"""\n-\n-import numpy as np\n-import warnings\n-\n-from itertools import chain\n-from math import ceil, floor\n-from sklearn.model_selection import (GroupShuffleSplit, ShuffleSplit,\n- StratifiedShuffleSplit)\n-from sklearn.model_selection._split import _BaseKFold, _RepeatedSplits\n-from sklearn.utils import check_random_state, indexable, safe_indexing\n-from sklearn.utils.validation import _num_samples, check_array\n-\n-\n-def _validate_shuffle_split(n_samples, test_size, train_size,\n- default_test_size=None):\n- """\n- Validation helper to check if the test/test sizes are meaningful wrt to the\n- size of the data (n_samples)\n- """\n- if test_size is None and train_size is None:\n- test_size = default_test_size\n-\n- test_size_type = np.asarray(test_size).dtype.kind\n- train_size_type = np.asarray(train_size).dtype.kind\n-\n- if (test_size_type == \'i\' and (test_size >= n_samples or test_size <= 0)\n- or test_size_type == \'f\' and (test_size <= 0 or test_size >= 1)):\n- raise ValueError(\'test_size={0} should be either positive and smaller\'\n- \' than the number of samples {1} or a float in the \'\n- \'(0, 1) range\'.format(test_size, n_samples))\n-\n- if (train_size_type == \'i\' and (train_size >= n_samples or train_size <= 0)\n- or train_size_type == \'f\' and (train_size <= 0 or train_size >= 1)):\n- raise ValueError(\'train_size={0} should be either positive and smaller\'\n- \' than the number of samples {1} or a float in the \'\n- \'(0, 1) range\'.format(train_size, n_samples))\n-\n- if train_size is not None and train_size_type not in (\'i\', \'f\'):\n- raise ValueError("Invalid value for train_size: {}".format(train_size))\n- if test_size is not None and test_size_type not in (\'i\', \'f\'):\n- raise ValueError("Invalid value for test_size: {}".format(test_size))\n-\n- if (train_size_type == \'f\' and test_size_type == \'f\' and\n- train_size + test_size > 1):\n- raise ValueError(\n- \'The sum of test_size and train_size = {}, should be in the (0, 1)\'\n- \' range. Reduce test_size and/or train_size.\'\n- .format(train_size + test_size))\n-\n- if test_size_type == \'f\':\n- n_test = ceil(test_size * n_samples)\n- elif test_size_type == \'i\':\n- n_test = float(test_size)\n-\n- if train_size_type == \'f\':\n- n_train = floor(train_size * n_samples)\n- elif train_size_type == \'i\':\n- n_train = float(train_size)\n-\n- if train_size is None:\n- n_train = n_samples - n_test\n- elif test_size is None:\n- n_test = n_samples - n_train\n-\n- if n_train + n_test > n_samples:\n- raise ValueError(\'The sum of train_size and test_size = %d, \'\n- \'should be smaller than the number of \'\n- \'samples %d. Reduce test_size and/or \'\n- \'train_size.\' % (n_train + n_test, n_samples))\n-\n- n_train, n_test = int(n_train), int(n_test)\n-\n- if n_train == 0:\n- raise ValueError(\n- \'With n_samples={}, test_size={} and train_size={}, the \'\n- \'resulting train set will be empty. Adjust any of the \'\n- \'aforementioned parameters.\'.format(n_samples, test_size,\n- train_size)\n- )\n-\n- return n_train, n_test\n-\n-\n-def train_test_split(*arrays, **options):\n- """Extend sklearn.model_selection.train_test_slit to have group split.\n-\n- Parameters\n- ----------\n- *arrays : sequence of indexables with same length / shape[0]\n- Allowed inputs are lists, numpy arrays, scipy-sparse\n- matrices or pandas dataframes.\n-\n- test_size : float, int or None, optional (default=None)\n- If float, should be betw'..b'arrays == 0:\n- raise ValueError("At least one array required as input")\n- test_size = options.pop(\'test_size\', None)\n- train_size = options.pop(\'train_size\', None)\n- random_state = options.pop(\'random_state\', None)\n- shuffle = options.pop(\'shuffle\', \'simple\')\n- labels = options.pop(\'labels\', None)\n-\n- if options:\n- raise TypeError("Invalid parameters passed: %s" % str(options))\n-\n- arrays = indexable(*arrays)\n-\n- n_samples = _num_samples(arrays[0])\n- if shuffle == \'group\':\n- if labels is None:\n- raise ValueError("When shuffle=\'group\', "\n- "labels should not be None!")\n- labels = check_array(labels, ensure_2d=False, dtype=None)\n- uniques = np.unique(labels)\n- n_samples = uniques.size\n-\n- n_train, n_test = _validate_shuffle_split(n_samples, test_size, train_size,\n- default_test_size=0.25)\n-\n- shuffle_options = dict(test_size=n_test,\n- train_size=n_train,\n- random_state=random_state)\n-\n- if shuffle is None:\n- if labels is not None:\n- warnings.warn("The `labels` is ignored for "\n- "shuffle being None!")\n-\n- train = np.arange(n_train)\n- test = np.arange(n_train, n_train + n_test)\n-\n- elif shuffle == \'simple\':\n- if labels is not None:\n- warnings.warn("The `labels` is not needed and therefore "\n- "ignored for ShuffleSplit, as shuffle=\'simple\'!")\n-\n- cv = ShuffleSplit(**shuffle_options)\n- train, test = next(cv.split(X=arrays[0], y=None))\n-\n- elif shuffle == \'stratified\':\n- cv = StratifiedShuffleSplit(**shuffle_options)\n- train, test = next(cv.split(X=arrays[0], y=labels))\n-\n- elif shuffle == \'group\':\n- cv = GroupShuffleSplit(**shuffle_options)\n- train, test = next(cv.split(X=arrays[0], y=None, groups=labels))\n-\n- else:\n- raise ValueError("The argument `shuffle` only supports None, "\n- "\'simple\', \'stratified\' and \'group\', but got `%s`!"\n- % shuffle)\n-\n- return list(chain.from_iterable((safe_indexing(a, train),\n- safe_indexing(a, test)) for a in arrays))\n-\n-\n-class OrderedKFold(_BaseKFold):\n- """\n- Split into K fold based on ordered target value\n-\n- Parameters\n- ----------\n- n_splits : int, default=3\n- Number of folds. Must be at least 2.\n- shuffle: bool\n- random_state: None or int\n- """\n-\n- def __init__(self, n_splits=3, shuffle=False, random_state=None):\n- super(OrderedKFold, self).__init__(n_splits, shuffle, random_state)\n-\n- def _iter_test_indices(self, X, y, groups=None):\n- n_samples = _num_samples(X)\n- n_splits = self.n_splits\n- y = np.asarray(y)\n- sorted_index = np.argsort(y)\n- if self.shuffle:\n- current = 0\n- rng = check_random_state(self.random_state)\n- for i in range(n_samples // int(n_splits)):\n- start, stop = current, current + n_splits\n- rng.shuffle(sorted_index[start:stop])\n- current = stop\n- rng.shuffle(sorted_index[current:])\n-\n- for i in range(n_splits):\n- yield sorted_index[i:n_samples:n_splits]\n-\n-\n-class RepeatedOrderedKFold(_RepeatedSplits):\n- """ Repeated OrderedKFold runs mutiple times with different randomization.\n-\n- Parameters\n- ----------\n- n_splits : int, default=5\n- Number of folds. Must be at least 2.\n-\n- n_repeats : int, default=5\n- Number of times cross-validator to be repeated.\n-\n- random_state: int, RandomState instance or None. Optional\n- """\n- def __init__(self, n_splits=5, n_repeats=5, random_state=None):\n- super(RepeatedOrderedKFold, self).__init__(\n- OrderedKFold, n_repeats, random_state, n_splits=n_splits)\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb pk_whitelist.json --- a/pk_whitelist.json Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,768 +0,0 @@\n-{ "SK_NAMES": [\n- "sklearn._ASSUME_FINITE", "sklearn._isotonic._inplace_contiguous_isotonic_regression",\n- "sklearn._isotonic._make_unique", "sklearn.base.BaseEstimator",\n- "sklearn.base.BiclusterMixin", "sklearn.base.ClassifierMixin",\n- "sklearn.base.ClusterMixin", "sklearn.base.DensityMixin",\n- "sklearn.base.MetaEstimatorMixin", "sklearn.base.RegressorMixin",\n- "sklearn.base.TransformerMixin", "sklearn.base._first_and_last_element",\n- "sklearn.base._pprint", "sklearn.base.clone",\n- "sklearn.base.is_classifier", "sklearn.base.is_regressor",\n- "sklearn.clone", "sklearn.cluster.AffinityPropagation",\n- "sklearn.cluster.AgglomerativeClustering", "sklearn.cluster.Birch",\n- "sklearn.cluster.DBSCAN", "sklearn.cluster.FeatureAgglomeration",\n- "sklearn.cluster.KMeans", "sklearn.cluster.MeanShift",\n- "sklearn.cluster.MiniBatchKMeans", "sklearn.cluster.SpectralBiclustering",\n- "sklearn.cluster.SpectralClustering", "sklearn.cluster.SpectralCoclustering",\n- "sklearn.cluster._dbscan_inner.dbscan_inner", "sklearn.cluster._feature_agglomeration.AgglomerationTransform",\n- "sklearn.cluster._hierarchical.WeightedEdge", "sklearn.cluster._hierarchical._get_parents",\n- "sklearn.cluster._hierarchical._hc_get_descendent", "sklearn.cluster._hierarchical.average_merge",\n- "sklearn.cluster._hierarchical.compute_ward_dist", "sklearn.cluster._hierarchical.hc_get_heads",\n- "sklearn.cluster._hierarchical.max_merge", "sklearn.cluster._k_means._assign_labels_array",\n- "sklearn.cluster._k_means._assign_labels_csr", "sklearn.cluster._k_means._centers_dense",\n- "sklearn.cluster._k_means._centers_sparse", "sklearn.cluster._k_means._mini_batch_update_csr",\n- "sklearn.cluster._k_means_elkan.k_means_elkan", "sklearn.cluster.affinity_propagation",\n- "sklearn.cluster.affinity_propagation_.AffinityPropagation", "sklearn.cluster.affinity_propagation_.affinity_propagation",\n- "sklearn.cluster.bicluster.BaseSpectral", "sklearn.cluster.bicluster.SpectralBiclustering",\n- "sklearn.cluster.bicluster.SpectralCoclustering", "sklearn.cluster.bicluster._bistochastic_normalize",\n- "sklearn.cluster.bicluster._log_normalize", "sklearn.cluster.bicluster._scale_normalize",\n- "sklearn.cluster.birch.Birch", "sklearn.cluster.birch._CFNode",\n- "sklearn.cluster.birch._CFSubcluster", "sklearn.cluster.birch._iterate_sparse_X",\n- "sklearn.cluster.birch._split_node", "sklearn.cluster.dbscan",\n- "sklearn.cluster.dbscan_.DBSCAN", "sklearn.cluster.dbscan_.dbscan",\n- "sklearn.cluster.estimate_bandwidth", "sklearn.cluster.get_bin_seeds",\n- "sklearn.cluster.hierarchical.AgglomerativeClustering", "sklearn.cluster.hierarchical.FeatureAgglomeration",\n- "sklearn.cluster.hierarchical._TREE_BUILDERS", "sklearn.cluster.hierarchical._average_linkage",\n- "sklearn.cluster.hierarchical._complete_linkage", "sklearn.cluster.hierarchical._fix_connectivity",\n- "sklearn.cluster.hierarchical._hc_cut", "sklearn.cluster.hierarchical.linkage_tree",\n- "sklearn.cluster.hierarchical.ward_tree", "sklearn.cluster.k_means",\n- "sklearn.cluster.k_means_.FLOAT_DTYPES", "sklearn.cluster.k_means_.KMeans",\n- "sklearn.cluster.k_means_.MiniBatchKMeans", "sklearn.cluster.k_means_._init_centroids",\n- "sklearn.cluster.k_means_._k_init", "sklearn.cluster.k_means_._kmeans_single_elkan",\n- "sklearn.cluster.k_means_._kmeans_single_lloyd", "sklearn.cluster.k_means_._labels_inertia",\n- "sklearn.cluster.k_means_._labels_inertia_precompute_dense", "sklearn.cluster.k_means_._mini_batch_convergence",\n- "sklearn.cluster.k_means_._mini_batch_step", "sklearn.cluster.k_means_._tolerance",\n- "sklearn.cluster.k_means_._validate_center_shape", "sklearn.cluster.k_means_.k_means",\n- "sklearn.cluster.k_means_.string_types", "sklearn.cluster.linkage_tree",\n- "sklearn.cluster.mean_shift", "sklearn.cluster.mean_shift_.MeanShift",\n- "sklearn.cluster.mean_shift_._mean_shift_single_seed", "sklearn.cluster'..b'ltiSURFstar",\n- "skrebate.ReliefF", "skrebate.SURF",\n- "skrebate.SURFstar", "skrebate.TuRF",\n- "skrebate.multisurf.MultiSURF", "skrebate.multisurfstar.MultiSURFstar",\n- "skrebate.relieff.ReliefF", "skrebate.scoring_utils.MultiSURF_compute_scores",\n- "skrebate.scoring_utils.MultiSURFstar_compute_scores", "skrebate.scoring_utils.ReliefF_compute_scores",\n- "skrebate.scoring_utils.SURF_compute_scores", "skrebate.scoring_utils.SURFstar_compute_scores",\n- "skrebate.scoring_utils.compute_score", "skrebate.scoring_utils.get_row_missing",\n- "skrebate.scoring_utils.ramp_function", "skrebate.surf.SURF",\n- "skrebate.surfstar.SURFstar", "skrebate.turf.TuRF"\n- ],\n-\n- "XGB_NAMES": [\n- "xgboost.Booster", "xgboost.DMatrix",\n- "xgboost.VERSION_FILE", "xgboost.XGBClassifier",\n- "xgboost.XGBModel", "xgboost.XGBRegressor",\n- "xgboost.callback._fmt_metric", "xgboost.callback._get_callback_context",\n- "xgboost.callback.early_stop", "xgboost.callback.print_evaluation",\n- "xgboost.callback.record_evaluation", "xgboost.callback.reset_learning_rate",\n- "xgboost.compat.PANDAS_INSTALLED", "xgboost.compat.PY3",\n- "xgboost.compat.SKLEARN_INSTALLED", "xgboost.compat.STRING_TYPES",\n- "xgboost.compat.py_str", "xgboost.core.Booster",\n- "xgboost.core.CallbackEnv", "xgboost.core.DMatrix",\n- "xgboost.core.EarlyStopException", "xgboost.core.PANDAS_DTYPE_MAPPER",\n- "xgboost.core.PANDAS_INSTALLED", "xgboost.core.PY3",\n- "xgboost.core.STRING_TYPES", "xgboost.core.XGBoostError",\n- "xgboost.core._check_call", "xgboost.core._load_lib",\n- "xgboost.core._maybe_pandas_data", "xgboost.core._maybe_pandas_label",\n- "xgboost.core.c_array", "xgboost.core.c_str",\n- "xgboost.core.ctypes2buffer", "xgboost.core.ctypes2numpy",\n- "xgboost.core.from_cstr_to_pystr", "xgboost.core.from_pystr_to_cstr",\n- "xgboost.cv", "xgboost.f",\n- "xgboost.libpath.XGBoostLibraryNotFound", "xgboost.libpath.find_lib_path",\n- "xgboost.plot_importance", "xgboost.plot_tree",\n- "xgboost.plotting._EDGEPAT", "xgboost.plotting._EDGEPAT2",\n- "xgboost.plotting._LEAFPAT", "xgboost.plotting._NODEPAT",\n- "xgboost.plotting._parse_edge", "xgboost.plotting._parse_node",\n- "xgboost.plotting.plot_importance", "xgboost.plotting.plot_tree",\n- "xgboost.plotting.to_graphviz", "xgboost.rabit.DTYPE_ENUM__",\n- "xgboost.rabit.STRING_TYPES", "xgboost.rabit._init_rabit",\n- "xgboost.rabit.allreduce", "xgboost.rabit.broadcast",\n- "xgboost.rabit.finalize", "xgboost.rabit.get_processor_name",\n- "xgboost.rabit.get_rank", "xgboost.rabit.get_world_size",\n- "xgboost.rabit.init", "xgboost.rabit.tracker_print",\n- "xgboost.rabit.version_number", "xgboost.sklearn.SKLEARN_INSTALLED",\n- "xgboost.sklearn.XGBClassifier", "xgboost.sklearn.XGBModel",\n- "xgboost.sklearn.XGBRegressor", "xgboost.sklearn._objective_decorator",\n- "xgboost.to_graphviz", "xgboost.train",\n- "xgboost.training.CVPack", "xgboost.training.SKLEARN_INSTALLED",\n- "xgboost.training.STRING_TYPES", "xgboost.training._train_internal",\n- "xgboost.training.aggcv", "xgboost.training.cv",\n- "xgboost.training.mknfold", "xgboost.training.train"\n- ],\n-\n-\n- "NUMPY_NAMES": [\n- "numpy.core.multiarray._reconstruct", "numpy.ndarray",\n- "numpy.dtype", "numpy.core.multiarray.scalar", "numpy.random.__RandomState_ctor",\n- "numpy.ma.core._mareconstruct", "numpy.ma.core.MaskedArray"\n- ],\n-\n- "IMBLEARN_NAMES":[\n- "imblearn.pipeline.Pipeline", "imblearn.over_sampling._random_over_sampler.RandomOverSampler",\n- "imblearn.under_sampling._prototype_selection._edited_nearest_neighbours.EditedNearestNeighbours"\n- ],\n-\n- "MLXTEND_NAMES":[\n- "mlxtend.classifier.stacking_cv_classification.StackingCVClassifier",\n- "mlxtend.classifier.stacking_classification.StackingClassifier",\n- "mlxtend.regressor.stacking_cv_regression.StackingCVRegressor",\n- "mlxtend.regressor.stacking_regression.StackingRegressor"\n- ]\n-}\n\\ No newline at end of file\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb preprocessors.py --- a/preprocessors.py Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,184 +0,0 @@ -""" -Z_RandomOverSampler -""" - -import imblearn -import numpy as np - -from collections import Counter -from imblearn.over_sampling.base import BaseOverSampler -from imblearn.over_sampling import RandomOverSampler -from imblearn.pipeline import Pipeline as imbPipeline -from imblearn.utils import check_target_type -from scipy import sparse -from sklearn.base import BaseEstimator, TransformerMixin -from sklearn.preprocessing.data import _handle_zeros_in_scale -from sklearn.utils import check_array, safe_indexing -from sklearn.utils.fixes import nanpercentile -from sklearn.utils.validation import (check_is_fitted, check_X_y, - FLOAT_DTYPES) - - -class Z_RandomOverSampler(BaseOverSampler): - - def __init__(self, sampling_strategy='auto', - return_indices=False, - random_state=None, - ratio=None, - negative_thres=0, - positive_thres=-1): - super(Z_RandomOverSampler, self).__init__( - sampling_strategy=sampling_strategy, ratio=ratio) - self.random_state = random_state - self.return_indices = return_indices - self.negative_thres = negative_thres - self.positive_thres = positive_thres - - @staticmethod - def _check_X_y(X, y): - y, binarize_y = check_target_type(y, indicate_one_vs_all=True) - X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'], dtype=None) - return X, y, binarize_y - - def _fit_resample(self, X, y): - n_samples = X.shape[0] - - # convert y to z_score - y_z = (y - y.mean()) / y.std() - - index0 = np.arange(n_samples) - index_negative = index0[y_z > self.negative_thres] - index_positive = index0[y_z <= self.positive_thres] - index_unclassified = [x for x in index0 - if x not in index_negative - and x not in index_positive] - - y_z[index_negative] = 0 - y_z[index_positive] = 1 - y_z[index_unclassified] = -1 - - ros = RandomOverSampler( - sampling_strategy=self.sampling_strategy, - random_state=self.random_state, - ratio=self.ratio) - _, _ = ros.fit_resample(X, y_z) - sample_indices = ros.sample_indices_ - - print("Before sampler: %s. Total after: %s" - % (Counter(y_z), sample_indices.shape)) - - self.sample_indices_ = np.array(sample_indices) - - if self.return_indices: - return (safe_indexing(X, sample_indices), - safe_indexing(y, sample_indices), - sample_indices) - return (safe_indexing(X, sample_indices), - safe_indexing(y, sample_indices)) - - -def _get_quantiles(X, quantile_range): - """ - Calculate column percentiles for 2d array - - Parameters - ---------- - X : array-like, shape [n_samples, n_features] - """ - quantiles = [] - for feature_idx in range(X.shape[1]): - if sparse.issparse(X): - column_nnz_data = X.data[ - X.indptr[feature_idx]: X.indptr[feature_idx + 1]] - column_data = np.zeros(shape=X.shape[0], dtype=X.dtype) - column_data[:len(column_nnz_data)] = column_nnz_data - else: - column_data = X[:, feature_idx] - quantiles.append(nanpercentile(column_data, quantile_range)) - - quantiles = np.transpose(quantiles) - - return quantiles - - -class TDMScaler(BaseEstimator, TransformerMixin): - """ - Scale features using Training Distribution Matching (TDM) algorithm - - References - ---------- - .. [1] Thompson JA, Tan J and Greene CS (2016) Cross-platform - normalization of microarray and RNA-seq data for machine - learning applications. PeerJ 4, e1621. - """ - - def __init__(self, q_lower=25.0, q_upper=75.0, ): - self.q_lower = q_lower - self.q_upper = q_upper - - def fit(self, X, y=None): - """ - Parameters - ---------- - X : array-like, shape [n_samples, n_features] - """ - X = check_array(X, copy=True, estimator=self, dtype=FLOAT_DTYPES, - force_all_finite=True) - - if not 0 <= self.q_lower <= self.q_upper <= 100: - raise ValueError("Invalid quantile parameter values: " - "q_lower %s, q_upper: %s" - % (str(self.q_lower), str(self.q_upper))) - - # TODO sparse data - quantiles = nanpercentile(X, (self.q_lower, self.q_upper)) - iqr = quantiles[1] - quantiles[0] - - self.q_lower_ = quantiles[0] - self.q_upper_ = quantiles[1] - self.iqr_ = _handle_zeros_in_scale(iqr, copy=False) - - self.max_ = np.nanmax(X) - self.min_ = np.nanmin(X) - - return self - - def transform(self, X): - """ - Parameters - ---------- - X : {array-like, sparse matrix} - The data used to scale along the specified axis. - """ - check_is_fitted(self, 'iqr_', 'max_') - X = check_array(X, copy=True, estimator=self, dtype=FLOAT_DTYPES, - force_all_finite=True) - - # TODO sparse data - train_upper_scale = (self.max_ - self.q_upper_) / self.iqr_ - train_lower_scale = (self.q_lower_ - self.min_) / self.iqr_ - - test_quantiles = nanpercentile(X, (self.q_lower, self.q_upper)) - test_iqr = _handle_zeros_in_scale( - test_quantiles[1] - test_quantiles[0], copy=False) - - test_upper_bound = test_quantiles[1] + train_upper_scale * test_iqr - test_lower_bound = test_quantiles[0] - train_lower_scale * test_iqr - - test_min = np.nanmin(X) - if test_lower_bound < test_min: - test_lower_bound = test_min - - X[X > test_upper_bound] = test_upper_bound - X[X < test_lower_bound] = test_lower_bound - - X = (X - test_lower_bound) / (test_upper_bound - test_lower_bound)\ - * (self.max_ - self.min_) + self.min_ - - return X - - def inverse_transform(self, X): - """ - Scale the data back to the original state - """ - raise NotImplementedError("Inverse transformation is not implemented!") |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb search_model_validation.py --- a/search_model_validation.py Tue Jul 09 19:36:00 2019 -0400 +++ b/search_model_validation.py Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| b"@@ -1,22 +1,20 @@\n import argparse\n import collections\n import imblearn\n+import joblib\n import json\n import numpy as np\n-import pandas\n+import pandas as pd\n import pickle\n import skrebate\n import sklearn\n import sys\n import xgboost\n import warnings\n-import iraps_classifier\n-import model_validations\n-import preprocessors\n-import feature_selectors\n from imblearn import under_sampling, over_sampling, combine\n from scipy.io import mmread\n from mlxtend import classifier, regressor\n+from sklearn.base import clone\n from sklearn import (cluster, compose, decomposition, ensemble,\n feature_extraction, feature_selection,\n gaussian_process, kernel_approximation, metrics,\n@@ -24,18 +22,23 @@\n pipeline, preprocessing, svm, linear_model,\n tree, discriminant_analysis)\n from sklearn.exceptions import FitFailedWarning\n-from sklearn.externals import joblib\n-from sklearn.model_selection._validation import _score\n+from sklearn.model_selection._validation import _score, cross_validate\n+from sklearn.model_selection import _search, _validation\n \n-from utils import (SafeEval, get_cv, get_scoring, get_X_y,\n- load_model, read_columns)\n-from model_validations import train_test_split\n+from galaxy_ml.utils import (SafeEval, get_cv, get_scoring, load_model,\n+ read_columns, try_get_attr, get_module)\n+\n \n+_fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score')\n+setattr(_search, '_fit_and_score', _fit_and_score)\n+setattr(_validation, '_fit_and_score', _fit_and_score)\n \n N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1))\n CACHE_DIR = './cached'\n-NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', 'steps',\n- 'nthread', 'verbose')\n+NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', '_path',\n+ 'nthread', 'callbacks')\n+ALLOWED_CALLBACKS = ('EarlyStopping', 'TerminateOnNaN', 'ReduceLROnPlateau',\n+ 'CSVLogger', 'None')\n \n \n def _eval_search_params(params_builder):\n@@ -62,9 +65,9 @@\n search_list = search_list[1:].strip()\n # TODO maybe add regular express check\n ev = safe_eval_es(search_list)\n- preprocessors = (\n+ preprocessings = (\n preprocessing.StandardScaler(), preprocessing.Binarizer(),\n- preprocessing.Imputer(), preprocessing.MaxAbsScaler(),\n+ preprocessing.MaxAbsScaler(),\n preprocessing.Normalizer(), preprocessing.MinMaxScaler(),\n preprocessing.PolynomialFeatures(),\n preprocessing.RobustScaler(), feature_selection.SelectKBest(),\n@@ -133,21 +136,21 @@\n if obj is None:\n newlist.append(None)\n elif obj == 'all_0':\n- newlist.extend(preprocessors[0:36])\n+ newlist.extend(preprocessings[0:35])\n elif obj == 'sk_prep_all': # no KernalCenter()\n- newlist.extend(preprocessors[0:8])\n+ newlist.extend(preprocessings[0:7])\n elif obj == 'fs_all':\n- newlist.extend(preprocessors[8:15])\n+ newlist.extend(preprocessings[7:14])\n elif obj == 'decomp_all':\n- newlist.extend(preprocessors[15:26])\n+ newlist.extend(preprocessings[14:25])\n elif obj == 'k_appr_all':\n- newlist.extend(preprocessors[26:30])\n+ newlist.extend(preprocessings[25:29])\n elif obj == 'reb_all':\n- newlist.extend(preprocessors[31:36])\n+ newlist.extend(preprocessings[30:35])\n elif obj == 'imb_all':\n- newlist.extend(preprocessors[36:55])\n- elif type(obj) is int and -1 < obj < len(preprocessors):\n- newlist.append(preprocessors[obj]"..b' # train_test_split, output test result using best_estimator_\n+ # or rebuild the trained estimator using weights if applicable.\n+ else:\n+ scorer_ = searcher.scorer_\n+ if isinstance(scorer_, collections.Mapping):\n+ is_multimetric = True\n+ else:\n+ is_multimetric = False\n+\n+ best_estimator_ = getattr(searcher, \'best_estimator_\', None)\n+ if not best_estimator_:\n+ raise ValueError("GridSearchCV object has no "\n+ "`best_estimator_` when `refit`=False!")\n+\n+ if best_estimator_.__class__.__name__ == \'KerasGBatchClassifier\' \\\n+ and hasattr(estimator.data_batch_generator, \'target_path\'):\n+ test_score = best_estimator_.evaluate(\n+ X_test, scorer=scorer_, is_multimetric=is_multimetric)\n+ else:\n+ test_score = _score(best_estimator_, X_test,\n+ y_test, scorer_,\n+ is_multimetric=is_multimetric)\n+\n+ if not is_multimetric:\n+ test_score = {primary_scoring: test_score}\n+ for key, value in test_score.items():\n+ test_score[key] = [value]\n+ result_df = pd.DataFrame(test_score)\n+ result_df.to_csv(path_or_buf=outfile_result, sep=\'\\t\',\n+ header=True, index=False)\n \n memory.clear(warn=False)\n \n if outfile_object:\n+ best_estimator_ = getattr(searcher, \'best_estimator_\', None)\n+ if not best_estimator_:\n+ warnings.warn("GridSearchCV object has no attribute "\n+ "\'best_estimator_\', because either it\'s "\n+ "nested gridsearch or `refit` is False!")\n+ return\n+\n+ main_est = best_estimator_\n+ if isinstance(best_estimator_, pipeline.Pipeline):\n+ main_est = best_estimator_.steps[-1][-1]\n+\n+ if hasattr(main_est, \'model_\') \\\n+ and hasattr(main_est, \'save_weights\'):\n+ if outfile_weights:\n+ main_est.save_weights(outfile_weights)\n+ del main_est.model_\n+ del main_est.fit_params\n+ del main_est.model_class_\n+ del main_est.validation_data\n+ if getattr(main_est, \'data_generator_\', None):\n+ del main_est.data_generator_\n+ del main_est.data_batch_generator\n+\n with open(outfile_object, \'wb\') as output_handler:\n- pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL)\n+ pickle.dump(best_estimator_, output_handler,\n+ pickle.HIGHEST_PROTOCOL)\n \n \n if __name__ == \'__main__\':\n@@ -356,11 +555,18 @@\n aparser.add_argument("-e", "--estimator", dest="infile_estimator")\n aparser.add_argument("-X", "--infile1", dest="infile1")\n aparser.add_argument("-y", "--infile2", dest="infile2")\n- aparser.add_argument("-r", "--outfile_result", dest="outfile_result")\n+ aparser.add_argument("-O", "--outfile_result", dest="outfile_result")\n aparser.add_argument("-o", "--outfile_object", dest="outfile_object")\n+ aparser.add_argument("-w", "--outfile_weights", dest="outfile_weights")\n aparser.add_argument("-g", "--groups", dest="groups")\n+ aparser.add_argument("-r", "--ref_seq", dest="ref_seq")\n+ aparser.add_argument("-b", "--intervals", dest="intervals")\n+ aparser.add_argument("-t", "--targets", dest="targets")\n+ aparser.add_argument("-f", "--fasta_path", dest="fasta_path")\n args = aparser.parse_args()\n \n main(args.inputs, args.infile_estimator, args.infile1, args.infile2,\n args.outfile_result, outfile_object=args.outfile_object,\n- groups=args.groups)\n+ outfile_weights=args.outfile_weights, groups=args.groups,\n+ ref_seq=args.ref_seq, intervals=args.intervals,\n+ targets=args.targets, fasta_path=args.fasta_path)\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb stacking_ensembles.py --- a/stacking_ensembles.py Tue Jul 09 19:36:00 2019 -0400 +++ b/stacking_ensembles.py Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -1,26 +1,17 @@ import argparse +import ast import json +import mlxtend.regressor +import mlxtend.classifier import pandas as pd import pickle -import xgboost +import sklearn +import sys import warnings -from sklearn import (cluster, compose, decomposition, ensemble, - feature_extraction, feature_selection, - gaussian_process, kernel_approximation, metrics, - model_selection, naive_bayes, neighbors, - pipeline, preprocessing, svm, linear_model, - tree, discriminant_analysis) -from sklearn.model_selection._split import check_cv -from feature_selectors import (DyRFE, DyRFECV, - MyPipeline, MyimbPipeline) -from iraps_classifier import (IRAPSCore, IRAPSClassifier, - BinarizeTargetClassifier, - BinarizeTargetRegressor) -from preprocessors import Z_RandomOverSampler -from utils import load_model, get_cv, get_estimator, get_search_params +from sklearn import ensemble -from mlxtend.regressor import StackingCVRegressor, StackingRegressor -from mlxtend.classifier import StackingCVClassifier, StackingClassifier +from galaxy_ml.utils import (load_model, get_cv, get_estimator, + get_search_params) warnings.filterwarnings('ignore') @@ -51,6 +42,8 @@ with open(inputs_path, 'r') as param_handler: params = json.load(param_handler) + estimator_type = params['algo_selection']['estimator_type'] + # get base estimators base_estimators = [] for idx, base_file in enumerate(base_paths.split(',')): if base_file and base_file != 'None': @@ -60,14 +53,23 @@ estimator_json = (params['base_est_builder'][idx] ['estimator_selector']) model = get_estimator(estimator_json) - base_estimators.append(model) + + if estimator_type.startswith('sklearn'): + named = model.__class__.__name__.lower() + named = 'base_%d_%s' % (idx, named) + base_estimators.append((named, model)) + else: + base_estimators.append(model) - if meta_path: - with open(meta_path, 'rb') as f: - meta_estimator = load_model(f) - else: - estimator_json = params['meta_estimator']['estimator_selector'] - meta_estimator = get_estimator(estimator_json) + # get meta estimator, if applicable + if estimator_type.startswith('mlxtend'): + if meta_path: + with open(meta_path, 'rb') as f: + meta_estimator = load_model(f) + else: + estimator_json = (params['algo_selection'] + ['meta_estimator']['estimator_selector']) + meta_estimator = get_estimator(estimator_json) options = params['algo_selection']['options'] @@ -78,26 +80,26 @@ # set n_jobs options['n_jobs'] = N_JOBS - if params['algo_selection']['estimator_type'] == 'StackingCVClassifier': - ensemble_estimator = StackingCVClassifier( + weights = options.pop('weights', None) + if weights: + options['weights'] = ast.literal_eval(weights) + + mod_and_name = estimator_type.split('_') + mod = sys.modules[mod_and_name[0]] + klass = getattr(mod, mod_and_name[1]) + + if estimator_type.startswith('sklearn'): + options['n_jobs'] = N_JOBS + ensemble_estimator = klass(base_estimators, **options) + + elif mod == mlxtend.classifier: + ensemble_estimator = klass( classifiers=base_estimators, meta_classifier=meta_estimator, **options) - elif params['algo_selection']['estimator_type'] == 'StackingClassifier': - ensemble_estimator = StackingClassifier( - classifiers=base_estimators, - meta_classifier=meta_estimator, - **options) - - elif params['algo_selection']['estimator_type'] == 'StackingCVRegressor': - ensemble_estimator = StackingCVRegressor( - regressors=base_estimators, - meta_regressor=meta_estimator, - **options) - else: - ensemble_estimator = StackingRegressor( + ensemble_estimator = klass( regressors=base_estimators, meta_regressor=meta_estimator, **options) |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/RandomForestClassifier.zip |
| b |
| Binary file test-data/RandomForestClassifier.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/StackingCVRegressor01.zip |
| b |
| Binary file test-data/StackingCVRegressor01.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/StackingRegressor02.zip |
| b |
| Binary file test-data/StackingRegressor02.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/StackingVoting03.zip |
| b |
| Binary file test-data/StackingVoting03.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/deepsear_1feature.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/deepsear_1feature.json Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,1 @@ +{"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "batch_input_shape": [null, 1000, 4], "dtype": "float32", "filters": 320, "kernel_size": [8], "strides": [1], "padding": "valid", "data_format": "channels_last", "dilation_rate": [1], "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "MaxPooling1D", "config": {"name": "max_pooling1d_1", "trainable": true, "strides": [4], "pool_size": [4], "padding": "valid", "data_format": "channels_last"}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": 999}}, {"class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "filters": 480, "kernel_size": [8], "strides": [1], "padding": "valid", "data_format": "channels_last", "dilation_rate": [1], "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "MaxPooling1D", "config": {"name": "max_pooling1d_2", "trainable": true, "strides": [4], "pool_size": [4], "padding": "valid", "data_format": "channels_last"}}, {"class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "rate": 0.2, "noise_shape": null, "seed": 999}}, {"class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "filters": 960, "kernel_size": [8], "strides": [1], "padding": "valid", "data_format": "channels_last", "dilation_rate": [1], "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_3", "trainable": true, "rate": 0.5, "noise_shape": null, "seed": 999}}, {"class_name": "Reshape", "config": {"name": "reshape_1", "trainable": true, "target_shape": [50880]}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "units": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/fitted_keras_g_regressor01.zip |
| b |
| Binary file test-data/fitted_keras_g_regressor01.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras01.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras01.json Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,1 @@ +{"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 784], "dtype": "float32", "units": 32, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Activation", "config": {"name": "activation_1", "trainable": true, "activation": "relu"}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 10, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": true, "activation": "softmax"}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} \ No newline at end of file |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras02.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras02.json Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,1 @@ +{"class_name": "Model", "config": {"name": "model_1", "layers": [{"name": "main_input", "class_name": "InputLayer", "config": {"batch_input_shape": [null, 100], "dtype": "int32", "sparse": false, "name": "main_input"}, "inbound_nodes": []}, {"name": "embedding_1", "class_name": "Embedding", "config": {"name": "embedding_1", "trainable": true, "batch_input_shape": [null, 100], "dtype": "float32", "input_dim": 10000, "output_dim": 512, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": 100}, "inbound_nodes": [[["main_input", 0, 0, {}]]]}, {"name": "lstm_1", "class_name": "LSTM", "config": {"name": "lstm_1", "trainable": true, "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "units": 32, "activation": "linear", "recurrent_activation": "hard_sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 1}, "inbound_nodes": [[["embedding_1", 0, 0, {}]]]}, {"name": "dense_1", "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["lstm_1", 0, 0, {}]]]}, {"name": "aux_input", "class_name": "InputLayer", "config": {"batch_input_shape": [null, 5], "dtype": "float32", "sparse": false, "name": "aux_input"}, "inbound_nodes": []}, {"name": "concatenate_1", "class_name": "Concatenate", "config": {"name": "concatenate_1", "trainable": true, "axis": -1}, "inbound_nodes": [[["dense_1", 0, 0, {}], ["aux_input", 0, 0, {}]]]}, {"name": "dense_2", "class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 64, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["concatenate_1", 0, 0, {}]]]}, {"name": "dense_3", "class_name": "Dense", "config": {"name": "dense_3", "trainable": true, "units": 64, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dense_2", 0, 0, {}]]]}, {"name": "dense_4", "class_name": "Dense", "config": {"name": "dense_4", "trainable": true, "units": 64, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dense_3", 0, 0, {}]]]}, {"name": "dense_5", "class_name": "Dense", "config": {"name": "dense_5", "trainable": true, "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["dense_4", 0, 0, {}]]]}], "input_layers": [["main_input", 0, 0], ["aux_input", 0, 0]], "output_layers": [["dense_1", 0, 0], ["dense_5", 0, 0]]}, "keras_version": "2.2.4", "backend": "tensorflow"} \ No newline at end of file |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras03.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras03.json Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,1 @@ +{"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 17], "dtype": "float32", "units": 100, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": 0}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "rate": 0.1, "noise_shape": null, "seed": 0}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": 0}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} \ No newline at end of file |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras04.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras04.json Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,1 @@ +{"class_name": "Sequential", "config": {"name": "sequential_1", "layers": [{"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "batch_input_shape": [null, 17], "dtype": "float32", "units": 32, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Activation", "config": {"name": "activation_1", "trainable": true, "activation": "linear"}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "units": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "VarianceScaling", "config": {"scale": 1.0, "mode": "fan_avg", "distribution": "uniform", "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": true, "activation": "linear"}}]}, "keras_version": "2.2.4", "backend": "tensorflow"} \ No newline at end of file |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_batch_model01 |
| b |
| Binary file test-data/keras_batch_model01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_batch_model02 |
| b |
| Binary file test-data/keras_batch_model02 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_batch_model03 |
| b |
| Binary file test-data/keras_batch_model03 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_batch_params01.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras_batch_params01.tabular Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,94 @@ + Parameter Value +@ amsgrad amsgrad: None +@ batch_size batch_size: 32 +@ beta_1 beta_1: None +@ beta_2 beta_2: None +@ callbacks callbacks: [{'callback_selection': {'callback_type': 'None'}}] +@ class_positive_factor class_positive_factor: 1.0 +@ config config: {'name': 'sequential_1', 'layers': [{'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable +@ data_batch_generator "data_batch_generator: FastaDNABatchGenerator(fasta_path='to_be_determined', seed=999, + seq_length=1000, shuffle=True)" +@ decay decay: 0.0 +@ epochs epochs: 100 +@ epsilon epsilon: None +@ layers_0_Dense layers_0_Dense: {'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'batch_input_shape': [None, +@ layers_1_Activation layers_1_Activation: {'class_name': 'Activation', 'config': {'name': 'activation_1', 'trainable': True, 'activation': 're +@ layers_2_Dense layers_2_Dense: {'class_name': 'Dense', 'config': {'name': 'dense_2', 'trainable': True, 'units': 10, 'activation': +@ layers_3_Activation layers_3_Activation: {'class_name': 'Activation', 'config': {'name': 'activation_2', 'trainable': True, 'activation': 'so +@ loss loss: 'binary_crossentropy' +@ lr lr: 0.01 +@ metrics metrics: ['acc'] +@ model_type model_type: 'sequential' +@ momentum momentum: 0.0 +* n_jobs n_jobs: 1 +@ nesterov nesterov: False +@ optimizer optimizer: 'sgd' +@ prediction_steps prediction_steps: None +@ rho rho: None +@ schedule_decay schedule_decay: None +@ seed seed: None +@ steps_per_epoch steps_per_epoch: None +@ validation_data validation_data: None +@ validation_steps validation_steps: None +@ verbose verbose: 0 +* data_batch_generator__fasta_path data_batch_generator__fasta_path: 'to_be_determined' +@ data_batch_generator__seed data_batch_generator__seed: 999 +@ data_batch_generator__seq_length data_batch_generator__seq_length: 1000 +@ data_batch_generator__shuffle data_batch_generator__shuffle: True +* layers_0_Dense__class_name layers_0_Dense__class_name: 'Dense' +@ layers_0_Dense__config layers_0_Dense__config: {'name': 'dense_1', 'trainable': True, 'batch_input_shape': [None, 784], 'dtype': 'float32', 'units' +@ layers_0_Dense__config__activation layers_0_Dense__config__activation: 'linear' +@ layers_0_Dense__config__activity_regularizer layers_0_Dense__config__activity_regularizer: None +@ layers_0_Dense__config__batch_input_shape layers_0_Dense__config__batch_input_shape: [None, 784] +@ layers_0_Dense__config__bias_constraint layers_0_Dense__config__bias_constraint: None +@ layers_0_Dense__config__bias_initializer layers_0_Dense__config__bias_initializer: {'class_name': 'Zeros', 'config': {}} +* layers_0_Dense__config__bias_initializer__class_name layers_0_Dense__config__bias_initializer__class_name: 'Zeros' +@ layers_0_Dense__config__bias_initializer__config layers_0_Dense__config__bias_initializer__config: {} +@ layers_0_Dense__config__bias_regularizer layers_0_Dense__config__bias_regularizer: None +@ layers_0_Dense__config__dtype layers_0_Dense__config__dtype: 'float32' +@ layers_0_Dense__config__kernel_constraint layers_0_Dense__config__kernel_constraint: None +@ layers_0_Dense__config__kernel_initializer layers_0_Dense__config__kernel_initializer: {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'unifo +* layers_0_Dense__config__kernel_initializer__class_name layers_0_Dense__config__kernel_initializer__class_name: 'VarianceScaling' +@ layers_0_Dense__config__kernel_initializer__config layers_0_Dense__config__kernel_initializer__config: {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None} +@ layers_0_Dense__config__kernel_initializer__config__distribution layers_0_Dense__config__kernel_initializer__config__distribution: 'uniform' +@ layers_0_Dense__config__kernel_initializer__config__mode layers_0_Dense__config__kernel_initializer__config__mode: 'fan_avg' +@ layers_0_Dense__config__kernel_initializer__config__scale layers_0_Dense__config__kernel_initializer__config__scale: 1.0 +@ layers_0_Dense__config__kernel_initializer__config__seed layers_0_Dense__config__kernel_initializer__config__seed: None +@ layers_0_Dense__config__kernel_regularizer layers_0_Dense__config__kernel_regularizer: None +* layers_0_Dense__config__name layers_0_Dense__config__name: 'dense_1' +@ layers_0_Dense__config__trainable layers_0_Dense__config__trainable: True +@ layers_0_Dense__config__units layers_0_Dense__config__units: 32 +@ layers_0_Dense__config__use_bias layers_0_Dense__config__use_bias: True +* layers_1_Activation__class_name layers_1_Activation__class_name: 'Activation' +@ layers_1_Activation__config layers_1_Activation__config: {'name': 'activation_1', 'trainable': True, 'activation': 'relu'} +@ layers_1_Activation__config__activation layers_1_Activation__config__activation: 'relu' +* layers_1_Activation__config__name layers_1_Activation__config__name: 'activation_1' +@ layers_1_Activation__config__trainable layers_1_Activation__config__trainable: True +* layers_2_Dense__class_name layers_2_Dense__class_name: 'Dense' +@ layers_2_Dense__config layers_2_Dense__config: {'name': 'dense_2', 'trainable': True, 'units': 10, 'activation': 'linear', 'use_bias': True, 'kerne +@ layers_2_Dense__config__activation layers_2_Dense__config__activation: 'linear' +@ layers_2_Dense__config__activity_regularizer layers_2_Dense__config__activity_regularizer: None +@ layers_2_Dense__config__bias_constraint layers_2_Dense__config__bias_constraint: None +@ layers_2_Dense__config__bias_initializer layers_2_Dense__config__bias_initializer: {'class_name': 'Zeros', 'config': {}} +* layers_2_Dense__config__bias_initializer__class_name layers_2_Dense__config__bias_initializer__class_name: 'Zeros' +@ layers_2_Dense__config__bias_initializer__config layers_2_Dense__config__bias_initializer__config: {} +@ layers_2_Dense__config__bias_regularizer layers_2_Dense__config__bias_regularizer: None +@ layers_2_Dense__config__kernel_constraint layers_2_Dense__config__kernel_constraint: None +@ layers_2_Dense__config__kernel_initializer layers_2_Dense__config__kernel_initializer: {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'unifo +* layers_2_Dense__config__kernel_initializer__class_name layers_2_Dense__config__kernel_initializer__class_name: 'VarianceScaling' +@ layers_2_Dense__config__kernel_initializer__config layers_2_Dense__config__kernel_initializer__config: {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None} +@ layers_2_Dense__config__kernel_initializer__config__distribution layers_2_Dense__config__kernel_initializer__config__distribution: 'uniform' +@ layers_2_Dense__config__kernel_initializer__config__mode layers_2_Dense__config__kernel_initializer__config__mode: 'fan_avg' +@ layers_2_Dense__config__kernel_initializer__config__scale layers_2_Dense__config__kernel_initializer__config__scale: 1.0 +@ layers_2_Dense__config__kernel_initializer__config__seed layers_2_Dense__config__kernel_initializer__config__seed: None +@ layers_2_Dense__config__kernel_regularizer layers_2_Dense__config__kernel_regularizer: None +* layers_2_Dense__config__name layers_2_Dense__config__name: 'dense_2' +@ layers_2_Dense__config__trainable layers_2_Dense__config__trainable: True +@ layers_2_Dense__config__units layers_2_Dense__config__units: 10 +@ layers_2_Dense__config__use_bias layers_2_Dense__config__use_bias: True +* layers_3_Activation__class_name layers_3_Activation__class_name: 'Activation' +@ layers_3_Activation__config layers_3_Activation__config: {'name': 'activation_2', 'trainable': True, 'activation': 'softmax'} +@ layers_3_Activation__config__activation layers_3_Activation__config__activation: 'softmax' +* layers_3_Activation__config__name layers_3_Activation__config__name: 'activation_2' +@ layers_3_Activation__config__trainable layers_3_Activation__config__trainable: True + Note: @, params eligible for search in searchcv tool. |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_model01 |
| b |
| Binary file test-data/keras_model01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_model02 |
| b |
| Binary file test-data/keras_model02 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_model04 |
| b |
| Binary file test-data/keras_model04 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_params04.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/keras_params04.tabular Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| @@ -0,0 +1,85 @@ + Parameter Value +@ amsgrad amsgrad: False +@ batch_size batch_size: 32 +@ beta_1 beta_1: 0.9 +@ beta_2 beta_2: 0.999 +@ callbacks callbacks: [{'callback_selection': {'callback_type': 'None'}}] +@ config config: {'name': 'sequential_1', 'layers': [{'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable +@ decay decay: 0.0 +@ epochs epochs: 100 +@ epsilon epsilon: None +@ layers_0_Dense layers_0_Dense: {'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'batch_input_shape': [None, +@ layers_1_Activation layers_1_Activation: {'class_name': 'Activation', 'config': {'name': 'activation_1', 'trainable': True, 'activation': 'li +@ layers_2_Dense layers_2_Dense: {'class_name': 'Dense', 'config': {'name': 'dense_2', 'trainable': True, 'units': 1, 'activation': ' +@ layers_3_Activation layers_3_Activation: {'class_name': 'Activation', 'config': {'name': 'activation_2', 'trainable': True, 'activation': 'li +@ loss loss: 'mean_squared_error' +@ lr lr: 0.001 +@ metrics metrics: ['mse'] +@ model_type model_type: 'sequential' +@ momentum momentum: None +@ nesterov nesterov: None +@ optimizer optimizer: 'adam' +@ rho rho: None +@ schedule_decay schedule_decay: None +@ seed seed: 42 +@ steps_per_epoch steps_per_epoch: None +@ validation_data validation_data: None +@ validation_steps validation_steps: None +@ verbose verbose: 0 +* layers_0_Dense__class_name layers_0_Dense__class_name: 'Dense' +@ layers_0_Dense__config layers_0_Dense__config: {'name': 'dense_1', 'trainable': True, 'batch_input_shape': [None, 17], 'dtype': 'float32', 'units': +@ layers_0_Dense__config__activation layers_0_Dense__config__activation: 'linear' +@ layers_0_Dense__config__activity_regularizer layers_0_Dense__config__activity_regularizer: None +@ layers_0_Dense__config__batch_input_shape layers_0_Dense__config__batch_input_shape: [None, 17] +@ layers_0_Dense__config__bias_constraint layers_0_Dense__config__bias_constraint: None +@ layers_0_Dense__config__bias_initializer layers_0_Dense__config__bias_initializer: {'class_name': 'Zeros', 'config': {}} +* layers_0_Dense__config__bias_initializer__class_name layers_0_Dense__config__bias_initializer__class_name: 'Zeros' +@ layers_0_Dense__config__bias_initializer__config layers_0_Dense__config__bias_initializer__config: {} +@ layers_0_Dense__config__bias_regularizer layers_0_Dense__config__bias_regularizer: None +@ layers_0_Dense__config__dtype layers_0_Dense__config__dtype: 'float32' +@ layers_0_Dense__config__kernel_constraint layers_0_Dense__config__kernel_constraint: None +@ layers_0_Dense__config__kernel_initializer layers_0_Dense__config__kernel_initializer: {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'unifo +* layers_0_Dense__config__kernel_initializer__class_name layers_0_Dense__config__kernel_initializer__class_name: 'VarianceScaling' +@ layers_0_Dense__config__kernel_initializer__config layers_0_Dense__config__kernel_initializer__config: {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None} +@ layers_0_Dense__config__kernel_initializer__config__distribution layers_0_Dense__config__kernel_initializer__config__distribution: 'uniform' +@ layers_0_Dense__config__kernel_initializer__config__mode layers_0_Dense__config__kernel_initializer__config__mode: 'fan_avg' +@ layers_0_Dense__config__kernel_initializer__config__scale layers_0_Dense__config__kernel_initializer__config__scale: 1.0 +@ layers_0_Dense__config__kernel_initializer__config__seed layers_0_Dense__config__kernel_initializer__config__seed: None +@ layers_0_Dense__config__kernel_regularizer layers_0_Dense__config__kernel_regularizer: None +* layers_0_Dense__config__name layers_0_Dense__config__name: 'dense_1' +@ layers_0_Dense__config__trainable layers_0_Dense__config__trainable: True +@ layers_0_Dense__config__units layers_0_Dense__config__units: 32 +@ layers_0_Dense__config__use_bias layers_0_Dense__config__use_bias: True +* layers_1_Activation__class_name layers_1_Activation__class_name: 'Activation' +@ layers_1_Activation__config layers_1_Activation__config: {'name': 'activation_1', 'trainable': True, 'activation': 'linear'} +@ layers_1_Activation__config__activation layers_1_Activation__config__activation: 'linear' +* layers_1_Activation__config__name layers_1_Activation__config__name: 'activation_1' +@ layers_1_Activation__config__trainable layers_1_Activation__config__trainable: True +* layers_2_Dense__class_name layers_2_Dense__class_name: 'Dense' +@ layers_2_Dense__config layers_2_Dense__config: {'name': 'dense_2', 'trainable': True, 'units': 1, 'activation': 'linear', 'use_bias': True, 'kernel +@ layers_2_Dense__config__activation layers_2_Dense__config__activation: 'linear' +@ layers_2_Dense__config__activity_regularizer layers_2_Dense__config__activity_regularizer: None +@ layers_2_Dense__config__bias_constraint layers_2_Dense__config__bias_constraint: None +@ layers_2_Dense__config__bias_initializer layers_2_Dense__config__bias_initializer: {'class_name': 'Zeros', 'config': {}} +* layers_2_Dense__config__bias_initializer__class_name layers_2_Dense__config__bias_initializer__class_name: 'Zeros' +@ layers_2_Dense__config__bias_initializer__config layers_2_Dense__config__bias_initializer__config: {} +@ layers_2_Dense__config__bias_regularizer layers_2_Dense__config__bias_regularizer: None +@ layers_2_Dense__config__kernel_constraint layers_2_Dense__config__kernel_constraint: None +@ layers_2_Dense__config__kernel_initializer layers_2_Dense__config__kernel_initializer: {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'unifo +* layers_2_Dense__config__kernel_initializer__class_name layers_2_Dense__config__kernel_initializer__class_name: 'VarianceScaling' +@ layers_2_Dense__config__kernel_initializer__config layers_2_Dense__config__kernel_initializer__config: {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None} +@ layers_2_Dense__config__kernel_initializer__config__distribution layers_2_Dense__config__kernel_initializer__config__distribution: 'uniform' +@ layers_2_Dense__config__kernel_initializer__config__mode layers_2_Dense__config__kernel_initializer__config__mode: 'fan_avg' +@ layers_2_Dense__config__kernel_initializer__config__scale layers_2_Dense__config__kernel_initializer__config__scale: 1.0 +@ layers_2_Dense__config__kernel_initializer__config__seed layers_2_Dense__config__kernel_initializer__config__seed: None +@ layers_2_Dense__config__kernel_regularizer layers_2_Dense__config__kernel_regularizer: None +* layers_2_Dense__config__name layers_2_Dense__config__name: 'dense_2' +@ layers_2_Dense__config__trainable layers_2_Dense__config__trainable: True +@ layers_2_Dense__config__units layers_2_Dense__config__units: 1 +@ layers_2_Dense__config__use_bias layers_2_Dense__config__use_bias: True +* layers_3_Activation__class_name layers_3_Activation__class_name: 'Activation' +@ layers_3_Activation__config layers_3_Activation__config: {'name': 'activation_2', 'trainable': True, 'activation': 'linear'} +@ layers_3_Activation__config__activation layers_3_Activation__config__activation: 'linear' +* layers_3_Activation__config__name layers_3_Activation__config__name: 'activation_2' +@ layers_3_Activation__config__trainable layers_3_Activation__config__trainable: True + Note: @, params eligible for search in searchcv tool. |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_prefitted01.zip |
| b |
| Binary file test-data/keras_prefitted01.zip has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/keras_save_weights01.h5 |
| b |
| Binary file test-data/keras_save_weights01.h5 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/model_pred01.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/model_pred01.tabular Fri Aug 09 07:22:24 2019 -0400 |
| b |
| @@ -0,0 +1,262 @@ +Predicted +71.129364 +60.96111 +77.885765 +57.212738 +51.806957 +52.089592 +51.571884 +80.762184 +36.772987 +41.643093 +46.386948 +77.97063 +72.768776 +40.0386 +79.81385 +74.40216 +52.089592 +75.51107 +55.705868 +39.944202 +49.643826 +59.17941 +69.848915 +64.62096 +48.310116 +43.391766 +68.25893 +60.198105 +65.16974 +72.130005 +56.351482 +53.20132 +56.86578 +54.342987 +43.521133 +59.663773 +66.097626 +51.960022 +41.559486 +45.16049 +66.40008 +71.488754 +45.16049 +63.34996 +69.83631 +55.652687 +61.311596 +71.85501 +75.12588 +54.93247 +70.09855 +74.20223 +57.898273 +55.23022 +75.70524 +66.94729 +65.12762 +59.3189 +61.22922 +61.2382 +54.017147 +51.633373 +51.633373 +65.16974 +65.16873 +57.874527 +59.740753 +43.990814 +66.06423 +64.436615 +41.245773 +63.278465 +63.27533 +71.13793 +65.47819 +72.620995 +62.598015 +36.986706 +73.2002 +71.966644 +72.912926 +75.46711 +55.12616 +46.19641 +87.20736 +72.11753 +57.952766 +84.67858 +69.21688 +64.257095 +43.59384 +44.723145 +67.051605 +50.021965 +69.202095 +75.10072 +70.80699 +83.08025 +69.62026 +42.441116 +64.38655 +59.430386 +69.366035 +73.87479 +59.973484 +75.76153 +56.195892 +71.16636 +60.419106 +61.630756 +51.81593 +54.924137 +60.73048 +78.496635 +77.921555 +73.66453 +60.904953 +71.26717 +72.01454 +53.52841 +46.66952 +54.504898 +56.28563 +59.398067 +72.71433 +51.745968 +67.80466 +51.571823 +52.010742 +54.19355 +74.193825 +64.57627 +67.48214 +68.41867 +82.102806 +55.8638 +76.90198 +62.577324 +73.70229 +78.93923 +73.51925 +54.81887 +65.2422 +59.700085 +84.08965 +64.35592 +54.001873 +41.397793 +64.64837 +62.784557 +42.990005 +45.430832 +52.089592 +60.374348 +51.67288 +62.4257 +79.536285 +76.4169 +55.978775 +74.43581 +76.89248 +65.3203 +72.10233 +59.23278 +51.736633 +73.13266 +59.45746 +73.0939 +70.58273 +53.08009 +49.893116 +73.89228 +52.64392 +54.801548 +63.534626 +68.1002 +63.70472 +63.8851 +63.268097 +62.438057 +61.989746 +71.47914 +73.92875 +48.089043 +54.874943 +50.261494 +69.11724 +57.448387 +50.528027 +58.67657 +73.969376 +53.745205 +74.81751 +85.582954 +75.10767 +48.855537 +70.66616 +41.341694 +48.55276 +63.48302 +73.02358 +69.50546 +55.603634 +74.26824 +76.03213 +62.601646 +81.99045 +59.26651 +44.504597 +53.54178 +55.247334 +82.123795 +51.84111 +66.27524 +66.23033 +58.565033 +67.452 +72.54107 +49.840427 +70.26608 +62.447872 +67.045 +42.600086 +64.88309 +55.31232 +39.07865 +71.81975 +59.447086 +53.20132 +75.12621 +72.9902 +53.1043 +72.42816 +72.10233 +55.836628 +53.2467 +74.670074 +74.5721 +54.103737 +49.212822 +67.238785 +60.09495 +74.5011 +63.0043 +67.7362 +53.029213 +74.860016 +78.597946 +75.369064 +60.000134 +68.83947 +40.24504 +81.21449 +61.465557 +42.74572 +52.089592 +73.162025 +52.033802 +79.690926 +62.542553 +59.557045 |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/model_pred02.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/model_pred02.tabular Fri Aug 09 07:22:24 2019 -0400 |
| b |
| @@ -0,0 +1,262 @@ +Predicted +71.08584 +61.23427 +75.80197 +66.19323 +52.35754 +52.987312 +51.777576 +75.66966 +49.61427 +51.20531 +49.255173 +76.143936 +74.00767 +50.80104 +72.37281 +68.69481 +52.816956 +76.27541 +57.82054 +49.72029 +52.400383 +57.968666 +61.28138 +58.4683 +53.114418 +50.45093 +67.63649 +60.31344 +66.52325 +72.48887 +58.755577 +53.332912 +55.175415 +53.437675 +50.452156 +61.153603 +66.69711 +51.1279 +51.37375 +50.732525 +67.677734 +74.2334 +51.287792 +70.154366 +68.460396 +58.35005 +59.828957 +74.98557 +73.3624 +54.043793 +73.04924 +77.22285 +59.452316 +56.143288 +74.41183 +60.254143 +67.18662 +63.53044 +60.43683 +60.07025 +57.257767 +52.143753 +52.872334 +67.748436 +63.986977 +55.532387 +59.70022 +49.43772 +65.30266 +67.30055 +49.907486 +57.864845 +56.207542 +70.46542 +55.503044 +73.822784 +63.741142 +49.693428 +71.36254 +71.87617 +72.02608 +65.63652 +54.059746 +51.300495 +76.06125 +73.98534 +63.071587 +75.93381 +69.479454 +63.85415 +51.218174 +49.468956 +68.23912 +50.83457 +70.77809 +72.129776 +74.53812 +68.9107 +72.47451 +50.62992 +62.99655 +56.105698 +72.927025 +65.86492 +58.282486 +75.063446 +54.558403 +65.59456 +57.257263 +58.336494 +51.988983 +57.355415 +56.631332 +62.632957 +76.11209 +76.99285 +65.670746 +74.464355 +68.042145 +54.761986 +51.070145 +56.55138 +55.53712 +57.753426 +75.02803 +57.397556 +71.05187 +51.134808 +53.119152 +52.581924 +70.8574 +66.85955 +67.29634 +66.589584 +76.06389 +54.559666 +60.37111 +63.455887 +72.6416 +75.51883 +63.990837 +53.491386 +59.82952 +60.56826 +76.53373 +66.729385 +52.592728 +48.729107 +68.03414 +56.391117 +50.800247 +50.053703 +52.03207 +55.326523 +52.58854 +60.38707 +75.923096 +75.2882 +54.893684 +78.00183 +76.06732 +60.791916 +70.38205 +60.582397 +53.582005 +77.20325 +54.903778 +68.63178 +70.27207 +54.5502 +53.928703 +74.93919 +52.267735 +51.70433 +59.89312 +74.00166 +66.61868 +70.04806 +55.62455 +65.638214 +55.330837 +65.8484 +65.45604 +50.942883 +56.04741 +52.147808 +69.9472 +52.90547 +51.568893 +57.65322 +76.28175 +53.421043 +73.63155 +77.357666 +77.49912 +51.669907 +67.80663 +49.745773 +52.792336 +62.308838 +76.21391 +70.10635 +53.58763 +76.36336 +75.63791 +66.51898 +59.851395 +53.114918 +50.095005 +54.76951 +58.387985 +76.39301 +53.754196 +66.004395 +59.4105 +53.724583 +63.857407 +70.29119 +50.46862 +58.864563 +61.946457 +70.4472 +50.738815 +65.65154 +52.600437 +49.42977 +70.38036 +56.012196 +53.824024 +71.119225 +75.3495 +49.078987 +74.36192 +71.18959 +54.9702 +54.477818 +72.231705 +68.62958 +52.298077 +52.34682 +70.110405 +60.08683 +74.98835 +55.85307 +66.53965 +53.608902 +67.770744 +66.93648 +68.07121 +59.94021 +58.784706 +50.237366 +77.0887 +65.06997 +50.1484 +51.08928 +74.907234 +56.82161 +62.303955 +62.67704 +61.49601 |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline01 |
| b |
| Binary file test-data/pipeline01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline02 |
| b |
| Binary file test-data/pipeline02 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline03 |
| b |
| Binary file test-data/pipeline03 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline04 |
| b |
| Binary file test-data/pipeline04 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline05 |
| b |
| Binary file test-data/pipeline05 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline06 |
| b |
| Binary file test-data/pipeline06 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline07 |
| b |
| Binary file test-data/pipeline07 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline08 |
| b |
| Binary file test-data/pipeline08 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline09 |
| b |
| Binary file test-data/pipeline09 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline10 |
| b |
| Binary file test-data/pipeline10 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline11 |
| b |
| Binary file test-data/pipeline11 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline12 |
| b |
| Binary file test-data/pipeline12 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline13 |
| b |
| Binary file test-data/pipeline13 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline14 |
| b |
| Binary file test-data/pipeline14 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline15 |
| b |
| Binary file test-data/pipeline15 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/pipeline16 |
| b |
| Binary file test-data/pipeline16 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model01 |
| b |
| Binary file test-data/prp_model01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model02 |
| b |
| Binary file test-data/prp_model02 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model04 |
| b |
| Binary file test-data/prp_model04 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model05 |
| b |
| Binary file test-data/prp_model05 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model07 |
| b |
| Binary file test-data/prp_model07 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model08 |
| b |
| Binary file test-data/prp_model08 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_model09 |
| b |
| Binary file test-data/prp_model09 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/prp_result10 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/prp_result10 Fri Aug 09 07:22:24 2019 -0400 |
| b |
| b'@@ -0,0 +1,262 @@\n+year\tmonth\tday\ttemp_2\ttemp_1\taverage\tforecast_noaa\tforecast_acc\tforecast_under\tfriend\tweek_Fri\tweek_Mon\tweek_Sat\tweek_Sun\tweek_Thurs\tweek_Tues\tweek_Wed\n+-1.0\t0.4545454545454546\t0.19999999999999996\t0.22222222222222188\t-0.17073170731707288\t0.5232198142414863\t0.33333333333333304\t0.6000000000000001\t0.5428571428571427\t0.791044776119403\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.4545454545454546\t-0.1333333333333333\t-0.07407407407407396\t-0.41463414634146334\t-0.195046439628483\t-0.11111111111111116\t-0.02857142857142847\t-0.20000000000000018\t0.13432835820895517\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t0.09090909090909083\t0.9333333333333333\t0.8518518518518516\t0.29268292682926855\t0.9938080495356032\t0.8888888888888884\t0.8857142857142857\t0.8857142857142852\t0.25373134328358193\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.2727272727272727\t-0.06666666666666665\t0.7407407407407405\t-0.26829268292682906\t0.21362229102167207\t0.22222222222222232\t0.31428571428571406\t0.1428571428571428\t-0.10447761194029859\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-1.0\t0.1333333333333333\t-0.2962962962962963\t-0.6341463414634145\t-0.8513931888544892\t-0.8333333333333335\t-0.8857142857142857\t-0.7142857142857144\t-0.10447761194029859\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-1.0\t0.6000000000000001\t-0.5185185185185186\t-0.6097560975609755\t-0.8080495356037152\t-0.7777777777777777\t-0.7142857142857144\t-0.7142857142857144\t0.04477611940298498\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.8181818181818181\t0.6000000000000001\t-0.4814814814814816\t-0.5853658536585364\t-0.7832817337461302\t-0.7777777777777777\t-0.657142857142857\t-0.8285714285714287\t-0.6119402985074627\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.09090909090909083\t0.2666666666666666\t0.40740740740740744\t0.048780487804878314\t0.956656346749226\t0.8888888888888884\t0.8285714285714287\t0.8857142857142852\t0.13432835820895517\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.9999999999999998\t0.06666666666666665\t-0.8518518518518519\t-0.9999999999999999\t-0.9938080495356036\t-0.8888888888888888\t-0.9428571428571431\t-0.8857142857142857\t-0.7014925373134329\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.9999999999999998\t-0.5333333333333333\t-0.7407407407407409\t-0.8780487804878048\t-0.9380804953560373\t-0.7777777777777777\t-0.7142857142857144\t-0.8285714285714287\t-0.7611940298507462\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t0.9999999999999998\t0.8\t-0.7407407407407409\t-0.7073170731707314\t-0.9876160990712077\t-1.0\t-0.8285714285714287\t-1.0\t-0.10447761194029859\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.09090909090909083\t0.06666666666666665\t0.5185185185185186\t-0.09756097560975596\t0.9318885448916405\t0.9444444444444446\t0.8285714285714287\t0.8857142857142852\t0.791044776119403\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.09090909090909083\t-0.6\t0.2592592592592591\t2.220446049250313e-16\t0.8142414860681115\t0.7777777777777777\t0.7714285714285709\t0.7142857142857144\t0.31343283582089554\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t0.9999999999999998\t-0.06666666666666665\t-0.8148148148148149\t-0.9024390243902437\t-0.9876160990712077\t-0.7777777777777777\t-0.8285714285714287\t-0.8285714285714287\t-0.4626865671641791\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t-0.09090909090909105\t0.7333333333333334\t0.33333333333333304\t0.048780487804878314\t0.6780185758513935\t0.6111111111111112\t0.6000000000000001\t0.5999999999999996\t0.6716417910447763\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.2727272727272727\t0.9999999999999998\t0.07407407407407396\t-0.12195121951219501\t0.3746130030959747\t0.22222222222222232\t0.4857142857142853\t0.37142857142857144\t0.7014925373134326\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\n+-1.0\t-1.0\t0.2666666666666666\t-0.2962962962962963\t-0.6829268292682924\t-0.8390092879256965\t-0.8333333333333335\t-0.657142857142857\t-0.7142857142857144\t-0.014925373134328401\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.2727272727272727\t-0.4\t0.40740740740740744\t-0.09756097560975596\t0.9752321981424141\t1.0\t0.8285714285714287\t0.8857142857142852\t0.19402985074626877\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t-0.6363636363636365\t0.4666666666666'..b'444444444442\t0.024390243902439268\t0.888544891640866\t0.833333333333333\t0.8285714285714287\t0.8285714285714283\t-0.16417910447761197\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.8181818181818181\t-0.6\t-0.14814814814814836\t-0.36585365853658525\t-0.51702786377709\t-0.44444444444444464\t-0.4285714285714288\t-0.6000000000000001\t-0.791044776119403\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.6363636363636365\t-1.0\t0.14814814814814792\t-0.21951219512195097\t0.2507739938080493\t0.2777777777777777\t0.37142857142857144\t0.1428571428571428\t-0.22388059701492535\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t0.8181818181818181\t0.3999999999999999\t-0.2592592592592595\t-0.5365853658536583\t-0.7399380804953566\t-0.7222222222222223\t-0.5428571428571431\t-0.7142857142857144\t-0.10447761194029859\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\n+-1.0\t-0.09090909090909105\t-1.0\t0.33333333333333304\t0.07317073170731736\t0.3808049535603719\t0.33333333333333304\t0.31428571428571406\t0.2571428571428571\t-0.10447761194029859\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t-0.2727272727272727\t-0.19999999999999996\t0.7037037037037037\t0.024390243902439268\t0.1888544891640862\t0.22222222222222232\t0.19999999999999973\t0.2571428571428571\t0.16417910447761197\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.09090909090909105\t-0.8666666666666667\t0.4814814814814814\t-0.12195121951219501\t0.3993808049535601\t0.2777777777777777\t0.4285714285714284\t0.2571428571428571\t-0.19402985074626855\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.4545454545454546\t-0.2666666666666666\t-0.11111111111111116\t-0.43902439024390216\t-0.21981424148606798\t-0.2777777777777777\t-0.2571428571428571\t-0.2571428571428571\t-0.014925373134328401\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\n+-1.0\t-0.6363636363636365\t0.9999999999999998\t0.07407407407407396\t-0.19512195121951192\t-0.3312693498452015\t-0.22222222222222232\t-0.2571428571428571\t-0.3142857142857145\t-0.16417910447761197\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t0.9999999999999998\t-0.1333333333333333\t-0.7037037037037037\t-0.8780487804878048\t-0.9814241486068114\t-0.7777777777777777\t-0.8857142857142857\t-0.9428571428571431\t-0.3731343283582089\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.2727272727272727\t-0.7333333333333334\t0.4814814814814814\t0.09756097560975618\t0.9938080495356032\t0.8888888888888884\t0.9999999999999996\t0.9428571428571431\t0.28358208955223874\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.2727272727272727\t-0.8\t0.9259259259259256\t-0.04878048780487787\t0.06501547987616041\t0.0\t0.08571428571428541\t0.1428571428571428\t-0.014925373134328401\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\n+-1.0\t0.9999999999999998\t0.9999999999999998\t-0.5185185185185186\t-0.4634146341463412\t-0.975232198142415\t-0.9444444444444446\t-0.8857142857142857\t-0.8285714285714287\t-0.13432835820895517\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-1.0\t0.33333333333333326\t-0.5185185185185186\t-0.5853658536585364\t-0.8328173374613006\t-0.8888888888888888\t-0.7142857142857144\t-0.8857142857142857\t-0.13432835820895517\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\n+-1.0\t0.09090909090909083\t-0.4\t0.4444444444444442\t-0.12195121951219501\t0.8575851393188851\t0.6666666666666665\t0.7714285714285709\t0.8285714285714283\t0.9999999999999998\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.6363636363636365\t-0.06666666666666665\t-0.2962962962962963\t-0.6585365853658536\t-0.47368421052631593\t-0.5555555555555558\t-0.3142857142857145\t-0.5428571428571427\t0.25373134328358193\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\n+-1.0\t-0.4545454545454546\t0.19999999999999996\t0.5555555555555554\t0.3170731707317076\t-0.13931888544891669\t0.0\t-0.02857142857142847\t-0.1428571428571428\t-0.014925373134328401\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t1.0\t-1.0\n+-1.0\t0.6363636363636365\t-0.1333333333333333\t0.14814814814814792\t-0.3902439024390243\t-0.06501547987616085\t-0.16666666666666696\t0.02857142857142847\t-0.08571428571428585\t0.4925373134328359\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n+-1.0\t-0.4545454545454546\t-0.06666666666666665\t-0.11111111111111116\t-0.41463414634146334\t-0.18266253869969074\t-0.0555555555555558\t-0.1428571428571428\t-0.08571428571428585\t-0.6417910447761195\t1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\t-1.0\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/regression_groups.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/regression_groups.tabular Fri Aug 09 07:22:24 2019 -0400 |
| b |
| @@ -0,0 +1,262 @@ +groups +train +train +train +train +test +test +train +train +validation +train +train +train +train +train +validation +validation +train +train +train +test +test +validation +train +validation +test +validation +train +train +train +test +test +test +train +test +train +train +train +test +train +train +train +train +test +train +train +train +train +train +train +train +train +train +test +test +validation +train +validation +train +train +train +train +test +train +train +validation +validation +train +train +train +train +validation +test +test +train +train +train +train +train +train +train +validation +train +train +train +train +test +train +validation +train +test +test +test +train +train +train +test +train +train +train +train +train +train +train +train +train +train +validation +train +train +train +train +validation +train +validation +train +validation +validation +train +validation +train +test +train +train +train +train +test +validation +test +train +train +train +train +test +train +train +train +test +validation +train +train +train +train +train +validation +test +train +train +test +train +train +validation +train +train +train +train +train +test +test +validation +train +test +train +validation +train +train +train +test +train +train +train +train +train +train +validation +train +train +train +train +validation +test +train +train +train +validation +train +test +test +validation +train +validation +validation +test +test +test +train +train +test +train +train +validation +test +test +train +train +train +test +test +train +train +train +train +train +test +train +train +test +validation +test +train +train +test +train +train +train +validation +train +validation +train +validation +train +train +train +validation +validation +test +validation +train +test +train +validation +train +train +test +train +train +test +test +train +validation +train +train +train +train +train +train +train +train +validation +train +test +train |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/searchCV01 |
| b |
| Binary file test-data/searchCV01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/searchCV02 |
| b |
| Binary file test-data/searchCV02 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/train_test_eval01.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/train_test_eval01.tabular Fri Aug 09 07:22:24 2019 -0400 |
| b |
| @@ -0,0 +1,2 @@ +neg_mean_absolute_error r2 +-5.29904520286704 0.6841931628349759 |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/train_test_eval03.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/train_test_eval03.tabular Fri Aug 09 07:22:24 2019 -0400 |
| b |
| @@ -0,0 +1,2 @@ +neg_mean_absolute_error r2 +-4.811320754716981 0.7343422874316201 |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/train_test_eval_model01 |
| b |
| Binary file test-data/train_test_eval_model01 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/train_test_eval_weights01.h5 |
| b |
| Binary file test-data/train_test_eval_weights01.h5 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb test-data/train_test_eval_weights02.h5 |
| b |
| Binary file test-data/train_test_eval_weights02.h5 has changed |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb train_test_eval.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/train_test_eval.py Fri Aug 09 07:22:24 2019 -0400 |
| [ |
| b'@@ -0,0 +1,433 @@\n+import argparse\n+import joblib\n+import json\n+import numpy as np\n+import pandas as pd\n+import pickle\n+import warnings\n+from itertools import chain\n+from scipy.io import mmread\n+from sklearn.base import clone\n+from sklearn import (cluster, compose, decomposition, ensemble,\n+ feature_extraction, feature_selection,\n+ gaussian_process, kernel_approximation, metrics,\n+ model_selection, naive_bayes, neighbors,\n+ pipeline, preprocessing, svm, linear_model,\n+ tree, discriminant_analysis)\n+from sklearn.exceptions import FitFailedWarning\n+from sklearn.metrics.scorer import _check_multimetric_scoring\n+from sklearn.model_selection._validation import _score, cross_validate\n+from sklearn.model_selection import _search, _validation\n+from sklearn.utils import indexable, safe_indexing\n+\n+from galaxy_ml.model_validations import train_test_split\n+from galaxy_ml.utils import (SafeEval, get_scoring, load_model,\n+ read_columns, try_get_attr, get_module)\n+\n+\n+_fit_and_score = try_get_attr(\'galaxy_ml.model_validations\', \'_fit_and_score\')\n+setattr(_search, \'_fit_and_score\', _fit_and_score)\n+setattr(_validation, \'_fit_and_score\', _fit_and_score)\n+\n+N_JOBS = int(__import__(\'os\').environ.get(\'GALAXY_SLOTS\', 1))\n+CACHE_DIR = \'./cached\'\n+NON_SEARCHABLE = (\'n_jobs\', \'pre_dispatch\', \'memory\', \'_path\',\n+ \'nthread\', \'callbacks\')\n+ALLOWED_CALLBACKS = (\'EarlyStopping\', \'TerminateOnNaN\', \'ReduceLROnPlateau\',\n+ \'CSVLogger\', \'None\')\n+\n+\n+def _eval_swap_params(params_builder):\n+ swap_params = {}\n+\n+ for p in params_builder[\'param_set\']:\n+ swap_value = p[\'sp_value\'].strip()\n+ if swap_value == \'\':\n+ continue\n+\n+ param_name = p[\'sp_name\']\n+ if param_name.lower().endswith(NON_SEARCHABLE):\n+ warnings.warn("Warning: `%s` is not eligible for search and was "\n+ "omitted!" % param_name)\n+ continue\n+\n+ if not swap_value.startswith(\':\'):\n+ safe_eval = SafeEval(load_scipy=True, load_numpy=True)\n+ ev = safe_eval(swap_value)\n+ else:\n+ # Have `:` before search list, asks for estimator evaluatio\n+ safe_eval_es = SafeEval(load_estimators=True)\n+ swap_value = swap_value[1:].strip()\n+ # TODO maybe add regular express check\n+ ev = safe_eval_es(swap_value)\n+\n+ swap_params[param_name] = ev\n+\n+ return swap_params\n+\n+\n+def train_test_split_none(*arrays, **kwargs):\n+ """extend train_test_split to take None arrays\n+ and support split by group names.\n+ """\n+ nones = []\n+ new_arrays = []\n+ for idx, arr in enumerate(arrays):\n+ if arr is None:\n+ nones.append(idx)\n+ else:\n+ new_arrays.append(arr)\n+\n+ if kwargs[\'shuffle\'] == \'None\':\n+ kwargs[\'shuffle\'] = None\n+\n+ group_names = kwargs.pop(\'group_names\', None)\n+\n+ if group_names is not None and group_names.strip():\n+ group_names = [name.strip() for name in\n+ group_names.split(\',\')]\n+ new_arrays = indexable(*new_arrays)\n+ groups = kwargs[\'labels\']\n+ n_samples = new_arrays[0].shape[0]\n+ index_arr = np.arange(n_samples)\n+ test = index_arr[np.isin(groups, group_names)]\n+ train = index_arr[~np.isin(groups, group_names)]\n+ rval = list(chain.from_iterable(\n+ (safe_indexing(a, train),\n+ safe_indexing(a, test)) for a in new_arrays))\n+ else:\n+ rval = train_test_split(*new_arrays, **kwargs)\n+\n+ for pos in nones:\n+ rval[pos * 2: 2] = [None, None]\n+\n+ return rval\n+\n+\n+def main(inputs, infile_estimator, infile1, infile2,\n+ outfile_result, outfile_object=None,\n+ outfile_weights=None, groups=None,\n+ ref_seq=None, intervals=None, targets=None,\n+ fasta_path=None):\n+ """\n+ '..b'e(X, y, groups, **test_split_options)\n+\n+ exp_scheme = params[\'experiment_schemes\'][\'selected_exp_scheme\']\n+\n+ # handle validation (second) split\n+ if exp_scheme == \'train_val_test\':\n+ val_split_options = (params[\'experiment_schemes\']\n+ [\'val_split\'][\'split_algos\'])\n+\n+ if val_split_options[\'shuffle\'] == \'group\':\n+ val_split_options[\'labels\'] = groups_train\n+ if val_split_options[\'shuffle\'] == \'stratified\':\n+ if y_train is not None:\n+ val_split_options[\'labels\'] = y_train\n+ else:\n+ raise ValueError("Stratified shuffle split is not "\n+ "applicable on empty target values!")\n+\n+ X_train, X_val, y_train, y_val, groups_train, groups_val = \\\n+ train_test_split_none(X_train, y_train, groups_train,\n+ **val_split_options)\n+\n+ # train and eval\n+ if hasattr(estimator, \'validation_data\'):\n+ if exp_scheme == \'train_val_test\':\n+ estimator.fit(X_train, y_train,\n+ validation_data=(X_val, y_val))\n+ else:\n+ estimator.fit(X_train, y_train,\n+ validation_data=(X_test, y_test))\n+ else:\n+ estimator.fit(X_train, y_train)\n+\n+ if hasattr(estimator, \'evaluate\'):\n+ scores = estimator.evaluate(X_test, y_test=y_test,\n+ scorer=scorer,\n+ is_multimetric=True)\n+ else:\n+ scores = _score(estimator, X_test, y_test, scorer,\n+ is_multimetric=True)\n+ # handle output\n+ for name, score in scores.items():\n+ scores[name] = [score]\n+ df = pd.DataFrame(scores)\n+ df = df[sorted(df.columns)]\n+ df.to_csv(path_or_buf=outfile_result, sep=\'\\t\',\n+ header=True, index=False)\n+\n+ memory.clear(warn=False)\n+\n+ if outfile_object:\n+ main_est = estimator\n+ if isinstance(estimator, pipeline.Pipeline):\n+ main_est = estimator.steps[-1][-1]\n+\n+ if hasattr(main_est, \'model_\') \\\n+ and hasattr(main_est, \'save_weights\'):\n+ if outfile_weights:\n+ main_est.save_weights(outfile_weights)\n+ del main_est.model_\n+ del main_est.fit_params\n+ del main_est.model_class_\n+ del main_est.validation_data\n+ if getattr(main_est, \'data_generator_\', None):\n+ del main_est.data_generator_\n+ del main_est.data_batch_generator\n+\n+ with open(outfile_object, \'wb\') as output_handler:\n+ pickle.dump(estimator, output_handler,\n+ pickle.HIGHEST_PROTOCOL)\n+\n+\n+if __name__ == \'__main__\':\n+ aparser = argparse.ArgumentParser()\n+ aparser.add_argument("-i", "--inputs", dest="inputs", required=True)\n+ aparser.add_argument("-e", "--estimator", dest="infile_estimator")\n+ aparser.add_argument("-X", "--infile1", dest="infile1")\n+ aparser.add_argument("-y", "--infile2", dest="infile2")\n+ aparser.add_argument("-O", "--outfile_result", dest="outfile_result")\n+ aparser.add_argument("-o", "--outfile_object", dest="outfile_object")\n+ aparser.add_argument("-w", "--outfile_weights", dest="outfile_weights")\n+ aparser.add_argument("-g", "--groups", dest="groups")\n+ aparser.add_argument("-r", "--ref_seq", dest="ref_seq")\n+ aparser.add_argument("-b", "--intervals", dest="intervals")\n+ aparser.add_argument("-t", "--targets", dest="targets")\n+ aparser.add_argument("-f", "--fasta_path", dest="fasta_path")\n+ args = aparser.parse_args()\n+\n+ main(args.inputs, args.infile_estimator, args.infile1, args.infile2,\n+ args.outfile_result, outfile_object=args.outfile_object,\n+ outfile_weights=args.outfile_weights, groups=args.groups,\n+ ref_seq=args.ref_seq, intervals=args.intervals,\n+ targets=args.targets, fasta_path=args.fasta_path)\n' |
| b |
| diff -r 68afcd163b3d -r 44f0ce9488fb utils.py --- a/utils.py Tue Jul 09 19:36:00 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| b'@@ -1,599 +0,0 @@\n-import ast\n-import json\n-import imblearn\n-import numpy as np\n-import pandas\n-import pickle\n-import re\n-import scipy\n-import sklearn\n-import skrebate\n-import sys\n-import warnings\n-import xgboost\n-\n-from collections import Counter\n-from asteval import Interpreter, make_symbol_table\n-from imblearn import under_sampling, over_sampling, combine\n-from imblearn.pipeline import Pipeline as imbPipeline\n-from mlxtend import regressor, classifier\n-from scipy.io import mmread\n-from sklearn import (\n- cluster, compose, decomposition, ensemble, feature_extraction,\n- feature_selection, gaussian_process, kernel_approximation, metrics,\n- model_selection, naive_bayes, neighbors, pipeline, preprocessing,\n- svm, linear_model, tree, discriminant_analysis)\n-\n-try:\n- import iraps_classifier\n-except ImportError:\n- pass\n-\n-try:\n- import model_validations\n-except ImportError:\n- pass\n-\n-try:\n- import feature_selectors\n-except ImportError:\n- pass\n-\n-try:\n- import preprocessors\n-except ImportError:\n- pass\n-\n-# handle pickle white list file\n-WL_FILE = __import__(\'os\').path.join(\n- __import__(\'os\').path.dirname(__file__), \'pk_whitelist.json\')\n-\n-N_JOBS = int(__import__(\'os\').environ.get(\'GALAXY_SLOTS\', 1))\n-\n-\n-class _SafePickler(pickle.Unpickler, object):\n- """\n- Used to safely deserialize scikit-learn model objects\n- Usage:\n- eg.: _SafePickler.load(pickled_file_object)\n- """\n- def __init__(self, file):\n- super(_SafePickler, self).__init__(file)\n- # load global white list\n- with open(WL_FILE, \'r\') as f:\n- self.pk_whitelist = json.load(f)\n-\n- self.bad_names = (\n- \'and\', \'as\', \'assert\', \'break\', \'class\', \'continue\',\n- \'def\', \'del\', \'elif\', \'else\', \'except\', \'exec\',\n- \'finally\', \'for\', \'from\', \'global\', \'if\', \'import\',\n- \'in\', \'is\', \'lambda\', \'not\', \'or\', \'pass\', \'print\',\n- \'raise\', \'return\', \'try\', \'system\', \'while\', \'with\',\n- \'True\', \'False\', \'None\', \'eval\', \'execfile\', \'__import__\',\n- \'__package__\', \'__subclasses__\', \'__bases__\', \'__globals__\',\n- \'__code__\', \'__closure__\', \'__func__\', \'__self__\', \'__module__\',\n- \'__dict__\', \'__class__\', \'__call__\', \'__get__\',\n- \'__getattribute__\', \'__subclasshook__\', \'__new__\',\n- \'__init__\', \'func_globals\', \'func_code\', \'func_closure\',\n- \'im_class\', \'im_func\', \'im_self\', \'gi_code\', \'gi_frame\',\n- \'__asteval__\', \'f_locals\', \'__mro__\')\n-\n- # unclassified good globals\n- self.good_names = [\n- \'copy_reg._reconstructor\', \'__builtin__.object\',\n- \'__builtin__.bytearray\', \'builtins.object\',\n- \'builtins.bytearray\', \'keras.engine.sequential.Sequential\',\n- \'keras.engine.sequential.Model\']\n-\n- # custom module in Galaxy-ML\n- self.custom_modules = [\n- \'__main__\', \'keras_galaxy_models\', \'feature_selectors\',\n- \'preprocessors\', \'iraps_classifier\', \'model_validations\']\n-\n- # override\n- def find_class(self, module, name):\n- # balack list first\n- if name in self.bad_names:\n- raise pickle.UnpicklingError("global \'%s.%s\' is forbidden"\n- % (module, name))\n-\n- # custom module in Galaxy-ML\n- if module in self.custom_modules:\n- cutom_module = sys.modules.get(module, None)\n- if cutom_module:\n- return getattr(cutom_module, name)\n- else:\n- raise pickle.UnpicklingError("Module %s\' is not imported"\n- % module)\n-\n- # For objects from outside libraries, it\'s necessary to verify\n- # both module and name. Currently only a blacklist checker\n- # is working.\n- # TODO: replace with a whitelist checker.\n- good_names = self.good_names\n- pk_whitelist = self.pk_whitelist\n- if re.m'..b' groups = read_columns(\n- infile_g,\n- c=c,\n- c_option=column_option,\n- sep=\'\\t\',\n- header=header,\n- parse_dates=True)\n- groups = groups.ravel()\n-\n- for k, v in cv_json.items():\n- if v == \'\':\n- cv_json[k] = None\n-\n- test_fold = cv_json.get(\'test_fold\', None)\n- if test_fold:\n- if test_fold.startswith(\'__ob__\'):\n- test_fold = test_fold[6:]\n- if test_fold.endswith(\'__cb__\'):\n- test_fold = test_fold[:-6]\n- cv_json[\'test_fold\'] = [int(x.strip()) for x in test_fold.split(\',\')]\n-\n- test_size = cv_json.get(\'test_size\', None)\n- if test_size and test_size > 1.0:\n- cv_json[\'test_size\'] = int(test_size)\n-\n- if cv == \'OrderedKFold\':\n- cv_class = try_get_attr(\'model_validations\', \'OrderedKFold\')\n- elif cv == \'RepeatedOrderedKFold\':\n- cv_class = try_get_attr(\'model_validations\', \'RepeatedOrderedKFold\')\n- else:\n- cv_class = getattr(model_selection, cv)\n- splitter = cv_class(**cv_json)\n-\n- return splitter, groups\n-\n-\n-# needed when sklearn < v0.20\n-def balanced_accuracy_score(y_true, y_pred):\n- """Compute balanced accuracy score, which is now available in\n- scikit-learn from v0.20.0.\n- """\n- C = metrics.confusion_matrix(y_true, y_pred)\n- with np.errstate(divide=\'ignore\', invalid=\'ignore\'):\n- per_class = np.diag(C) / C.sum(axis=1)\n- if np.any(np.isnan(per_class)):\n- warnings.warn(\'y_pred contains classes not in y_true\')\n- per_class = per_class[~np.isnan(per_class)]\n- score = np.mean(per_class)\n- return score\n-\n-\n-def get_scoring(scoring_json):\n- """Return single sklearn scorer class\n- or multiple scoers in dictionary\n- """\n- if scoring_json[\'primary_scoring\'] == \'default\':\n- return None\n-\n- my_scorers = metrics.SCORERS\n- my_scorers[\'binarize_auc_scorer\'] =\\\n- try_get_attr(\'iraps_classifier\', \'binarize_auc_scorer\')\n- my_scorers[\'binarize_average_precision_scorer\'] =\\\n- try_get_attr(\'iraps_classifier\', \'binarize_average_precision_scorer\')\n- if \'balanced_accuracy\' not in my_scorers:\n- my_scorers[\'balanced_accuracy\'] =\\\n- metrics.make_scorer(balanced_accuracy_score)\n-\n- if scoring_json[\'secondary_scoring\'] != \'None\'\\\n- and scoring_json[\'secondary_scoring\'] !=\\\n- scoring_json[\'primary_scoring\']:\n- return_scoring = {}\n- primary_scoring = scoring_json[\'primary_scoring\']\n- return_scoring[primary_scoring] = my_scorers[primary_scoring]\n- for scorer in scoring_json[\'secondary_scoring\'].split(\',\'):\n- if scorer != scoring_json[\'primary_scoring\']:\n- return_scoring[scorer] = my_scorers[scorer]\n- return return_scoring\n-\n- return my_scorers[scoring_json[\'primary_scoring\']]\n-\n-\n-def get_search_params(estimator):\n- """Format the output of `estimator.get_params()`\n- """\n- params = estimator.get_params()\n- results = []\n- for k, v in params.items():\n- # params below won\'t be shown for search in the searchcv tool\n- keywords = (\'n_jobs\', \'pre_dispatch\', \'memory\', \'steps\',\n- \'nthread\', \'verbose\')\n- if k.endswith(keywords):\n- results.append([\'*\', k, k+": "+repr(v)])\n- else:\n- results.append([\'@\', k, k+": "+repr(v)])\n- results.append(\n- ["", "Note:",\n- "@, params eligible for search in searchcv tool."])\n-\n- return results\n-\n-\n-def try_get_attr(module, name):\n- """try to get attribute from a custom module\n-\n- Parameters\n- ----------\n- module : str\n- Module name\n- name : str\n- Attribute (class/function) name.\n-\n- Returns\n- -------\n- class or function\n- """\n- mod = sys.modules.get(module, None)\n- if mod:\n- return getattr(mod, name)\n- else:\n- raise Exception("No module named %s." % module)\n' |