+
+
+"""
+ # The CSS needed to style and hide/show the modal
+ modal_css = """
+
+"""
+ # The JavaScript to open/close the modal on button click
+ modal_js = """
+
+"""
+ return modal_css + modal_html + modal_js
+
+
+def encode_image_to_base64(image_path):
+ with open(image_path, "rb") as img_file:
+ return base64.b64encode(img_file.read()).decode("utf-8")
+
+
+def generate_feature_importance(*args, **kwargs):
+ return "Feature importance visualizations are not supported for this MultiModal workflow.
" diff -r 000000000000 -r 375c36923da1 feature_importance.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/feature_importance.py Tue Dec 09 23:49:47 2025 +0000 @@ -0,0 +1,11 @@ +"""Feature importance visualization utilities.""" + +import pandas as pd + + +def build_feature_importance_html(predictor, df_train: pd.DataFrame, label_column: str) -> str: + """Feature importance is not currently available for the MultiModal workflow.""" + return ( + "Feature importance visualization is not supported for the current " + "MultiModal workflow.
" + ) diff -r 000000000000 -r 375c36923da1 metrics_logic.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/metrics_logic.py Tue Dec 09 23:49:47 2025 +0000 @@ -0,0 +1,313 @@ +from collections import OrderedDict +from typing import Dict, Optional, Tuple + +import numpy as np +import pandas as pd +from sklearn.metrics import ( + accuracy_score, + average_precision_score, + cohen_kappa_score, + confusion_matrix, + f1_score, + log_loss, + matthews_corrcoef, + mean_absolute_error, + mean_squared_error, + median_absolute_error, + precision_score, + r2_score, + recall_score, + roc_auc_score, +) + + +# -------------------- Transparent Metrics (task-aware) -------------------- # + +def _safe_y_proba_to_array(y_proba) -> Optional[np.ndarray]: + """Convert predictor.predict_proba output (array/DataFrame/dict) to np.ndarray or None.""" + if y_proba is None: + return None + if isinstance(y_proba, pd.DataFrame): + return y_proba.values + if isinstance(y_proba, (list, tuple)): + return np.asarray(y_proba) + if isinstance(y_proba, np.ndarray): + return y_proba + if isinstance(y_proba, dict): + try: + return np.vstack([np.asarray(v) for _, v in sorted(y_proba.items())]).T + except Exception: + return None + return None + + +def _specificity_from_cm(cm: np.ndarray) -> float: + """Specificity (TNR) for binary confusion matrix.""" + if cm.shape != (2, 2): + return np.nan + tn, fp, fn, tp = cm.ravel() + denom = (tn + fp) + return float(tn / denom) if denom > 0 else np.nan + + +def _compute_regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> "OrderedDict[str, float]": + mse = mean_squared_error(y_true, y_pred) + rmse = float(np.sqrt(mse)) + mae = mean_absolute_error(y_true, y_pred) + # Avoid division by zero using clip + mape = float(np.mean(np.abs((y_true - y_pred) / np.clip(np.abs(y_true), 1e-12, None))) * 100.0) + r2 = r2_score(y_true, y_pred) + medae = median_absolute_error(y_true, y_pred) + + metrics = OrderedDict() + metrics["MSE"] = mse + metrics["RMSE"] = rmse + metrics["MAE"] = mae + metrics["MAPE_%"] = mape + metrics["R2"] = r2 + metrics["MedianAE"] = medae + return metrics + + +def _compute_binary_metrics( + y_true: pd.Series, + y_pred: pd.Series, + y_proba: Optional[np.ndarray], + predictor +) -> "OrderedDict[str, float]": + metrics = OrderedDict() + classes_sorted = np.sort(pd.unique(y_true)) + # Choose the lexicographically larger class as "positive" + pos_label = classes_sorted[-1] + + metrics["Accuracy"] = accuracy_score(y_true, y_pred) + metrics["Precision"] = precision_score(y_true, y_pred, pos_label=pos_label, zero_division=0) + metrics["Recall_(Sensitivity/TPR)"] = recall_score(y_true, y_pred, pos_label=pos_label, zero_division=0) + metrics["F1-Score"] = f1_score(y_true, y_pred, pos_label=pos_label, zero_division=0) + + try: + cm = confusion_matrix(y_true, y_pred, labels=classes_sorted) + metrics["Specificity_(TNR)"] = _specificity_from_cm(cm) + except Exception: + metrics["Specificity_(TNR)"] = np.nan + + # Probabilistic metrics + if y_proba is not None: + # pick column of positive class + if y_proba.ndim == 1: + pos_scores = y_proba + else: + pos_col_idx = -1 + try: + if hasattr(predictor, "class_labels") and predictor.class_labels: + pos_col_idx = list(predictor.class_labels).index(pos_label) + except Exception: + pos_col_idx = -1 + pos_scores = y_proba[:, pos_col_idx] + try: + metrics["ROC-AUC"] = roc_auc_score(y_true == pos_label, pos_scores) + except Exception: + metrics["ROC-AUC"] = np.nan + try: + metrics["PR-AUC"] = average_precision_score(y_true == pos_label, pos_scores) + except Exception: + metrics["PR-AUC"] = np.nan + try: + if y_proba.ndim == 1: + y_proba_ll = np.column_stack([1 - pos_scores, pos_scores]) + else: + y_proba_ll = y_proba + metrics["LogLoss"] = log_loss(y_true, y_proba_ll, labels=classes_sorted) + except Exception: + metrics["LogLoss"] = np.nan + else: + metrics["ROC-AUC"] = np.nan + metrics["PR-AUC"] = np.nan + metrics["LogLoss"] = np.nan + + try: + metrics["MCC"] = matthews_corrcoef(y_true, y_pred) + except Exception: + metrics["MCC"] = np.nan + + return metrics + + +def _compute_multiclass_metrics( + y_true: pd.Series, + y_pred: pd.Series, + y_proba: Optional[np.ndarray] +) -> "OrderedDict[str, float]": + metrics = OrderedDict() + metrics["Accuracy"] = accuracy_score(y_true, y_pred) + metrics["Macro Precision"] = precision_score(y_true, y_pred, average="macro", zero_division=0) + metrics["Macro Recall"] = recall_score(y_true, y_pred, average="macro", zero_division=0) + metrics["Macro F1"] = f1_score(y_true, y_pred, average="macro", zero_division=0) + metrics["Weighted Precision"] = precision_score(y_true, y_pred, average="weighted", zero_division=0) + metrics["Weighted Recall"] = recall_score(y_true, y_pred, average="weighted", zero_division=0) + metrics["Weighted F1"] = f1_score(y_true, y_pred, average="weighted", zero_division=0) + + try: + metrics["Cohen_Kappa"] = cohen_kappa_score(y_true, y_pred) + except Exception: + metrics["Cohen_Kappa"] = np.nan + try: + metrics["MCC"] = matthews_corrcoef(y_true, y_pred) + except Exception: + metrics["MCC"] = np.nan + + # Probabilistic metrics + classes_sorted = np.sort(pd.unique(y_true)) + if y_proba is not None and y_proba.ndim == 2: + try: + metrics["LogLoss"] = log_loss(y_true, y_proba, labels=classes_sorted) + except Exception: + metrics["LogLoss"] = np.nan + # Macro ROC-AUC / PR-AUC via OVR + try: + class_to_index = {c: i for i, c in enumerate(classes_sorted)} + y_true_idx = np.vectorize(class_to_index.get)(y_true) + metrics["ROC-AUC_macro"] = roc_auc_score(y_true_idx, y_proba, multi_class="ovr", average="macro") + except Exception: + metrics["ROC-AUC_macro"] = np.nan + try: + Y_true_ind = np.zeros_like(y_proba) + idx_map = {c: i for i, c in enumerate(classes_sorted)} + Y_true_ind[np.arange(y_proba.shape[0]), np.vectorize(idx_map.get)(y_true)] = 1 + metrics["PR-AUC_macro"] = average_precision_score(Y_true_ind, y_proba, average="macro") + except Exception: + metrics["PR-AUC_macro"] = np.nan + else: + metrics["LogLoss"] = np.nan + metrics["ROC-AUC_macro"] = np.nan + metrics["PR-AUC_macro"] = np.nan + + return metrics + + +def aggregate_metrics(list_of_dicts): + """Aggregate a list of metrics dicts (per split) into mean/std.""" + agg_mean = {} + agg_std = {} + for split in ("Train", "Validation", "Test", "Test (external)"): + keys = set() + for m in list_of_dicts: + if isinstance(m, dict) and split in m: + keys.update(m[split].keys()) + if not keys: + continue + agg_mean[split] = {} + agg_std[split] = {} + for k in keys: + vals = [m[split][k] for m in list_of_dicts if split in m and k in m[split]] + numeric_vals = [] + for v in vals: + try: + numeric_vals.append(float(v)) + except Exception: + pass + if numeric_vals: + agg_mean[split][k] = float(np.mean(numeric_vals)) + agg_std[split][k] = float(np.std(numeric_vals, ddof=0)) + else: + agg_mean[split][k] = vals[-1] if vals else None + agg_std[split][k] = None + return agg_mean, agg_std + + +def compute_metrics_for_split( + predictor, + df: pd.DataFrame, + target_col: str, + problem_type: str, + threshold: Optional[float] = None, # <— NEW +) -> "OrderedDict[str, float]": + """Compute transparency metrics for one split (Train/Val/Test) based on task type.""" + # Prepare inputs + features = df.drop(columns=[target_col], errors="ignore") + y_true_series = df[target_col].reset_index(drop=True) + + # Probabilities (if available) + y_proba = None + try: + y_proba_raw = predictor.predict_proba(features) + y_proba = _safe_y_proba_to_array(y_proba_raw) + except Exception: + y_proba = None + + # Labels (optionally thresholded for binary) + y_pred_series = None + if problem_type == "binary" and (threshold is not None) and (y_proba is not None): + classes_sorted = np.sort(pd.unique(y_true_series)) + pos_label = classes_sorted[-1] + neg_label = classes_sorted[0] + if y_proba.ndim == 1: + pos_scores = y_proba + else: + pos_col_idx = -1 + try: + if hasattr(predictor, "class_labels") and predictor.class_labels: + pos_col_idx = list(predictor.class_labels).index(pos_label) + except Exception: + pos_col_idx = -1 + pos_scores = y_proba[:, pos_col_idx] + y_pred_series = pd.Series(np.where(pos_scores >= float(threshold), pos_label, neg_label)).reset_index(drop=True) + else: + # Fall back to model's default label prediction (argmax / 0.5 equivalent) + y_pred_series = pd.Series(predictor.predict(features)).reset_index(drop=True) + + if problem_type == "regression": + y_true_arr = np.asarray(y_true_series, dtype=float) + y_pred_arr = np.asarray(y_pred_series, dtype=float) + return _compute_regression_metrics(y_true_arr, y_pred_arr) + + if problem_type == "binary": + return _compute_binary_metrics(y_true_series, y_pred_series, y_proba, predictor) + + # multiclass + return _compute_multiclass_metrics(y_true_series, y_pred_series, y_proba) + + +def evaluate_all_transparency( + predictor, + train_df: Optional[pd.DataFrame], + val_df: Optional[pd.DataFrame], + test_df: Optional[pd.DataFrame], + target_col: str, + problem_type: str, + threshold: Optional[float] = None, +) -> Tuple[pd.DataFrame, Dict[str, Dict[str, float]]]: + """ + Evaluate Train/Val/Test with the transparent metrics suite. + Returns: + - metrics_table: DataFrame with index=Metric, columns subset of [Train, Validation, Test] + - raw_dict: nested dict {split -> {metric -> value}} + """ + split_results: Dict[str, Dict[str, float]] = {} + splits = [] + + # IMPORTANT: do NOT apply threshold to Train/Val + if train_df is not None and len(train_df): + split_results["Train"] = compute_metrics_for_split(predictor, train_df, target_col, problem_type, threshold=None) + splits.append("Train") + if val_df is not None and len(val_df): + split_results["Validation"] = compute_metrics_for_split(predictor, val_df, target_col, problem_type, threshold=None) + splits.append("Validation") + if test_df is not None and len(test_df): + split_results["Test"] = compute_metrics_for_split(predictor, test_df, target_col, problem_type, threshold=threshold) + splits.append("Test") + + # Preserve order from the first split; include any extras from others + order_source = split_results[splits[0]] if splits else {} + all_metrics = list(order_source.keys()) + for s in splits[1:]: + for m in split_results[s].keys(): + if m not in all_metrics: + all_metrics.append(m) + + metrics_table = pd.DataFrame(index=all_metrics, columns=splits, dtype=float) + for s in splits: + for m, v in split_results[s].items(): + metrics_table.loc[m, s] = v + + return metrics_table, split_results diff -r 000000000000 -r 375c36923da1 multimodal_learner.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multimodal_learner.py Tue Dec 09 23:49:47 2025 +0000 @@ -0,0 +1,391 @@ +#!/usr/bin/env python +""" +Main entrypoint for AutoGluon multimodal training wrapper. +""" + +import argparse +import logging +import os +import sys +from typing import List, Optional + +import pandas as pd +from metrics_logic import aggregate_metrics +from plot_logic import infer_problem_type +from report_utils import write_outputs +from sklearn.model_selection import KFold, StratifiedKFold +from split_logic import split_dataset +from test_pipeline import run_autogluon_test_experiment +from training_pipeline import autogluon_hyperparameters, handle_missing_images, run_autogluon_experiment +# ------------------------------------------------------------------ +# Local imports (your split utilities) +# ------------------------------------------------------------------ +from utils import ( + absolute_path_expander, + enable_deterministic_mode, + enable_tensor_cores_if_available, + ensure_local_tmp, + load_file, + prepare_image_search_dirs, + set_seeds, + str2bool, +) + +# ------------------------------------------------------------------ +# Logger setup +# ------------------------------------------------------------------ +logger = logging.getLogger(__name__) + + +# ------------------------------------------------------------------ +# Argument parsing (unchanged from your original, only minor fixes) +# ------------------------------------------------------------------ +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="Train & report an AutoGluon model") + + parser.add_argument("--input_csv_train", dest="train_dataset", required=True) + parser.add_argument("--input_csv_test", dest="test_dataset", default=None) + parser.add_argument("--target_column", required=True) + parser.add_argument("--output_json", default="results.json") + parser.add_argument("--output_html", default="report.html") + parser.add_argument("--output_config", default=None) + parser.add_argument("--images_zip", nargs="*", default=None, + help="One or more ZIP files that contain image assets") + parser.add_argument("--missing_image_strategy", default="false", + help="true/false: remove rows with missing images or use placeholder") + parser.add_argument("--threshold", type=float, default=None) + parser.add_argument("--time_limit", type=int, default=None) + parser.add_argument("--deterministic", action="store_true", default=False, + help="Enable deterministic algorithms to reduce run-to-run variance") + parser.add_argument("--random_seed", type=int, default=42) + parser.add_argument("--cross_validation", type=str, default="false") + parser.add_argument("--num_folds", type=int, default=5) + parser.add_argument("--epochs", type=int, default=None) + parser.add_argument("--learning_rate", type=float, default=None) + parser.add_argument("--batch_size", type=int, default=None) + parser.add_argument("--backbone_image", type=str, default="swin_base_patch4_window7_224") + parser.add_argument("--backbone_text", type=str, default="microsoft/deberta-v3-base") + parser.add_argument("--validation_size", type=float, default=0.2) + parser.add_argument("--split_probabilities", type=float, nargs=3, + default=[0.7, 0.1, 0.2], metavar=("train", "val", "test")) + parser.add_argument("--preset", choices=["medium_quality", "high_quality", "best_quality"], + default="medium_quality") + parser.add_argument("--eval_metric", default="roc_auc") + parser.add_argument("--hyperparameters", default=None) + + args, unknown = parser.parse_known_args(argv) + if unknown: + logger.warning("Ignoring unknown CLI tokens: %s", unknown) + + # -------------------------- Validation -------------------------- + if not (0.0 <= args.validation_size <= 1.0): + parser.error("--validation_size must be in [0, 1]") + if len(args.split_probabilities) != 3 or abs(sum(args.split_probabilities) - 1.0) > 1e-6: + parser.error("--split_probabilities must be three numbers summing to 1.0") + if args.cross_validation.lower() == "true" and (args.num_folds < 2): + parser.error("--num_folds must be >= 2 when --cross_validation is true") + + return args + + +def run_cross_validation( + args, + df_full: pd.DataFrame, + test_dataset: Optional[pd.DataFrame], + image_cols: List[str], + ag_config: dict, +): + """Cross-validation loop returning aggregated metrics and last predictor.""" + df_full = df_full.drop(columns=["split"], errors="ignore") + y = df_full[args.target_column] + try: + use_stratified = y.dtype == object or y.nunique() <= 20 + except Exception: + use_stratified = False + + kf = StratifiedKFold(n_splits=int(args.num_folds), shuffle=True, random_state=int(args.random_seed)) if use_stratified else KFold(n_splits=int(args.num_folds), shuffle=True, random_state=int(args.random_seed)) + + raw_folds = [] + ag_folds = [] + folds_info = [] + last_predictor = None + last_data_ctx = None + + for fold_idx, (train_idx, val_idx) in enumerate(kf.split(df_full, y if use_stratified else None), start=1): + logger.info(f"CV fold {fold_idx}/{args.num_folds}") + df_tr = df_full.iloc[train_idx].copy() + df_va = df_full.iloc[val_idx].copy() + + df_tr["split"] = "train" + df_va["split"] = "val" + fold_dataset = pd.concat([df_tr, df_va], ignore_index=True) + + predictor_fold, data_ctx = run_autogluon_experiment( + train_dataset=fold_dataset, + test_dataset=test_dataset, + target_column=args.target_column, + image_columns=image_cols, + ag_config=ag_config, + ) + last_predictor = predictor_fold + last_data_ctx = data_ctx + problem_type = infer_problem_type(predictor_fold, df_tr, args.target_column) + eval_results = run_autogluon_test_experiment( + predictor=predictor_fold, + data_ctx=data_ctx, + target_column=args.target_column, + eval_metric=args.eval_metric, + ag_config=ag_config, + problem_type=problem_type, + ) + + raw_metrics_fold = eval_results.get("raw_metrics", {}) + ag_by_split_fold = eval_results.get("ag_eval", {}) + raw_folds.append(raw_metrics_fold) + ag_folds.append(ag_by_split_fold) + folds_info.append( + { + "fold": int(fold_idx), + "predictor_path": getattr(predictor_fold, "path", None), + "raw_metrics": raw_metrics_fold, + "ag_eval": ag_by_split_fold, + } + ) + + raw_metrics_mean, raw_metrics_std = aggregate_metrics(raw_folds) + ag_by_split_mean, ag_by_split_std = aggregate_metrics(ag_folds) + return ( + last_predictor, + raw_metrics_mean, + ag_by_split_mean, + raw_folds, + ag_folds, + raw_metrics_std, + ag_by_split_std, + folds_info, + last_data_ctx, + ) + + +# ------------------------------------------------------------------ +# Main execution +# ------------------------------------------------------------------ +def main(): + args = parse_args() + + # ------------------------------------------------------------------ + # Debug output + # ------------------------------------------------------------------ + logger.info("=== AutoGluon Training Wrapper Started ===") + logger.info(f"Working directory: {os.getcwd()}") + logger.info(f"Command line: {' '.join(sys.argv)}") + logger.info(f"Parsed args: {vars(args)}") + + # ------------------------------------------------------------------ + # Reproducibility & performance + # ------------------------------------------------------------------ + set_seeds(args.random_seed) + if args.deterministic: + enable_deterministic_mode(args.random_seed) + logger.info("Deterministic mode enabled (seed=%s)", args.random_seed) + ensure_local_tmp() + enable_tensor_cores_if_available() + + # ------------------------------------------------------------------ + # Load datasets + # ------------------------------------------------------------------ + train_dataset = load_file(args.train_dataset) + test_dataset = load_file(args.test_dataset) if args.test_dataset else None + + logger.info(f"Train dataset loaded: {len(train_dataset)} rows") + if test_dataset is not None: + logger.info(f"Test dataset loaded: {len(test_dataset)} rows") + + # ------------------------------------------------------------------ + # Resolve target column by name; if Galaxy passed a numeric index, + # translate it to the corresponding header so downstream checks pass. + # Galaxy's data_column widget is 1-based. + # ------------------------------------------------------------------ + if args.target_column not in train_dataset.columns and str(args.target_column).isdigit(): + idx = int(args.target_column) - 1 + if 0 <= idx < len(train_dataset.columns): + resolved = train_dataset.columns[idx] + logger.info(f"Target column '{args.target_column}' not found; using column #{idx + 1} header '{resolved}' instead.") + args.target_column = resolved + else: + logger.error(f"Numeric target index '{args.target_column}' is out of range for dataset with {len(train_dataset.columns)} columns.") + sys.exit(1) + + # ------------------------------------------------------------------ + # Image handling (ZIP extraction + absolute path expansion) + # ------------------------------------------------------------------ + extracted_imgs_path = prepare_image_search_dirs(args) + + image_cols = absolute_path_expander(train_dataset, extracted_imgs_path, None) + if test_dataset is not None: + absolute_path_expander(test_dataset, extracted_imgs_path, image_cols) + + # ------------------------------------------------------------------ + # Handle missing images + # ------------------------------------------------------------------ + train_dataset = handle_missing_images( + train_dataset, + image_columns=image_cols, + strategy=args.missing_image_strategy, + ) + if test_dataset is not None: + test_dataset = handle_missing_images( + test_dataset, + image_columns=image_cols, + strategy=args.missing_image_strategy, + ) + + logger.info(f"After cleanup → train: {len(train_dataset)}, test: {len(test_dataset) if test_dataset is not None else 0}") + + # ------------------------------------------------------------------ + # Dataset splitting logic (adds 'split' column to train_dataset) + # ------------------------------------------------------------------ + split_dataset( + train_dataset=train_dataset, + test_dataset=test_dataset, + target_column=args.target_column, + split_probabilities=args.split_probabilities, + validation_size=args.validation_size, + random_seed=args.random_seed, + ) + + logger.info("Preprocessing complete — ready for AutoGluon training!") + logger.info(f"Final split counts:\n{train_dataset['split'].value_counts().sort_index()}") + + # Verify target/image/text columns exist + if args.target_column not in train_dataset.columns: + logger.error(f"Target column '{args.target_column}' not found in training data.") + sys.exit(1) + if test_dataset is not None and args.target_column not in test_dataset.columns: + logger.error(f"Target column '{args.target_column}' not found in test data.") + sys.exit(1) + + # Threshold is only meaningful for binary classification; ignore otherwise. + threshold_for_run = args.threshold + unique_labels = None + target_looks_binary = False + try: + unique_labels = train_dataset[args.target_column].nunique(dropna=True) + target_looks_binary = unique_labels == 2 + except Exception: + logger.warning("Could not inspect target column '%s' for threshold validation; proceeding without binary check.", args.target_column) + + if threshold_for_run is not None: + if target_looks_binary: + threshold_for_run = float(threshold_for_run) + logger.info("Applying custom decision threshold %.4f for binary evaluation.", threshold_for_run) + else: + logger.warning( + "Threshold %.3f provided but target '%s' does not appear binary (unique labels=%s); ignoring threshold.", + threshold_for_run, + args.target_column, + unique_labels if unique_labels is not None else "unknown", + ) + threshold_for_run = None + args.threshold = threshold_for_run + # Image columns are auto-inferred; image_cols already resolved to absolute paths. + # ------------------------------------------------------------------ + # Build AutoGluon configuration from CLI knobs + # ------------------------------------------------------------------ + ag_config = autogluon_hyperparameters( + threshold=args.threshold, + time_limit=args.time_limit, + random_seed=args.random_seed, + epochs=args.epochs, + learning_rate=args.learning_rate, + batch_size=args.batch_size, + backbone_image=args.backbone_image, + backbone_text=args.backbone_text, + preset=args.preset, + eval_metric=args.eval_metric, + hyperparameters=args.hyperparameters, + ) + logger.info(f"AutoGluon config prepared: fit={ag_config.get('fit')}, hyperparameters keys={list(ag_config.get('hyperparameters', {}).keys())}") + + cv_enabled = str2bool(args.cross_validation) + if cv_enabled: + ( + predictor, + raw_metrics, + ag_by_split, + raw_folds, + ag_folds, + raw_metrics_std, + ag_by_split_std, + folds_info, + data_ctx, + ) = run_cross_validation( + args=args, + df_full=train_dataset, + test_dataset=test_dataset, + image_cols=image_cols, + ag_config=ag_config, + ) + if predictor is None: + logger.error("All CV folds failed. Exiting.") + sys.exit(1) + eval_results = { + "raw_metrics": raw_metrics, + "ag_eval": ag_by_split, + "fit_summary": None, + } + else: + predictor, data_ctx = run_autogluon_experiment( + train_dataset=train_dataset, + test_dataset=test_dataset, + target_column=args.target_column, + image_columns=image_cols, + ag_config=ag_config, + ) + logger.info("AutoGluon training finished. Model path: %s", getattr(predictor, "path", None)) + + # Evaluate predictor on Train/Val/Test splits + problem_type = infer_problem_type(predictor, train_dataset, args.target_column) + eval_results = run_autogluon_test_experiment( + predictor=predictor, + data_ctx=data_ctx, + target_column=args.target_column, + eval_metric=args.eval_metric, + ag_config=ag_config, + problem_type=problem_type, + ) + raw_metrics = eval_results.get("raw_metrics", {}) + ag_by_split = eval_results.get("ag_eval", {}) + raw_folds = ag_folds = raw_metrics_std = ag_by_split_std = None + + logger.info("Transparent metrics by split: %s", eval_results["raw_metrics"]) + logger.info("AutoGluon evaluate() by split: %s", eval_results["ag_eval"]) + + if "problem_type" in eval_results: + problem_type_final = eval_results["problem_type"] + else: + problem_type_final = infer_problem_type(predictor, train_dataset, args.target_column) + + write_outputs( + args=args, + predictor=predictor, + problem_type=problem_type_final, + eval_results=eval_results, + data_ctx=data_ctx, + raw_folds=raw_folds, + ag_folds=ag_folds, + raw_metrics_std=raw_metrics_std, + ag_by_split_std=ag_by_split_std, + ) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s", + datefmt="%H:%M:%S" + ) + # Quiet noisy image parsing logs (e.g., PIL.PngImagePlugin debug streams) + logging.getLogger("PIL").setLevel(logging.WARNING) + logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) + main() diff -r 000000000000 -r 375c36923da1 multimodal_learner.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multimodal_learner.xml Tue Dec 09 23:49:47 2025 +0000 @@ -0,0 +1,316 @@ + header so it matches the
+ green table section headers.
+ """
+ # kill Plotly’s built-in title
+ fig.update_layout(title=None)
+
+ # figure HTML without PlotlyJS (we load it once globally)
+ plot_html = fig.to_html(full_html=False, include_plotlyjs=False)
+
+ # use — your CSS already styles like the table headers
+ return f"""
+{html.escape(title)}
+{plot_html}
+""".strip()
+
+
+def _save_plotly(fig: go.Figure, path: Optional[str]) -> None:
+ """
+ Save a Plotly figure. If `path` ends with `.html`, save interactive HTML.
+ If it ends with a raster extension (png/jpg/jpeg/webp), uses Kaleido.
+ If None, do nothing (caller may choose to display in notebook).
+ """
+ if not path:
+ return
+ ext = os.path.splitext(path)[1].lower()
+ if ext == ".html":
+ fig.write_html(path, include_plotlyjs="cdn", full_html=True)
+ else:
+ # Requires kaleido: pip install -U kaleido
+ fig.write_image(path)
+
+
+def _save_matplotlib(path: Optional[str]) -> None:
+ """Save current Matplotlib figure if `path` is provided, else show()."""
+ if path:
+ plt.savefig(path, bbox_inches="tight")
+ plt.close()
+ else:
+ plt.show()
+
+# =========================
+# Classification Plots
+# =========================
+
+
+def generate_confusion_matrix_plot(
+ y_true,
+ y_pred,
+ title: str = "Confusion Matrix",
+) -> go.Figure:
+ y_true = np.asarray(y_true)
+ y_pred = np.asarray(y_pred)
+
+ # Class order (works for strings or numbers)
+ labels = pd.Index(np.unique(np.concatenate([y_true, y_pred])), dtype=object).tolist()
+ cm = confusion_matrix(y_true, y_pred, labels=labels)
+ max_val = cm.max() if cm.size else 0
+
+ # Use categorical axes by passing string labels for x/y
+ cats = [str(label) for label in labels]
+ total = int(cm.sum())
+
+ fig = go.Figure(
+ data=go.Heatmap(
+ z=cm,
+ x=cats, # categorical x
+ y=cats, # categorical y
+ colorscale="Blues",
+ showscale=True,
+ colorbar=dict(title="Count"),
+ xgap=2,
+ ygap=2,
+ hovertemplate="True=%{y}
Pred=%{x}
Count=%{z}
Precision=%{y:.3f}
Recall=%{y:.3f}
F1=%{y:.3f}
Queue Rate=%{y:.3f}
%{x}: %{z:.2f}
%{y:.2f}%Training Diagnostics
like the table headers
+ return f"""
+{html.escape(title)}
+{plot_html}
+""".strip()
+
+
+def _save_plotly(fig: go.Figure, path: Optional[str]) -> None:
+ """
+ Save a Plotly figure. If `path` ends with `.html`, save interactive HTML.
+ If it ends with a raster extension (png/jpg/jpeg/webp), uses Kaleido.
+ If None, do nothing (caller may choose to display in notebook).
+ """
+ if not path:
+ return
+ ext = os.path.splitext(path)[1].lower()
+ if ext == ".html":
+ fig.write_html(path, include_plotlyjs="cdn", full_html=True)
+ else:
+ # Requires kaleido: pip install -U kaleido
+ fig.write_image(path)
+
+
+def _save_matplotlib(path: Optional[str]) -> None:
+ """Save current Matplotlib figure if `path` is provided, else show()."""
+ if path:
+ plt.savefig(path, bbox_inches="tight")
+ plt.close()
+ else:
+ plt.show()
+
+# =========================
+# Classification Plots
+# =========================
+
+
+def generate_confusion_matrix_plot(
+ y_true,
+ y_pred,
+ title: str = "Confusion Matrix",
+) -> go.Figure:
+ y_true = np.asarray(y_true)
+ y_pred = np.asarray(y_pred)
+
+ # Class order (works for strings or numbers)
+ labels = pd.Index(np.unique(np.concatenate([y_true, y_pred])), dtype=object).tolist()
+ cm = confusion_matrix(y_true, y_pred, labels=labels)
+ max_val = cm.max() if cm.size else 0
+
+ # Use categorical axes by passing string labels for x/y
+ cats = [str(label) for label in labels]
+ total = int(cm.sum())
+
+ fig = go.Figure(
+ data=go.Heatmap(
+ z=cm,
+ x=cats, # categorical x
+ y=cats, # categorical y
+ colorscale="Blues",
+ showscale=True,
+ colorbar=dict(title="Count"),
+ xgap=2,
+ ygap=2,
+ hovertemplate="True=%{y}
Pred=%{x}
Count=%{z}
Precision=%{y:.3f}
Recall=%{y:.3f}
F1=%{y:.3f}
Queue Rate=%{y:.3f}
%{x}: %{z:.2f}
%{y:.2f}%Training Diagnostics
No training diagnostics available for this run.
" + + return "Train and Validation Performance Summary
" + "".join(pieces) + + +def generate_learning_curve( + estimator, + X, + y, + scoring: str = "r2", + cv_folds: int = 5, + n_jobs: int = -1, + train_sizes: np.ndarray = np.linspace(0.1, 1.0, 10), + title: str = "Learning Curve", + path: Optional[str] = None, +) -> go.Figure: + """ + Learning curve using sklearn.learning_curve, visualized with Plotly. + """ + sizes, train_scores, test_scores = skl_learning_curve( + estimator, X, y, cv=cv_folds, scoring=scoring, n_jobs=n_jobs, train_sizes=train_sizes + ) + train_mean = train_scores.mean(axis=1) + train_std = train_scores.std(axis=1) + test_mean = test_scores.mean(axis=1) + test_std = test_scores.std(axis=1) + + fig = go.Figure() + fig.add_trace(go.Scatter( + x=sizes, y=train_mean, mode="lines+markers", name="Training score", + error_y=dict(type="data", array=train_std, visible=True) + )) + fig.add_trace(go.Scatter( + x=sizes, y=test_mean, mode="lines+markers", name="CV score", + error_y=dict(type="data", array=test_std, visible=True) + )) + fig.update_layout( + title=None, + xaxis_title="Training examples", + yaxis_title=scoring, + template="plotly_white", + ) + _save_plotly(fig, path) + return fig + +# ========================= +# SHAP (Matplotlib-based) +# ========================= + + +def generate_shap_summary_plot( + shap_values, features: pd.DataFrame, title: str = "SHAP Summary Plot", path: Optional[str] = None +) -> None: + """ + SHAP summary plot (Matplotlib). SHAP's interactive support with Plotly is limited; + keep matplotlib for clarity and stability. + """ + plt.figure(figsize=(10, 8)) + shap.summary_plot(shap_values, features, show=False) + plt.title(title) + _save_matplotlib(path) + + +def generate_shap_force_plot( + explainer, instance: pd.DataFrame, title: str = "SHAP Force Plot", path: Optional[str] = None +) -> None: + """ + SHAP force plot (Matplotlib). + """ + shap_values = explainer(instance) + plt.figure(figsize=(10, 4)) + shap.plots.force(shap_values[0], show=False) + plt.title(title) + _save_matplotlib(path) + + +def generate_shap_waterfall_plot( + explainer, instance: pd.DataFrame, title: str = "SHAP Waterfall Plot", path: Optional[str] = None +) -> None: + """ + SHAP waterfall plot (Matplotlib). + """ + shap_values = explainer(instance) + plt.figure(figsize=(10, 6)) + shap.plots.waterfall(shap_values[0], show=False) + plt.title(title) + _save_matplotlib(path) + + +def infer_problem_type(predictor, df_train_full: pd.DataFrame, label_column: str) -> str: + """ + Return 'binary', 'multiclass', or 'regression'. + Prefer the predictor's own metadata when available; otherwise infer from label dtype/uniques. + """ + # AutoGluon predictors usually expose .problem_type; be defensive. + pt = getattr(predictor, "problem_type", None) + if isinstance(pt, str): + pt_l = pt.lower() + if "regression" in pt_l: + return "regression" + if "binary" in pt_l: + return "binary" + if "multiclass" in pt_l or "multiclass" in pt_l: + return "multiclass" + + y = df_train_full[label_column] + if pd.api.types.is_numeric_dtype(y) and y.nunique() > 10: + return "regression" + return "binary" if y.nunique() == 2 else "multiclass" + + +def _safe_floatify(d: Dict[str, Any]) -> Dict[str, float]: + """Make evaluate() outputs JSON/csv friendly floats.""" + out = {} + for k, v in d.items(): + try: + out[k] = float(v) + except Exception: + # keep only real-valued scalars + pass + return out + + +def evaluate_all( + predictor, + df_train: pd.DataFrame, + df_val: pd.DataFrame, + df_test: pd.DataFrame, + label_column: str, + problem_type: str, +) -> Tuple[Dict[str, float], Dict[str, float], Dict[str, float]]: + """ + Run predictor.evaluate on train/val/test and normalize the result dicts to floats. + MultiModalPredictor does not accept the `silent` kwarg, so call defensively. + """ + def _evaluate(df): + try: + return predictor.evaluate(df, silent=True) + except TypeError: + return predictor.evaluate(df) + + train_scores = _safe_floatify(_evaluate(df_train)) + val_scores = _safe_floatify(_evaluate(df_val)) + test_scores = _safe_floatify(_evaluate(df_test)) + return train_scores, val_scores, test_scores + + +def build_summary_html( + predictor, + df_train: pd.DataFrame, + df_val: Optional[pd.DataFrame], + df_test: Optional[pd.DataFrame], + label_column: str, + extra_run_rows: Optional[list[tuple[str, str]]] = None, + class_balance_html: Optional[str] = None, + perf_table_html: Optional[str] = None, +) -> str: + sections = [] + + # Dataset Overview (first section in the tab) + if class_balance_html: + sections.append(f""" +Dataset Overview
+
+ {class_balance_html}
+
+Model Performance Summary
+
+ {perf_table_html}
+
+Model Configuration
+
+
+
+
+| Key | Value |
|---|
Could not compute feature importance: {e}
" + + if fi is None or (isinstance(fi, pd.DataFrame) and fi.empty): + return "Feature importance not available for this model.
" + + # Format as a sortable table + rows = [] + if isinstance(fi, pd.DataFrame): + fi = fi.sort_values("importance", ascending=False) + for _, row in fi.iterrows(): + feat = row.index[0] if isinstance(row.index, pd.Index) else row["feature"] + imp = float(row["importance"]) + rows.append(f"No feature importance values available.
" + + table_html = f""" +| Feature | +Importance | +
|---|
Error building feature importance visualization: {e}
" + + +def build_test_html_and_plots( + predictor, + problem_type: str, + df_test: pd.DataFrame, + label_column: str, + tmpdir: str, + threshold: Optional[float] = None, +) -> Tuple[str, List[str]]: + """ + Create a test-summary section (with a placeholder for metric rows) and a list of Plotly HTML divs. + Returns: (html_template_with_{}, list_of_plot_divs) + """ + plots: List[str] = [] + + y_true = df_test[label_column].values + classes = np.unique(y_true) + + # Try proba/labels where meaningful + pred_labels = None + pred_proba = None + try: + pred_labels = predictor.predict(df_test) + except Exception: + pass + try: + # MultiModalPredictor exposes predict_proba for classification problems. + pred_proba = predictor.predict_proba(df_test) + except Exception: + pred_proba = None + + proba_arr = None + if pred_proba is not None: + if isinstance(pred_proba, pd.Series): + proba_arr = pred_proba.to_numpy().reshape(-1, 1) + elif isinstance(pred_proba, pd.DataFrame): + proba_arr = pred_proba.to_numpy() + else: + proba_arr = np.asarray(pred_proba) + + # Thresholded labels for binary + if problem_type == "binary" and threshold is not None and proba_arr is not None: + pos_label, neg_label = classes.max(), classes.min() + pos_scores = proba_arr.reshape(-1) if (proba_arr.ndim == 1 or proba_arr.shape[1] == 1) else proba_arr[:, -1] + pred_labels = np.where(pos_scores >= float(threshold), pos_label, neg_label) + + # Confusion matrix / per-class now reflect thresholded labels + if problem_type in ("binary", "multiclass") and pred_labels is not None: + cm_title = "Confusion Matrix" + if threshold is not None and problem_type == "binary": + thr_str = f"{float(threshold):.3f}".rstrip("0").rstrip(".") + cm_title = f"Confusion Matrix (Threshold = {thr_str})" + fig_cm = generate_confusion_matrix_plot(y_true, pred_labels, title=cm_title) + plots.append(plot_with_table_style_title(fig_cm, cm_title)) + + fig_pc = generate_per_class_metrics_plot(y_true, pred_labels, title="Per-Class Metrics") + plots.append(plot_with_table_style_title(fig_pc, "Per-Class Metrics")) + + # ROC/PR where possible — choose positive-class scores safely + pos_label = classes.max() # or set explicitly, e.g., 1 or "yes" + + if isinstance(pred_proba, pd.DataFrame): + proba_arr = pred_proba.to_numpy() + if pos_label in pred_proba.columns: + pos_idx = list(pred_proba.columns).index(pos_label) + else: + pos_idx = -1 # fallback to last column + elif isinstance(pred_proba, pd.Series): + proba_arr = pred_proba.to_numpy().reshape(-1, 1) + pos_idx = 0 + else: + proba_arr = np.asarray(pred_proba) if pred_proba is not None else None + pos_idx = -1 if (proba_arr is not None and proba_arr.ndim == 2 and proba_arr.shape[1] > 1) else 0 + + if proba_arr is not None: + y_bin = (y_true == pos_label).astype(int) + pos_scores = ( + proba_arr.reshape(-1) + if proba_arr.ndim == 1 or proba_arr.shape[1] == 1 + else proba_arr[:, pos_idx] + ) + + fig_roc = generate_roc_curve_plot(y_bin, pos_scores, title="ROC Curve", marker_threshold=threshold) + plots.append(plot_with_table_style_title(fig_roc, f"ROC Curve{'' if threshold is None else f' (marker at threshold={threshold:.2f})'}")) + + fig_pr = generate_pr_curve_plot(y_bin, pos_scores, title="Precision–Recall Curve", marker_threshold=threshold) + plots.append(plot_with_table_style_title(fig_pr, f"Precision–Recall Curve{'' if threshold is None else f' (marker at threshold={threshold:.2f})'}")) + + # Additional diagnostics aligned with ImageLearner style + if problem_type == "binary": + conf_fig = plot_confidence_histogram(pos_scores, bins=20, title="Prediction Confidence (Test)") + plots.append(plot_with_table_style_title(conf_fig, "Prediction Confidence (Test)")) + else: + conf_fig = plot_confidence_histogram(proba_arr, bins=20, title="Prediction Confidence (Top-1, Test)") + plots.append(plot_with_table_style_title(conf_fig, "Prediction Confidence (Top-1, Test)")) + + if problem_type == "multiclass" and proba_arr is not None and proba_arr.ndim >= 2: + fig_mc_roc = generate_multiclass_roc_curve_plot(y_true, proba_arr, classes, title="One-vs-Rest ROC (Test)") + plots.append(plot_with_table_style_title(fig_mc_roc, "One-vs-Rest ROC (Test)")) + + # Regression visuals + if problem_type == "regression": + if pred_labels is None: + pred_labels = predictor.predict(df_test) + fig_sc = generate_scatter_plot(y_true, pred_labels, title="Predicted vs Actual") + plots.append(plot_with_table_style_title(fig_sc, "Predicted vs Actual")) + + fig_res = generate_residual_plot(y_true, pred_labels, title="Residual Plot") + plots.append(plot_with_table_style_title(fig_res, "Residual Plot")) + + fig_hist = generate_residual_histogram(y_true, pred_labels, title="Residual Histogram") + plots.append(plot_with_table_style_title(fig_hist, "Residual Histogram")) + + fig_cal = generate_regression_calibration_plot(y_true, pred_labels, title="Regression Calibration") + plots.append(plot_with_table_style_title(fig_cal, "Regression Calibration")) + + # Small HTML template with placeholder for metric rows the caller fills in + test_html_template = """ +Test Performance Summary
+| Metric | Test |
|---|
Feature Importance
{fi_html}
Modalities & Inputs
{modalities_html}
Class Balance (Train Full)
{cb_html}
Feature analysis unavailable for this model.
" + + full_html = assemble_full_html_report( + summary_html, + train_html, + test_html_filled, + plots, + feature_text + transparency_blocks, + ) + with open(args.output_html, "w") as f: + f.write(full_html) + logger.info(f"Wrote HTML report → {args.output_html}") + + pred_path = _write_predictor_path(predictor) + _copy_config_if_available(pred_path, args.output_config) + + outputs_to_check = [ + (args.output_json, "JSON results"), + (args.output_html, "HTML report"), + ] + if args.output_config: + outputs_to_check.append((args.output_config, "AutoGluon config")) + verify_outputs(outputs_to_check) + + +def get_html_template() -> str: + """ + Returns the opening HTML, (with CSS/JS), and opens + .container. + Includes: + - Base styling for layout and tables + - Sortable table headers with 3-state arrows (none ⇅, asc ↑, desc ↓) + - A scroll helper class (.scroll-rows-30) that approximates ~30 visible rows + - A guarded script so initializing runs only once even if injected twice + """ + return """ + + + + +
+"""
+
+
+def get_html_closing():
+ """Closes .container, body, and html."""
+ return """
+
+
+
+"""
+
+
+def build_tabbed_html(
+ summary_html: str,
+ train_html: str,
+ test_html: str,
+ feature_html: str,
+ explainer_html: Optional[str] = None,
+) -> str:
+ """
+ Renders the tab headers, contents, and JS to switch tabs.
+ """
+ tabs = [
+ '',
+ '
')
+ tabs_section = "\n".join(tabs)
+
+ contents = [
+ f'Model Metric Summary and Config
',
+ 'Train and Validation Summary
',
+ 'Test Summary
',
+ ]
+ if explainer_html:
+ tabs.append('Explainer Plots
')
+ tabs.append('')
+ tabs.append('{summary_html}
',
+ f'{train_html}
',
+ f'{test_html}
',
+ ]
+ if explainer_html:
+ contents.append(f'{explainer_html}
')
+ content_section = "\n".join(contents)
+
+ js = """
+
+"""
+ return tabs_section + "\n" + content_section + "\n" + js
+
+
+def encode_image_to_base64(image_path: str) -> str:
+ """
+ Reads an image file from disk and returns a base64-encoded string
+ for embedding directly in HTML No label distribution available.
" + + return f""" +Label Counts by Split
+| Label | Train | Validation | Test |
|---|
Model Leaderboard (Validation)
" + lb[cols].to_html(index=False) + except Exception as e: + return f"Model Leaderboard
Unavailable: {_escape(e)}
" + + +def build_ignored_features_html(predictor, df_any: pd.DataFrame) -> str: + # MultiModalPredictor does not always expose .features(); guard accordingly. + used = set() + try: + used = set(predictor.features()) + except Exception: + # If we can't determine, don't emit a misleading section + return "" + raw_cols = [c for c in df_any.columns if c != getattr(predictor, "label", None)] + ignored = [c for c in raw_cols if c not in used] + if not ignored: + return "" + items = "".join(f"Ignored / Unused Features
+The following columns were not used by the trained predictor at inference time:
+- {items}
{html.escape(json.dumps(mm_hp, indent=2))}" if mm_hp else "Unavailable"
+ return f"Training Presets & Hyperparameters
Show hyperparameters
{hp_html}Warnings & Notices
+ {'Warnings
- '+w_html+'
Notices
- '+n_html+'
+from autogluon.multimodal import MultiModalPredictor
+predictor = MultiModalPredictor.load("{_escape(model_path)}")
+"""
+ pkg_rows = "".join(f"| Timestamp | {_escape(ctx.get('timestamp'))} |
|---|---|
| Python | {_escape(ctx.get('python_version'))} |
| Platform | {_escape(ctx.get('platform'))} |
| CPU Count | {_escape(ctx.get('cpu_count'))} |
| Memory (GB) | Total: {_escape(ctx.get('memory_total_gb'))} | Avail: {_escape(ctx.get('memory_available_gb'))} |
| Seed | {_escape(ctx.get('random_seed'))} |
| Time Limit (s) | {_escape(ctx.get('time_limit_sec'))} |
Package Versions
+| Package | Version |
|---|
Reproducibility
+Command
+{cmd}
+ {sys_table}
+ {pkgs_table}
+ Load Trained Model
+ {load_snippet or 'Model path not available'} + """ + + +def build_modalities_html(predictor, df_any: pd.DataFrame, label_col: str, image_col: Optional[str]) -> str: + """Summarize which inputs/modalities are used for MultiModalPredictor.""" + cols = [c for c in df_any.columns] + # exclude label from feature list + feat_cols = [c for c in cols if c != label_col] + # identify image vs tabular columns from args / presence + img_present = (image_col in df_any.columns) if image_col else False + tab_cols = [c for c in feat_cols if c != image_col] + + # brief lists (avoid dumping all, unless small) + def list_or_count(arr, max_show=20): + if len(arr) <= max_show: + items = "".join(f"- {items}
{len(arr)} columns
" + + img_block = f"Image column: {html.escape(image_col)}
" if img_present else "Image column: None
" + tab_block = f"Structured columns: {len(tab_cols)}{list_or_count(tab_cols, max_show=15)}
"
+
+ return f"""
+ Modalities & Inputs
+This run used MultiModalPredictor (images + structured features).
+Label column: {html.escape(label_col)}
+ {img_block} + {tab_block} + """ + + +def build_model_performance_summary_table( + train_scores: dict, + val_scores: dict, + test_scores: dict | None = None, + include_test: bool = True, + title: str | None = 'Model Performance Summary', + show_title: bool = True, +) -> str: + """ + Returns an HTML table for metrics, optionally hiding the Test column. + Keys across score dicts are unioned; missing values render as '—'. + """ + def fmt(v): + if v is None: + return '—' + if isinstance(v, (int, float)): + return f'{v:.4f}' + return str(v) + + # Collect union of metric keys across splits + metrics = set(train_scores.keys()) | set(val_scores.keys()) | (set(test_scores.keys()) if (include_test and test_scores) else set()) + + # Remove AG_roc_auc entirely as requested + metrics.discard('AG_roc_auc') + + # Helper: normalize metric keys for matching preferred names + def _norm(k: str) -> str: + return ''.join(ch for ch in str(k).lower() if ch.isalnum()) + + # Preferred metrics to appear at the end in this specific order (display names): + preferred_display = ['Accuracy', 'ROC-AUC', 'Precision', 'Recall', 'F1-Score', 'PR-AUC', 'Specificity', 'MCC', 'LogLoss'] + # Mapping of normalized key -> display label + norm_to_display = { + 'accuracy': 'Accuracy', + 'acc': 'Accuracy', + 'rocauc': 'ROC-AUC', + 'roc_auc': 'ROC-AUC', + 'rocaucscore': 'ROC-AUC', + 'precision': 'Precision', + 'prec': 'Precision', + 'recall': 'Recall', + 'recallsensitivitytpr': 'Recall', + 'f1': 'F1-Score', + 'f1score': 'F1-Score', + 'pr_auc': 'PR-AUC', + 'prauc': 'PR-AUC', + 'averageprecision': 'PR-AUC', + 'specificity': 'Specificity', + 'tnr': 'Specificity', + 'mcc': 'MCC', + 'logloss': 'LogLoss', + 'crossentropy': 'LogLoss', + } + + # Build ordered list: all non-preferred metrics sorted alphabetically, then preferred metrics in the requested order if present + preferred_norms = [_norm(x) for x in preferred_display] + all_metrics = list(metrics) + # Partition + preferred_present = [] + others = [] + for m in sorted(all_metrics): + nm = _norm(m) + if nm in preferred_norms or any( + p in nm for p in ["rocauc", "prauc", "f1", "mcc", "logloss", "accuracy", "precision", "recall", "specificity"] + ): + # Defer preferred-like metrics to the end (we will place them in canonical order) + preferred_present.append(m) + else: + others.append(m) + + # Now assemble final metric order: others (alpha), then preferred in exact requested order if they exist in metrics + final_metrics = [] + final_metrics.extend(others) + for disp in preferred_display: + # find any original key matching this display (by normalized mapping) + target_norm = _norm(disp) + found = None + for m in preferred_present: + if _norm(m) == target_norm or norm_to_display.get(_norm(m)) == disp or _norm(m).replace(' ', '') == target_norm: + found = m + break + # also allow substring matches (e.g., 'roc_auc' vs 'rocauc') + if target_norm in _norm(m): + found = m + break + if found: + final_metrics.append(found) + + metrics = final_metrics + + # Make all headers sortable by adding the 'sortable' class; the JS in utils.py hooks table.performance-summary + header_cells = [ + '{title}
' if (show_title and title) else '' + + table_html = f""" + {title_html} +
+
+
+
+
+
+
+
diff -r 000000000000 -r 375c36923da1 test-data/test.csv
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/test.csv Tue Dec 09 23:49:47 2025 +0000
@@ -0,0 +1,51 @@
+feature_0,feature_1,feature_2,feature_3,feature_4,text,target,image_path
+0.25739992534469336,-0.288440183,0.20367539201504153,2.504864685302447,2.0388993649376554,Hypopharynxkarzinom[C13.9 ] Larynxkarzinom[C32.9 ] Halslymphknotenmetastasen[C77.0 B],0,image_0.png
+-0.908481433,-1.067713065,-1.8565207,-1.333461127,0.3848107911787176,Supraglottisches Karzinom[C32.1 L] Halslymphknotenmetastasen[C77.0 ],1,image_1.png
+-0.378503106,-1.077600898,0.3781894561704642,-1.028272654,-0.700527555,Bösartige Neubildung der Gaumenbogentonsillen[C09.1 L],1,image_2.png
+-0.534915599,-0.796773763,1.2361403151716697,1.3631696640866262,-1.497154707,"Karzinom des Sulcus glossopalatinus[C09.1 ] Neubildung unsicheren oder unbekannten Verhaltens: Lippe, Mundhöhle und Pharynx[D37.0 ]",1,image_3.png
+0.8580733460716252,-1.486042582,-0.689971804,-1.142274593,-0.097944716,Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 R],0,image_4.png
+-0.413009982,0.5141287656735564,-0.198354692,-1.372025465,-0.749649593,Bösartige Neubildung: Weicher Gaumen[C05.1 ] Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 L],1,image_5.png
+0.49818858448728287,0.8517908644430702,-3.025739873,2.1153714355164954,-0.423670688,Glottiskarzinom[C32.0 R] Halslipom[D17.0 R],1,image_6.png
+2.0101992475719856,0.9586734372734298,-1.245626223,-0.274274886,0.3456524588826128,Bösartige Neubildung: Glottis[C32.0 ],0,image_7.png
+1.2628615445176543,-0.626484049,-2.084039472,-0.250145423,0.23501780256852894,Bösartige Neubildung Hypopharynx mehrere Teilbereiche überlappend[C13.8 ] Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 ],1,image_8.png
+-0.439214857,0.3079310128755357,0.10412164629262519,-0.5088304,0.19390051775098527,Supraglottisches Karzinom[C32.1 ],0,image_9.png
+-0.346437893,0.005205685,-0.150233494,0.5813407294084169,1.3922625492889247,,0,image_10.png
+0.4553196595572467,0.6915319076425244,-0.105816503,-1.133100081,0.4079556930770453,Halslymphknotenmetastasen[C77.0 B],1,image_11.png
+-1.668662707,0.4448621590773117,-0.436954139,1.7743892245873165,2.7134802556937045,Neubildung bösartig Gaumenbogen (vorderer) (hinterer)[C09.1 ],1,image_12.png
+-0.862085495,0.090279532,1.019136316265232,0.7530463240026412,0.1613349250415128,"Bösartige Neubildung: Oropharynx, nicht näher bezeichnet[C10.9 ]",0,image_13.png
+0.49291084812374153,-1.858342899,-1.232803226,-0.051800232,-0.40689619,Zungenrandkarzinom[C02.1 R] Halslymphknotenmetastasen[C77.0 B],1,image_14.png
+-0.124313396,-0.166580044,-0.074124538,-0.551705599,-1.497093461,"Zungenrandkarzinom[C02.1 ] Neubildung unsicheren oder unbekannten Verhaltens: Lippe, Mundhöhle und Pharynx[D37.0 ]",0,image_15.png
+1.9351362874003422,0.11087648013745463,0.5963891827515404,0.10825694005967285,-1.314371547,Bösartige Neubildung der Seitenwand des Oropharynx[C10.2 L],1,image_16.png
+-0.618442655,-0.694772644,-0.179392562,1.0439696514243961,0.9068151425024749,Bösartige Neubildung: Glottis[C32.0 L],0,image_17.png
+-1.046838986,-0.269175566,-0.090795068,2.6174426286362076,0.6489323286331116,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ],1,image_18.png
+-0.889617589,-1.29922515,-0.825079629,1.508768374358993,-0.089698726,,0,image_19.png
+0.014040537,-0.321105447,-0.334969461,1.012902487072667,-0.224378814,Bösartige Neubildung: Zungenrand[C02.1 ],1,image_20.png
+-0.160829686,0.5058687421604646,0.7183508075095323,0.043045740547390625,-0.887604847,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ] Bösartige Neubildung: Hypopharynx, mehrere Teilbereiche überlappend[C13.8 ]",0,image_21.png
+2.2303596488319384,2.0890595712039643,-2.019462185,1.2325046106386215,-0.16636758,Tonsillenkarzinom[C09.9 ] Halslymphknotenmetastasen[C77.0 ],1,image_22.png
+-0.399115719,-1.012709255,-0.158029049,-0.346410841,0.1234889741068257,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",0,image_23.png
+0.054444563,-0.02397407,-0.584215128,0.3318833834396524,-2.017685901,Bösartige Neubildung der Glottis[C32.0 R],1,image_24.png
+0.8841818151951191,-0.96146905,-0.070306498,1.1706959283854066,0.7251931071367159,Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 R],1,image_25.png
+-0.107980561,-0.092566189,1.7731812300260754,0.6067736917185195,-2.02724386,cT1b-Glottis-CA[C32.0 ],1,image_26.png
+0.5556069842518202,-0.223732081,-0.446629293,-0.664692993,0.33697833144899764,Neubildung bösartig Hypopharynx sonstige[C13.8 L] Neubildung bösartig Hypopharynx sonstige[C13.8 L],0,image_27.png
+0.3949066359501769,0.832892162,1.4893745786091637,-1.532773874,-0.044698324,Bösartige Neubildung: Vallecula epiglottica[C10.0 ],0,image_28.png
+0.8372050226472976,0.9741195783904064,1.100308721372732,0.5014966096525523,0.29207407178086175,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ] Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ]",1,image_29.png
+-1.40787817,0.16281815769810043,0.029896317954142267,0.13463643962491514,-0.648498896,"Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ] Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ]",1,image_30.png
+0.8078494136349023,-0.11449202,0.087105811,0.9299897685604038,1.6934782478194783,,1,image_31.png
+-0.138283645,1.1864684326166455,0.18123494219161268,0.057889462,0.7180924802384169,Bösartige Neubildung der Glottis[C32.0 L],1,image_32.png
+0.18717858533399576,0.17979165086317853,-0.825171797,-0.379583634,0.6354161714637024,Halslymphknotenmetastasen[C77.0 L] Tonsillenkarzinom[C09.9 L],0,image_33.png
+-0.386658136,1.5164416176334854,0.6268387669020417,1.0539531037975378,-0.178935605,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 R],1,image_34.png
+1.6590487297919927,-1.63403202,-0.612697847,0.9963133530846225,-0.075254355,Bösartige Neubildung der Seitenwand des Oropharynx[C10.2 L],1,image_35.png
+-2.047069131,1.7819709009957654,0.035742336,-2.109193418,0.8856980875005901,,0,image_36.png
+1.3993169863061743,-0.617727701,-2.195962611,-1.519115605,1.3022907171692861,,1,image_37.png
+-0.679007121,-1.086399293,-0.863282689,0.12983462557747558,1.4001751255531105,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 ],1,image_38.png
+1.5289851345959593,-0.044753328,0.5350993630394176,-0.065237868,-0.046387298,"Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ]",1,image_39.png
+1.2212159632534685,-1.16370242,-1.123463295,-0.18698505,0.056701348,Bösartige Neubildung: Zungenrand[C02.1 ],1,image_40.png
+1.0149885167826984,0.5689394950042188,0.33396509272135233,0.8239324982417736,0.33108021768184237,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 ],1,image_41.png
+0.8281299823497013,-0.462643856,-0.507230455,2.0154984149609985,-0.261780894,Bösartige Neubildung Oropharynx Seitenwand[C10.2 ] Bösartige Neubildung Hypopharynx mehrere Teilbereiche überlappend[C13.8 ] Bösartige Neubildung Nasopharynx Seitenwand[C11.2 ],1,image_42.png
+2.2662927082101914,0.28735510483252746,0.9901501711281284,-1.031448133,-0.70926492,"Bösartige Neubildung: Larynx, mehrere Teilbereiche überlappend[C32.8 ]",1,image_43.png
+-0.59495567,-0.977338711,1.2579439688721659,-0.067517776,0.16645918553019404,Bösartige Neubildung Hypopharynx mehrere Teilbereiche überlappend[C13.8 ],0,image_44.png
+-0.581269538,-0.222436258,0.17762491871456892,-0.025381558,0.47332265126602674,Karzinom Gaumen und Uvula[C05.8 ],1,image_45.png
+-0.655894146,0.096931768,-0.014867416,0.2052931948319547,0.5237041770156639,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 B] Halslymphknotenmetastasen[C77.0 B],0,image_46.png
+0.9251488533568732,-0.188384349,0.040436114,-1.223639731,-0.075535658,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",1,image_47.png
+-1.299161344,1.5916021583729445,-0.02144477,0.46742987833997945,-0.658039055,,0,image_48.png
+1.0111668672107617,0.5858108272747818,1.1916565829964165,0.036701016,0.4992660132510926,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 R] Dyspnoe onA[R06.0 ],0,image_49.png
\ No newline at end of file
diff -r 000000000000 -r 375c36923da1 test-data/train.csv
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/train.csv Tue Dec 09 23:49:47 2025 +0000
@@ -0,0 +1,177 @@
+feature_0,feature_1,feature_2,feature_3,feature_4,text,target,image_path
+0.4967141530112327,-1.415370742,0.3577873603482833,-0.828995011,-1.594427659,Glottiskarzinom[C32.0 L],1,image_0.png
+-0.138264301,-0.420645323,0.5607845263682344,-0.56018104,-0.599375023,Bösartige Neubildung: Zungenrand[C02.1 ],1,image_1.png
+0.6476885381006925,-0.342714517,1.083051243175277,0.7472936051232618,0.0052437,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ],1,image_2.png
+1.5230298564080254,-0.802277269,1.053802052034903,0.6103702654334648,0.046980593764742055,Neubildung unsicheren oder unbekannten Verhaltens: Sonstige näher bezeichnete Lokalisationen[D48.7 ] Rachenmandelkarzinom[C11.1 R] Leukoplakie Stimmlippe[J38.3 L],0,image_3.png
+-0.234153375,-0.161285712,-1.377669368,-0.020901594,-0.450065471,Bösartige Neubildung der Supraglottis[C32.1 R],1,image_4.png
+-0.234136957,0.4040508568145384,-0.93782504,0.117327383,0.6228499323474987,Uvulakarzinom[C05.2 ],0,image_5.png
+1.5792128155073915,1.8861859012105302,0.5150352672086598,1.277664895788425,-1.067620429,"Bösartige Neubildung: Zunge, mehrere Teilbereiche überlappend[C02.8 ] Bösartige Neubildung im Bereich der Zungengruben[C10.0 ]",0,image_6.png
+0.7674347291529088,0.17457781283183896,0.5137859509122088,-0.591571389,-0.142379485,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ],1,image_7.png
+-0.469474386,0.25755039072276437,0.5150476863060479,0.5470973811700379,0.12029563171189886,Neubildung bösartig Hypopharynx sonstige[C13.8 L],0,image_8.png
+0.5425600435859647,-0.074445916,3.852731490654721,-0.202192652,0.514438834,Bösartige Neubildung: Glottis[C32.0 ],0,image_9.png
+-0.463417693,-1.918771215,0.570890511,-0.217681203,0.7116148780888898,Unsichere Neubildung der Übergangsregion des Oropharynx[D37.0 ],0,image_10.png
+-0.465729754,-0.026513875,1.135565640180599,1.098776852,-1.124642092,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 R],0,image_11.png
+0.24196227156603412,0.06023021,0.9540017634932023,0.8254163489880298,-1.534114171,Bösartige Neubildung: Glottis[C32.0 ],1,image_12.png
+-1.913280245,2.463242112485286,0.651391251,0.8135096360006385,1.277676821898509,"Bösartige Neubildung: Hypopharynx, mehrere Teilbereiche überlappend[C13.8 ]",0,image_13.png
+-1.724917833,-0.192360965,-0.315269245,1.305478807154329,0.33231401197959165,"Bösartige Neubildung: Hypopharynx, mehrere Teilbereiche überlappend[C13.8 ]",1,image_14.png
+-0.562287529,0.30154734233361247,0.7589692204932674,0.021003842,-0.748486537,Halslymphknotenmetastasen[C77.0 B],1,image_15.png
+-1.01283112,-0.03471177,-0.772825215,0.6819529712949639,1.551151975522523,Bösartige Neubildung: Glottis[C32.0 ] Bösartige Neubildung der Glottis[C32.0 B],0,image_16.png
+0.3142473325952739,-1.168678038,-0.236818607,-0.310266757,0.11567463429285867,Karzinom des Zungenrandes und der Zungenspitze[C02.1 ],1,image_17.png
+-0.908024076,1.1428228145150205,-0.485363548,0.3241663524884421,1.1792971840638264,CUP [Cancer of Unknown Primary][C80.0 ] Halsmetastasen[C77.0 ],1,image_18.png
+-1.412303701,0.7519330326867741,0.081874139,-0.130143054,0.067518481,Zungenrandkarzinom[C02.1 ],1,image_19.png
+1.465648768921554,0.7910319470430469,2.3146585666735087,0.096995965,2.060747924881987,Bösartige Neubildung: Zungenrand[C02.1 ] Neubildung bösartig Mundboden sonstige[C04.8 R] Halslymphknotenmetastasen[C77.0 B],0,image_20.png
+-0.2257763,-0.909387455,-1.867265193,0.5951570254369136,1.7553408424432044,Tonsillenkarzinom[C09.9 ] Halslymphknotenmetastasen[C77.0 ],1,image_21.png
+0.067528205,1.4027943109360992,0.6862601903745135,-0.818220683,-0.248964148,Bösartige Neubildung: Glottis[C32.0 ],1,image_22.png
+-1.424748186,-1.401851063,-1.612715871,2.0923872756854602,0.9715709509543554,Glottiskarzinom[C32.0 L],1,image_23.png
+-0.544382725,0.5868570938002703,-0.471931866,-1.006017381,0.6453759495851475,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 L],0,image_24.png
+0.11092258970986608,2.1904556258099785,1.088950596967366,-1.214188613,1.3686315575323487,Bösartige Neubildung: Glottis[C32.0 ] Neubildung bösartig Kehlkopf sonstige[C32.8 L],1,image_25.png
+-1.150993577,-0.990536325,0.064280019,1.1581108735000678,-0.964923461,"Bösartige Neubildung: Oropharynx, nicht näher bezeichnet[C10.9 ]",0,image_26.png
+0.37569801834567196,-0.56629773,-1.077744778,0.7916626939629359,0.6860514599984393,"Bösartige Neubildung: Hypopharynx, nicht näher bezeichnet[C13.9 ]",0,image_27.png
+-0.60063869,0.099651365,-0.715303709,0.6241198170521551,1.0584244868495878,Bösartige Neubildung des inneren Larynx[C32.0 L],0,image_28.png
+-0.29169375,-0.503475654,0.6795977489346758,0.6283455092642799,-1.758739486,Bösartige Neubildung des postkrikoidalen Hypopharynx[C13.0 ] Neubildung bösartig Hypopharynx sonstige[C13.8 ],0,image_29.png
+-0.601706612,-1.550663431,-0.730366632,-0.012246773,-1.183258513,Karzinom bei unbekanntem Primärtumor (CUP)[C80.0 ],0,image_30.png
+1.8522781845089378,0.068562975,0.21645858958197486,-0.897254371,-2.039232178,Bösartige Neubildung: Seitenwand des Oropharynx[C10.2 ],0,image_31.png
+-0.013497225,-1.062303714,0.045571839903813784,0.075804558,-0.269406834,Nachblutung[T81.0 B],1,image_32.png
+-1.057710929,0.4735924306351816,-0.651600348,-0.677161712,0.7175422557959623,Unsichere Neubildung des seitlichen Zungenrandes[D37.0 R],0,image_33.png
+0.822544912,-0.919424234,2.1439440893253257,0.9751197334177512,1.502357052096028,Bösartige Neubildung des Mundrachenraums[C14.8 ],1,image_34.png
+-1.22084365,1.5499344050175394,0.6339190223180112,-0.147057382,0.07409478,Karzinom Oropharynx Seitenwand[C10.2 R] Neubildung bösartig sekundär und onA Lymphknoten Kopf Gesicht Hals[C77.0 R],1,image_35.png
+0.2088635950047554,-0.783253292,-2.025142587,-0.825497197,1.6286155455712918,Zungenrandkarzinom[C02.1 ],0,image_36.png
+-1.959670124,-0.322061516,0.18645431476942764,-0.321385842,-1.380101458,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 B],1,image_37.png
+-1.328186049,0.8135172173696698,-0.661786465,0.41293145427562433,-1.703382439,Bösartige Neubildung des weichen Gaumens[C05.1 R],1,image_38.png
+0.19686123586912352,-1.230864316,0.852433335,-0.563724553,-0.055547699,,1,image_39.png
+0.7384665799954104,0.22745993460412942,-0.792520738,-0.822220396,0.3840654489393073,Stimmlippenkarzinom[C32.0 L] Struma nodosa euthyreot[E04.1 ],1,image_40.png
+0.1713682811899705,1.307142754282428,-0.114736441,0.2436872114919123,-0.032694748,Glottiskarzinom[C32.0 L],0,image_41.png
+-0.115648282,-1.607483235,0.5049872789804571,0.24496657110872277,-2.0674421,Bösartige Neubildung: Glottis[C32.0 ],1,image_42.png
+-0.301103696,0.1846338585323042,0.8657551941701215,-0.506943175,-0.08912004,Neubildung unsicheren oder unbekannten Verhaltens: Larynx[D38.0 ] Neubildung bösartig Kehlkopf sonstige[C32.8 L],0,image_43.png
+-1.47852199,0.25988279424842353,-1.200296407,-0.471038306,-1.304469501,"Bösartige Neubildung: Larynx, mehrere Teilbereiche überlappend[C32.8 ]",1,image_44.png
+-0.719844208,0.7818228717773104,-0.334501236,0.2320499373576363,0.6696725488300385,Bösartige Neubildung der Glottis[C32.0 ],1,image_45.png
+-0.460638771,-1.236950711,-0.474945311,-1.448084341,0.36659824609684827,Bösartige Neubildung des Zungenrandes[C02.1 ],0,image_46.png
+1.0571222262189157,-1.320456613,-0.653329233,-1.407463774,-0.939879786,"Neubildung unsicheren oder unbekannten Verhaltens: Lippe, Mundhöhle und Pharynx[D37.0 ] Funktionsstörung des Tracheostomas[J95.0 ]",1,image_47.png
+0.3436182895684614,0.5219415656168976,1.7654542402810969,-0.718444221,-0.513866917,Stimmlippenkarzinom[C32.0 R],1,image_48.png
+-1.763040155,0.29698467323318606,0.40498171096095553,-0.213447152,-1.059213522,Karzinom des Mundrachenraums[C14.8 R],1,image_49.png
+0.324083969,0.25049285034587654,-1.260883954,0.3109075655980046,-0.062679097,"Bösartige Neubildung: Larynx, mehrere Teilbereiche überlappend[C32.8 ]",0,image_50.png
+-0.38508228,0.3464482094969757,0.9178619470547761,1.475356216949552,0.9551423205012383,,1,image_51.png
+-0.676922,-0.680024722,2.1221561970126332,0.8576596232020194,-0.985726046,Karzinom Mundboden mehrere Teilbereiche überlappend[C04.8 L],0,image_52.png
+0.6116762888408679,0.23225369716100355,1.0324652605511468,-0.15993853,0.5040465155178444,Bösartige Neubildung: Glottis[C32.0 ],1,image_53.png
+1.030999522495951,0.29307247329868125,-1.519369966,-0.019016208,-0.530257618,Bösartige Neubildung der Supraglottis[C32.1 B],0,image_54.png
+0.9312801191161986,-0.714351418,-0.484234073,-1.002529365,-0.792872832,Bösartige Neubildung Hypopharynx mehrere Teilbereiche überlappend[C13.8 L],0,image_55.png
+-0.839217523,1.8657745111447566,1.2669111491866227,-0.018513136,-0.10703036,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 ],1,image_56.png
+-0.309212376,0.4738329209117875,-0.707669466,-0.288658639,-1.035242322,"Bösartige Neubildung der Gaumenbogentonsillen[C09.1 L] Lymphknotenvergrößerung, umschrieben[R59.0 ]",1,image_57.png
+0.33126343140356396,-1.191303497,0.44381942814622843,0.3227185603380895,-0.553649305,Bösartige Neubildung des Zungenrandes und der Zungenspitze[C02.1 ],0,image_58.png
+0.9755451271223592,0.6565536086338297,0.7746340534293368,-0.827230944,-1.197877893,Karzinom der Plica aryepiglottica[C13.1 L],0,image_59.png
+-0.479174238,-0.97468167,-0.926930472,0.5193465142411723,1.9647251329163893,Bösartige Neubildung: Glottis[C32.0 ],1,image_60.png
+-0.185658977,0.787084604,-0.059525356,1.5327389130025777,0.035263552,"Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ]",0,image_61.png
+-1.106334974,1.158595579007404,-3.24126734,-0.108760148,-0.699725508,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 R],1,image_62.png
+-1.196206624,-0.820682318,-1.024387641,0.40171172209894146,0.213979911,Bösartige Neubildung: Glottis[C32.0 ] Glottiskarzinom[C32.0 R],1,image_63.png
+0.812525822,0.9633761292443218,-0.252568151,0.6901439917111125,-0.11232805,Glottiskarzinom[C32.0 L] Bösartige Neubildung der Glottis[C32.0 L],1,image_64.png
+1.356240028570823,0.4127809269364983,-1.247783182,-0.401220472,-0.2209696,,1,image_65.png
+-0.072010122,0.82206016,1.6324113039316352,0.22409248181041677,0.6141667000434252,Bösartige Neubildung der Zungenunterfläche[C02.2 R],0,image_66.png
+1.0035328978920242,1.8967929826539474,-1.430141378,0.012592401,0.7575077100473051,Bösartige Neubildung Hypopharynx mehrere Teilbereiche überlappend[C13.8 L] Neubildung bösartig sekundär und onA Lymphknoten Kopf Gesicht Hals[C77.0 B],1,image_67.png
+0.36163602504763415,-0.245388116,-0.440044487,0.097676099,-0.530501148,Rachenmandelkarzinom[C11.1 ],0,image_68.png
+-0.645119755,-0.753736164,0.13074057728609134,-0.773009784,-0.575818241,"Bösartige Neubildung: Hypopharynx, mehrere Teilbereiche überlappend[C13.8 ] Halslymphknotenmetastasen[C77.0 B]",1,image_69.png
+0.36139560550841393,-0.88951443,1.4412732890661155,0.024510174258942714,-0.275051697,"Bösartige Neubildung: Oropharynx, nicht näher bezeichnet[C10.9 ]",0,image_70.png
+1.5380365664659692,-0.815810285,-1.435862151,0.49799829124544975,-2.301921165,Bösartige Neubildung: Supraglottis[C32.1 ],1,image_71.png
+-0.035826039,-0.077101709,1.1631637521549596,1.4511436077950417,-1.515191062,Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 L],0,image_72.png
+1.5646436558140062,0.3411519748166439,0.010233061019587049,0.9592708260852069,1.3668742674445247,Bösartige Neubildung Oropharynx mehrere Teilbereiche überlappend[C10.8 ],0,image_73.png
+-2.619745104,0.27669079933001905,-0.981508651,2.1531824575115563,1.6449677135012837,Carcinoma in situ der Epiglottis[D02.0 ],1,image_74.png
+0.8219025043752238,0.8271832490360238,0.46210347426327075,-0.767347563,-0.24903604,Bösartige Neubildung der Gaumenbogentonsillen[C09.1 R],1,image_75.png
+0.087047068,0.013001892,0.19905969557347003,0.8723206367206782,0.5765569630557664,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",1,image_76.png
+-0.29900735,1.4535340771573169,-0.600216877,0.18334200573835174,0.3112501545435361,Uvulakarzinom[C05.2 ],1,image_77.png
+0.091760777,-0.264656833,0.069802085,2.1898029332176723,3.0788808084552377,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",0,image_78.png
+-1.987568915,2.720169166589619,-0.385313597,-0.808298285,1.1195749114345768,Bösartige Neubildung: Glottis[C32.0 ],1,image_79.png
+-0.219671888,0.6256673477650062,0.11351734525124804,-0.839721842,-0.127917591,Bösartige Neubildung der Glottis[C32.0 ],0,image_80.png
+0.3571125715117464,-0.857157556,0.6621306745210467,-0.599392645,-0.955540441,Stimmbandkarzinom[C32.0 L],0,image_81.png
+1.477894044741516,-1.070892498,1.586016816145352,-2.123895724,-1.60644632,,1,image_82.png
+-0.518270218,0.4824724152431853,-1.237815499,-0.525755022,0.2034636358672231,Zungenrandkarzinom[C02.1 ],0,image_83.png
+-0.808493603,-0.223462785,2.1330333746562666,-0.759132662,-0.756350745,Bösartige Neubildung: Zungenrand[C02.1 ] Bösartige Neubildung: Zungenrand[C02.1 ],0,image_84.png
+-0.501757044,0.714000494,-1.9520878,0.15039378647620763,-1.42225371,"Neubildung unsicheren oder unbekannten Verhaltens: Lippe, Mundhöhle und Pharynx[D37.0 ]",0,image_85.png
+0.9154021177020741,0.47323762457354485,-0.151785095,0.34175597577715944,-0.646572884,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 R],0,image_86.png
+0.32875110965968446,-0.072828913,0.5883172064845765,1.8761708392158862,-1.081548004,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 R] Halslymphknotenmetastasen[C77.0 B],1,image_87.png
+-0.529760204,-0.846793718,0.28099186773503265,0.9504238381860503,1.6871416350725648,,1,image_88.png
+0.5132674331133561,-1.514847225,-0.62269952,-0.576903656,0.8816397569494505,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ] Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ],1,image_89.png
+0.097077549,-0.446514952,-0.20812225,-0.898414671,-0.007972641,,1,image_90.png
+0.9686449905328892,0.8563987943234723,-0.493000935,0.4919191715065057,1.4799441388900259,Bösartige Neubildung: Glottis[C32.0 ],0,image_91.png
+-0.702053094,0.21409374413020396,-0.589364757,-1.320233207,0.077368308,,0,image_92.png
+-0.327662147,-1.245738779,0.8496020970210246,1.8314587658543537,-0.861284201,Bösartige Neubildung des Zungenrandes[C02.1 R],1,image_93.png
+-0.392108153,0.173180926,0.35701548596504734,1.179440120721287,1.5231240772696573,Supraglottisches Karzinom[C32.1 R],1,image_94.png
+-1.463514948,0.3853173797288368,-0.692909595,-0.469175652,0.5389100436846587,Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 ] Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 ],1,image_95.png
+0.29612027706457605,-0.883857436,0.8995998754332507,-1.713134529,-1.037246154,,0,image_96.png
+0.26105527217988933,0.1537251059455279,0.30729952087660933,1.3538723741654128,-0.190338678,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 B],1,image_97.png
+0.005113457,0.058208718445999896,0.8128621188389601,-0.114539845,-0.875618253,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",1,image_98.png
+-0.234587133,-1.142970298,0.6296288419236122,1.2378163119734618,-1.382799731,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ] Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 L]",1,image_99.png
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,,,
+,,,,,Glottiskarzinom[C32.0 L],,
+,,,,,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",,
+,,,,,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 L],,
+,,,,,Larynxkarzinom[C32.9 ],,
+,,,,,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",,
+,,,,,Bösartige Neubildung: Zungenrand[C02.1 ] Zungenrandkarzinom[C02.1 ],,
+,,,,,Bösartige Neubildung: Glottis[C32.0 ],,
+,,,,,"Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ]",,
+,,,,,,,
+,,,,,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 B],,
+,,,,,Bösartige Neubildung der Supraglottis[C32.1 ],,
+,,,,,Karzinom Tonsille mehrere Teilbereiche überlappend[C09.8 R],,
+,,,,,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 ],,
+,,,,,Bösartige Neubildung der Glottis[C32.0 L],,
+,,,,,"Bösartige Neubildung: Zunge, nicht näher bezeichnet[C02.9 ]",,
+,,,,,"Bösartige Neubildung: Zunge, mehrere Teilbereiche überlappend[C02.8 ]",,
+,,,,,"Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ] Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",,
+,,,,,Bösartige Neubildung der Fossa tonsillaris[C09.0 R],,
+,,,,,"Neubildung unsicheren oder unbekannten Verhaltens: Lippe, Mundhöhle und Pharynx[D37.0 ] Bösartige Neubildung der Zungenunterfläche[C02.2 ]",,
+,,,,,Neubildung bösartig Hypopharynx sonstige[C13.8 ],,
+,,,,,Karzinom der ventralen Zunge[C02.2 ],,
+,,,,,Bösartige Neubildung des weichen Gaumens[C05.1 L],,
+,,,,,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 R],,
+,,,,,Halsmetastase[C79.88 R],,
+,,,,,Karzinom Oropharynx mehrere Teilbereiche überlappend[C10.8 ],,
+,,,,,"Bösartige Neubildung: Hypopharynx, nicht näher bezeichnet[C13.9 ]",,
+,,,,,Bösartige Neubildung: Zungenrand[C02.1 ],,
+,,,,,Glottiskarzinom[C32.0 R],,
+,,,,,Supraglottisches Karzinom[C32.1 B] Halslymphknotenmetastasen[C77.0 B],,
+,,,,,CUP [Cancer of Unknown Primary][C80.0 L],,
+,,,,,Bösartige Neubildung: Zungenrand[C02.1 ],,
+,,,,,"Bösartige Neubildung: Hypopharynx, mehrere Teilbereiche überlappend[C13.8 ]",,
+,,,,,Uvulakarzinom[C05.2 ],,
+,,,,,Karzinom Uvula[C05.2 ],,
+,,,,,Karzinom Hypopharynx mehrere Teilbereiche überlappend[C13.8 R],,
+,,,,,"Bösartige Neubildung: Oropharynx, mehrere Teilbereiche überlappend[C10.8 ]",,
+,,,,,cT3 OrpharynxCa links[H93.1 ] Bösartige Neubildung Oropharynx mehrere Teilbereiche überlappend[C10.8 L],,
+,,,,,Bösartige Neubildung: Uvula[C05.2 ],,
+,,,,,"Bösartige Neubildung: Tonsille, mehrere Teilbereiche überlappend[C09.8 ]",,
+,,,,,"Bösartige Neubildung: Larynx, mehrere Teilbereiche überlappend[C32.8 ]",,
+,,,,,Bösartige Neubildung: Vallecula epiglottica[C10.0 ],,
+,,,,,Bösartige Neubildung: Zungenrand[C02.1 ] Bösartige Neubildung Zunge mehrere Teilbereiche überlappend[C02.8 L],,
+,,,,,Bösartige Neubildung Larynx mehrere Teilbereiche überlappend[C32.8 ],,
\ No newline at end of file
diff -r 000000000000 -r 375c36923da1 test_pipeline.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test_pipeline.py Tue Dec 09 23:49:47 2025 +0000
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import logging
+from typing import Dict, Optional
+
+import pandas as pd
+from plot_logic import infer_problem_type
+from training_pipeline import evaluate_predictor_all_splits, fit_summary_safely
+
+logger = logging.getLogger(__name__)
+
+
+def run_autogluon_test_experiment(
+ predictor,
+ data_ctx: Dict[str, pd.DataFrame],
+ target_column: str,
+ eval_metric: Optional[str] = None,
+ ag_config: Optional[dict] = None,
+ problem_type: Optional[str] = None,
+) -> Dict[str, object]:
+ """
+ Evaluate a trained predictor on train/val/test splits using prepared data_ctx.
+
+ data_ctx is typically the context returned by ``run_autogluon_experiment``:
+ {
+ "train": df_train,
+ "val": df_val,
+ "test_internal": df_test_internal,
+ "test_external": df_test_external,
+ "threshold": threshold,
+ }
+ """
+ if predictor is None:
+ raise ValueError("predictor is required for evaluation.")
+ if data_ctx is None:
+ raise ValueError("data_ctx is required; usually from run_autogluon_experiment.")
+
+ df_train = data_ctx.get("train")
+ df_val = data_ctx.get("val")
+ df_test_internal = data_ctx.get("test_internal")
+ df_test_external = data_ctx.get("test_external")
+ threshold = None
+ if ag_config is not None:
+ threshold = ag_config.get("threshold", threshold)
+ threshold = data_ctx.get("threshold", threshold)
+
+ if problem_type is None:
+ # Prefer inferring from training data and predictor metadata
+ base_df = df_train if df_train is not None else df_test_external
+ problem_type = infer_problem_type(predictor, base_df, target_column)
+
+ df_test_final = df_test_external if df_test_external is not None else df_test_internal
+ raw_metrics, ag_by_split = evaluate_predictor_all_splits(
+ predictor=predictor,
+ df_train=df_train,
+ df_val=df_val,
+ df_test=df_test_final,
+ label_col=target_column,
+ problem_type=problem_type,
+ eval_metric=eval_metric,
+ threshold_test=threshold,
+ df_test_external=df_test_external,
+ )
+
+ summary = fit_summary_safely(predictor)
+
+ result = {
+ "problem_type": problem_type,
+ "raw_metrics": raw_metrics,
+ "ag_eval": ag_by_split,
+ "fit_summary": summary,
+ }
+ logger.info("Evaluation complete; splits: %s", list(raw_metrics.keys()))
+ return result
diff -r 000000000000 -r 375c36923da1 training_pipeline.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/training_pipeline.py Tue Dec 09 23:49:47 2025 +0000
@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import contextlib
+import importlib
+import io
+import json
+import logging
+import os
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+import torch
+from autogluon.multimodal import MultiModalPredictor
+from metrics_logic import compute_metrics_for_split, evaluate_all_transparency
+from packaging.version import Version
+
+logger = logging.getLogger(__name__)
+
+# ---------------------- small utilities ----------------------
+
+
+def load_user_hparams(hp_arg: Optional[str]) -> dict:
+ """Parse --hyperparameters (inline JSON or path to .json)."""
+ if not hp_arg:
+ return {}
+ try:
+ s = hp_arg.strip()
+ if s.startswith("{"):
+ return json.loads(s)
+ with open(s, "r") as f:
+ return json.load(f)
+ except Exception as e:
+ logger.warning(f"Could not parse --hyperparameters: {e}. Ignoring.")
+ return {}
+
+
+def deep_update(dst: dict, src: dict) -> dict:
+ """Recursive dict update (src overrides dst)."""
+ for k, v in (src or {}).items():
+ if isinstance(v, dict) and isinstance(dst.get(k), dict):
+ deep_update(dst[k], v)
+ else:
+ dst[k] = v
+ return dst
+
+
+@contextlib.contextmanager
+def suppress_stdout_stderr():
+ """Silence noisy prints from AG internals (fit_summary)."""
+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
+ yield
+
+
+def ag_evaluate_safely(predictor, df: Optional[pd.DataFrame], metrics: Optional[List[str]] = None) -> Dict[str, float]:
+ """
+ Call predictor.evaluate and normalize the output to a dict.
+ """
+ if df is None or len(df) == 0:
+ return {}
+ try:
+ res = predictor.evaluate(df, metrics=metrics)
+ except TypeError:
+ if metrics and len(metrics) == 1:
+ res = predictor.evaluate(df, metrics[0])
+ else:
+ res = predictor.evaluate(df)
+ if isinstance(res, (int, float, np.floating)):
+ name = (metrics[0] if metrics else "metric")
+ return {name: float(res)}
+ if isinstance(res, dict):
+ return {k: float(v) for k, v in res.items()}
+ return {"metric": float(res)}
+
+
+# ---------------------- hparams & training ----------------------
+def build_mm_hparams(args, df_train: pd.DataFrame, image_columns: Optional[List[str]]) -> dict:
+ """
+ Build hyperparameters for MultiModalPredictor.
+ Handles text checkpoints for torch<2.6 and merges user overrides.
+ """
+ inferred_text_cols = [
+ c for c in df_train.columns
+ if c != args.label_column
+ and str(df_train[c].dtype) == "object"
+ and df_train[c].notna().any()
+ ]
+ text_cols = inferred_text_cols
+
+ ag_version = None
+ try:
+ ag_mod = importlib.import_module("autogluon")
+ ag_ver = getattr(ag_mod, "__version__", None)
+ if ag_ver:
+ ag_version = Version(str(ag_ver))
+ except Exception:
+ ag_mod = None
+
+ def _log_missing_support(key: str) -> None:
+ logger.info(
+ "AutoGluon version %s does not expose '%s'; skipping override.",
+ ag_version or "unknown",
+ key,
+ )
+
+ hp = {}
+
+ # Setup environment
+ hp["env"] = {
+ "seed": int(args.random_seed)
+ }
+
+ # Set eval metric through model config
+ model_block = hp.setdefault("model", {})
+ if args.eval_metric:
+ model_block.setdefault("metric_learning", {})["metric"] = str(args.eval_metric)
+
+ if text_cols and Version(torch.__version__) < Version("2.6"):
+ safe_ckpt = "distilbert-base-uncased"
+ logger.warning(f"Forcing HF text checkpoint with safetensors: {safe_ckpt}")
+ hp["model.hf_text.checkpoint_name"] = safe_ckpt
+ hp.setdefault(
+ "model.names",
+ ["hf_text", "timm_image", "numerical_mlp", "categorical_mlp", "fusion_mlp"],
+ )
+
+ def _is_valid_hp_dict(d) -> bool:
+ if not isinstance(d, dict):
+ logger.warning("User-supplied hyperparameters must be a dict; received %s", type(d).__name__)
+ return False
+ return True
+
+ user_hp = args.hyperparameters if isinstance(args.hyperparameters, dict) else load_user_hparams(args.hyperparameters)
+ if user_hp and _is_valid_hp_dict(user_hp):
+ hp = deep_update(hp, user_hp)
+
+ # Map CLI knobs into AutoMM optimization hyperparameters when provided.
+ # We set multiple common key names (nested dicts and dotted flat keys) to
+ # maximize compatibility across AutoMM/AutoGluon versions.
+ try:
+ if any(getattr(args, param, None) is not None for param in ["epochs", "learning_rate", "batch_size"]):
+ if getattr(args, "epochs", None) is not None:
+ hp["optim.max_epochs"] = int(args.epochs)
+ hp["optim.epochs"] = int(args.epochs)
+ if getattr(args, "learning_rate", None) is not None:
+ hp["optim.learning_rate"] = float(args.learning_rate)
+ hp["optim.lr"] = float(args.learning_rate)
+ if getattr(args, "batch_size", None) is not None:
+ hp["optim.batch_size"] = int(args.batch_size)
+ hp["optim.per_device_train_batch_size"] = int(args.batch_size)
+
+ # Also set dotted flat keys for max compatibility (e.g., 'optimization.max_epochs')
+ if getattr(args, "epochs", None) is not None:
+ hp["optimization.max_epochs"] = int(args.epochs)
+ hp["optimization.epochs"] = int(args.epochs)
+ if getattr(args, "learning_rate", None) is not None:
+ hp["optimization.learning_rate"] = float(args.learning_rate)
+ hp["optimization.lr"] = float(args.learning_rate)
+ if getattr(args, "batch_size", None) is not None:
+ hp["optimization.batch_size"] = int(args.batch_size)
+ hp["optimization.per_device_train_batch_size"] = int(args.batch_size)
+ except Exception:
+ logger.warning("Failed to attach epochs/learning_rate/batch_size to mm_hparams; continuing without them.")
+
+ # Map backbone selections into mm_hparams if provided
+ try:
+ has_text_cols = bool(text_cols)
+ has_image_cols = False
+ model_names_cache: Optional[List[str]] = None
+ model_names_modified = False
+
+ def _dedupe_preserve(seq: List[str]) -> List[str]:
+ seen = set()
+ ordered = []
+ for item in seq:
+ if item in seen:
+ continue
+ seen.add(item)
+ ordered.append(item)
+ return ordered
+
+ def _get_model_names() -> List[str]:
+ nonlocal model_names_cache
+ if model_names_cache is not None:
+ return model_names_cache
+ names = model_block.get("names")
+ if isinstance(names, list):
+ model_names_cache = list(names)
+ else:
+ model_names_cache = []
+ if has_text_cols:
+ model_names_cache.append("hf_text")
+ if has_image_cols:
+ model_names_cache.append("timm_image")
+ model_names_cache.extend(["numerical_mlp", "categorical_mlp"])
+ model_names_cache.append("fusion_mlp")
+ return model_names_cache
+
+ def _set_model_names(new_names: List[str]) -> None:
+ nonlocal model_names_cache, model_names_modified
+ model_names_cache = new_names
+ model_names_modified = True
+
+ if has_text_cols and getattr(args, "backbone_text", None):
+ text_choice = str(args.backbone_text)
+ model_block.setdefault("hf_text", {})["checkpoint_name"] = text_choice
+ hp["model.hf_text.checkpoint_name"] = text_choice
+ if has_image_cols and getattr(args, "backbone_image", None):
+ image_choice = str(args.backbone_image)
+ model_block.setdefault("timm_image", {})["checkpoint_name"] = image_choice
+ hp["model.timm_image.checkpoint_name"] = image_choice
+ if model_names_modified and model_names_cache is not None:
+ model_block["names"] = model_names_cache
+ except Exception:
+ logger.warning("Failed to attach backbone selections to mm_hparams; continuing without them.")
+
+ if ag_version:
+ logger.info(f"Detected AutoGluon version: {ag_version}; applied robust hyperparameter mappings.")
+
+ return hp
+
+
+def train_predictor(
+ args,
+ df_train: pd.DataFrame,
+ df_val: pd.DataFrame,
+ image_columns: Optional[List[str]],
+ mm_hparams: dict,
+):
+ """
+ Train a MultiModalPredictor, honoring common knobs (presets, eval_metric, etc.).
+ """
+ logger.info("Starting AutoGluon MultiModal training...")
+ predictor = MultiModalPredictor(label=args.label_column, path=None)
+ column_types = {}
+
+ mm_fit_kwargs = dict(
+ train_data=df_train,
+ time_limit=args.time_limit,
+ seed=int(args.random_seed),
+ hyperparameters=mm_hparams,
+ )
+ if df_val is not None and not df_val.empty:
+ mm_fit_kwargs["tuning_data"] = df_val
+ if column_types:
+ mm_fit_kwargs["column_types"] = column_types
+
+ preset_mm = getattr(args, "presets", None)
+ if preset_mm is None:
+ preset_mm = getattr(args, "preset", None)
+ if preset_mm is not None:
+ mm_fit_kwargs["presets"] = preset_mm
+
+ predictor.fit(**mm_fit_kwargs)
+ return predictor
+
+
+# ---------------------- evaluation ----------------------
+def evaluate_predictor_all_splits(
+ predictor,
+ df_train: Optional[pd.DataFrame],
+ df_val: Optional[pd.DataFrame],
+ df_test: Optional[pd.DataFrame],
+ label_col: str,
+ problem_type: str,
+ eval_metric: Optional[str],
+ threshold_test: Optional[float],
+ df_test_external: Optional[pd.DataFrame] = None,
+) -> Tuple[Dict[str, Dict[str, float]], Dict[str, Dict[str, float]]]:
+ """
+ Returns (raw_metrics, ag_scores_by_split)
+ - raw_metrics: our transparent suite (threshold applied to Test/External Test only inside metrics_logic)
+ - ag_scores_by_split: AutoGluon's evaluate() per split for the chosen eval_metric (or default)
+ """
+ metrics_req = None if (eval_metric is None or str(eval_metric).lower() == "auto") else [eval_metric]
+ ag_by_split: Dict[str, Dict[str, float]] = {}
+
+ if df_train is not None and len(df_train):
+ ag_by_split["Train"] = ag_evaluate_safely(predictor, df_train, metrics=metrics_req)
+ if df_val is not None and len(df_val):
+ ag_by_split["Validation"] = ag_evaluate_safely(predictor, df_val, metrics=metrics_req)
+
+ df_test_effective = df_test_external if df_test_external is not None else df_test
+ if df_test_effective is not None and len(df_test_effective):
+ ag_by_split["Test"] = ag_evaluate_safely(predictor, df_test_effective, metrics=metrics_req)
+
+ # Transparent suite (threshold on Test handled inside metrics_logic)
+ _, raw_metrics = evaluate_all_transparency(
+ predictor=predictor,
+ train_df=df_train,
+ val_df=df_val,
+ test_df=df_test_effective,
+ target_col=label_col,
+ problem_type=problem_type,
+ threshold=threshold_test,
+ )
+
+ if df_test_external is not None and df_test_external is not df_test and len(df_test_external):
+ raw_metrics["Test (external)"] = compute_metrics_for_split(
+ predictor, df_test_external, label_col, problem_type, threshold=threshold_test
+ )
+ ag_by_split["Test (external)"] = ag_evaluate_safely(predictor, df_test_external, metrics=metrics_req)
+
+ return raw_metrics, ag_by_split
+
+
+def fit_summary_safely(predictor) -> Optional[dict]:
+ """Get fit summary without printing misleading one-liners."""
+ with suppress_stdout_stderr():
+ try:
+ return predictor.fit_summary()
+ except Exception:
+ return None
+
+
+# ---------------------- image helpers ----------------------
+_PLACEHOLDER_PATH = None
+
+
+def _create_placeholder() -> str:
+ global _PLACEHOLDER_PATH
+ if _PLACEHOLDER_PATH and os.path.exists(_PLACEHOLDER_PATH):
+ return _PLACEHOLDER_PATH
+
+ dir_ = Path(tempfile.mkdtemp(prefix="ag_placeholder_"))
+ file_ = dir_ / f"placeholder_{uuid.uuid4().hex}.png"
+
+ try:
+ from PIL import Image
+ Image.new("RGB", (64, 64), (180, 180, 180)).save(file_)
+ except Exception:
+ import matplotlib.pyplot as plt
+ import numpy as np
+ plt.imsave(file_, np.full((64, 64, 3), 180, dtype=np.uint8))
+ plt.close("all")
+
+ _PLACEHOLDER_PATH = str(file_)
+ logger.info(f"Placeholder image created: {file_}")
+ return _PLACEHOLDER_PATH
+
+
+def _is_valid_path(val) -> bool:
+ if pd.isna(val):
+ return False
+ s = str(val).strip()
+ return s and os.path.isfile(s)
+
+
+def handle_missing_images(
+ df: pd.DataFrame,
+ image_columns: List[str],
+ strategy: str = "false",
+) -> pd.DataFrame:
+ if not image_columns or df.empty:
+ return df
+
+ remove = str(strategy).lower() == "true"
+ masks = [~df[col].apply(_is_valid_path) for col in image_columns if col in df.columns]
+ if not masks:
+ return df
+
+ any_missing = pd.concat(masks, axis=1).any(axis=1)
+ n_missing = int(any_missing.sum())
+
+ if n_missing == 0:
+ return df
+
+ if remove:
+ result = df[~any_missing].reset_index(drop=True)
+ logger.info(f"Dropped {n_missing} rows with missing images → {len(result)} remain")
+ else:
+ placeholder = _create_placeholder()
+ result = df.copy()
+ for col in image_columns:
+ if col in result.columns:
+ result.loc[~result[col].apply(_is_valid_path), col] = placeholder
+ logger.info(f"Filled {n_missing} missing images with placeholder")
+
+ return result
+
+
+# ---------------------- AutoGluon config helpers ----------------------
+def autogluon_hyperparameters(
+ threshold,
+ time_limit,
+ random_seed,
+ epochs,
+ learning_rate,
+ batch_size,
+ backbone_image,
+ backbone_text,
+ preset,
+ eval_metric,
+ hyperparameters,
+):
+ """
+ Build a MultiModalPredictor configuration (fit kwargs + hyperparameters) from CLI inputs.
+ The returned dict separates what should be passed to predictor.fit (under ``fit``)
+ from the model/optimization configuration (under ``hyperparameters``). Threshold is
+ preserved for downstream evaluation but not passed into AutoGluon directly.
+ """
+
+ def _prune_empty(d: dict) -> dict:
+ cleaned = {}
+ for k, v in (d or {}).items():
+ if isinstance(v, dict):
+ nested = _prune_empty(v)
+ if nested:
+ cleaned[k] = nested
+ elif v is not None:
+ cleaned[k] = v
+ return cleaned
+
+ # Base hyperparameters following the structure described in the AutoGluon
+ # customization guide (env / optimization / model).
+ env_cfg = {}
+ if random_seed is not None:
+ env_cfg["seed"] = int(random_seed)
+ if batch_size is not None:
+ env_cfg["per_gpu_batch_size"] = int(batch_size)
+
+ optim_cfg = {}
+ if epochs is not None:
+ optim_cfg["max_epochs"] = int(epochs)
+ if learning_rate is not None:
+ optim_cfg["learning_rate"] = float(learning_rate)
+ if batch_size is not None:
+ bs = int(batch_size)
+ optim_cfg["per_device_train_batch_size"] = bs
+ optim_cfg["train_batch_size"] = bs
+
+ model_cfg = {}
+ if eval_metric:
+ model_cfg.setdefault("metric_learning", {})["metric"] = str(eval_metric)
+ if backbone_image:
+ model_cfg.setdefault("timm_image", {})["checkpoint_name"] = str(backbone_image)
+ if backbone_text:
+ model_cfg.setdefault("hf_text", {})["checkpoint_name"] = str(backbone_text)
+
+ hp = {
+ "env": env_cfg,
+ "optimization": optim_cfg,
+ "model": model_cfg,
+ }
+
+ # Also expose the most common dotted aliases for robustness across AG versions.
+ if epochs is not None:
+ hp["optimization.max_epochs"] = int(epochs)
+ hp["optim.max_epochs"] = int(epochs)
+ if learning_rate is not None:
+ lr_val = float(learning_rate)
+ hp["optimization.learning_rate"] = lr_val
+ hp["optimization.lr"] = lr_val
+ hp["optim.learning_rate"] = lr_val
+ hp["optim.lr"] = lr_val
+ if batch_size is not None:
+ bs_val = int(batch_size)
+ hp["optimization.per_device_train_batch_size"] = bs_val
+ hp["optimization.batch_size"] = bs_val
+ hp["optim.per_device_train_batch_size"] = bs_val
+ hp["optim.batch_size"] = bs_val
+ hp["env.per_gpu_batch_size"] = bs_val
+ if backbone_image:
+ hp["model.timm_image.checkpoint_name"] = str(backbone_image)
+ if backbone_text:
+ hp["model.hf_text.checkpoint_name"] = str(backbone_text)
+
+ # Merge user-provided hyperparameters (inline JSON or path) last so they win.
+ if isinstance(hyperparameters, dict):
+ user_hp = hyperparameters
+ else:
+ user_hp = load_user_hparams(hyperparameters)
+ hp = deep_update(hp, user_hp)
+ hp = _prune_empty(hp)
+
+ fit_cfg = {}
+ if time_limit is not None:
+ fit_cfg["time_limit"] = time_limit
+ if random_seed is not None:
+ fit_cfg["seed"] = int(random_seed)
+ if preset:
+ fit_cfg["presets"] = preset
+
+ config = {
+ "fit": fit_cfg,
+ "hyperparameters": hp,
+ }
+ if threshold is not None:
+ config["threshold"] = float(threshold)
+
+ return config
+
+
+def run_autogluon_experiment(
+ train_dataset: pd.DataFrame,
+ test_dataset: Optional[pd.DataFrame],
+ target_column: str,
+ image_columns: Optional[List[str]],
+ ag_config: dict,
+):
+ """
+ Launch an AutoGluon MultiModal training run using the config from
+ autogluon_hyperparameters(). Returns (predictor, context dict) so callers
+ can evaluate downstream with the chosen threshold.
+ """
+ if ag_config is None:
+ raise ValueError("ag_config is required to launch AutoGluon training.")
+
+ hyperparameters = ag_config.get("hyperparameters") or {}
+ fit_cfg = dict(ag_config.get("fit") or {})
+ threshold = ag_config.get("threshold")
+
+ if "split" not in train_dataset.columns:
+ raise ValueError("train_dataset must contain a 'split' column. Did you call split_dataset?")
+
+ df_train = train_dataset[train_dataset["split"] == "train"].copy()
+ df_val = train_dataset[train_dataset["split"].isin(["val", "validation"])].copy()
+ df_test_internal = train_dataset[train_dataset["split"] == "test"].copy()
+
+ predictor = MultiModalPredictor(label=target_column, path=None)
+ column_types = {c: "image_path" for c in (image_columns or [])}
+
+ fit_kwargs = {
+ "train_data": df_train,
+ "hyperparameters": hyperparameters,
+ }
+ fit_kwargs.update(fit_cfg)
+ if not df_val.empty:
+ fit_kwargs.setdefault("tuning_data", df_val)
+ if column_types:
+ fit_kwargs.setdefault("column_types", column_types)
+
+ logger.info(
+ "Fitting AutoGluon with %d train / %d val rows (internal test rows: %d, external test provided: %s)",
+ len(df_train),
+ len(df_val),
+ len(df_test_internal),
+ (test_dataset is not None and not test_dataset.empty),
+ )
+ predictor.fit(**fit_kwargs)
+
+ return predictor, {
+ "train": df_train,
+ "val": df_val,
+ "test_internal": df_test_internal,
+ "test_external": test_dataset,
+ "threshold": threshold,
+ }
diff -r 000000000000 -r 375c36923da1 utils.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/utils.py Tue Dec 09 23:49:47 2025 +0000
@@ -0,0 +1,168 @@
+import json
+import logging
+import os
+import random
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import List, Optional
+
+import numpy as np
+import pandas as pd
+import torch
+
+LOG = logging.getLogger(__name__)
+
+
+def str2bool(val) -> bool:
+ """Parse common truthy strings to bool."""
+ return str(val).strip().lower() in ("1", "true", "yes", "y")
+
+
+def load_user_hparams(hp_arg: Optional[str]) -> dict:
+ """Parse --hyperparameters (inline JSON or path to .json)."""
+ if not hp_arg:
+ return {}
+ try:
+ s = hp_arg.strip()
+ if s.startswith("{"):
+ return json.loads(s)
+ with open(s, "r") as f:
+ return json.load(f)
+ except Exception as e:
+ LOG.warning(f"Could not parse --hyperparameters: {e}. Ignoring.")
+ return {}
+
+
+def set_seeds(seed: int = 42):
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed_all(seed)
+
+
+def ensure_local_tmp():
+ os.makedirs("/tmp", exist_ok=True)
+
+
+def enable_tensor_cores_if_available():
+ if torch.cuda.is_available():
+ torch.set_float32_matmul_precision("high")
+
+
+def enable_deterministic_mode(seed: Optional[int] = None):
+ """
+ Force deterministic algorithms where possible to reduce run-to-run variance.
+ """
+ if seed is not None:
+ set_seeds(seed)
+ os.environ.setdefault("PYTHONHASHSEED", str(int(seed)))
+ # cuBLAS determinism
+ os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
+ try:
+ torch.use_deterministic_algorithms(True)
+ except Exception as e:
+ LOG.warning(f"Could not enable torch deterministic algorithms: {e}")
+ try:
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ except Exception as e:
+ LOG.warning(f"Could not enforce deterministic cuDNN settings: {e}")
+ try:
+ torch.backends.cuda.matmul.allow_tf32 = False
+ except Exception:
+ pass
+ try:
+ torch.backends.cudnn.allow_tf32 = False
+ except Exception:
+ pass
+
+
+def load_file(path: str) -> pd.DataFrame:
+ if not path:
+ return None
+ path = Path(path)
+ if not path.exists():
+ raise FileNotFoundError(f"Dataset not found: {path}")
+ return pd.read_csv(path, sep=None, engine="python")
+
+
+def prepare_image_search_dirs(args) -> Optional[Path]:
+ if not args.images_zip:
+ return None
+
+ root = Path(tempfile.mkdtemp(prefix="autogluon_images_"))
+ LOG.info(f"Extracting {len(args.images_zip)} image ZIP(s) to {root}")
+
+ for zip_path in args.images_zip:
+ path = Path(zip_path)
+ if not path.exists():
+ raise FileNotFoundError(f"Image ZIP not found: {zip_path}")
+ with zipfile.ZipFile(path, 'r') as z:
+ z.extractall(root)
+ LOG.info(f"Extracted {path.name}")
+
+ return root
+
+
+def absolute_path_expander(df: pd.DataFrame, extracted_root: Optional[Path], image_columns: Optional[List[str]]) -> List[str]:
+ """
+ Resolve image paths to absolute paths. If no image_columns are provided,
+ infers candidate columns whose values resolve to existing files (checking
+ absolute paths first, then paths relative to the extracted_root).
+ """
+ if df is None or df.empty:
+ return []
+
+ image_columns = [c for c in (image_columns or []) if c in df.columns]
+
+ def resolve(p):
+ if pd.isna(p):
+ return None
+ orig = Path(str(p).strip())
+ candidates = []
+ if orig.is_absolute():
+ candidates.append(orig)
+ if extracted_root is not None:
+ candidates.extend([extracted_root / orig, extracted_root / orig.name])
+ for cand in candidates:
+ if cand.exists():
+ return str(cand.resolve())
+ return None
+
+ # Infer image columns if none were provided
+ if not image_columns:
+ obj_cols = [c for c in df.columns if str(df[c].dtype) == "object"]
+ inferred = []
+ for col in obj_cols:
+ sample = df[col].dropna().head(50)
+ if sample.empty:
+ continue
+ resolved_sample = sample.apply(resolve)
+ if resolved_sample.notna().any():
+ inferred.append(col)
+ image_columns = inferred
+ if image_columns:
+ LOG.info(f"Inferred image columns: {image_columns}")
+
+ for col in image_columns:
+ df[col] = df[col].apply(resolve)
+
+ return image_columns
+
+
+def verify_outputs(paths):
+ ok = True
+ for p, desc in paths:
+ if os.path.exists(p):
+ size = os.path.getsize(p)
+ LOG.info(f"✓ Output {desc}: {p} ({size:,} bytes)")
+ os.chmod(p, 0o644)
+ else:
+ LOG.error(f"✗ Output {desc} MISSING: {p}")
+ ok = False
+ if not ok:
+ LOG.error("Some outputs are missing!")
+ sys.exit(1)
+
+
+
+ Model Summary & Config
+ Train/Validation Summary
+ Test Summary
+ Feature Importance
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Model Performance Summary
+
+
+
+
+ | Metric | Train | Validation | Test |
|---|---|---|---|
| accuracy | 0.9234 | 0.8912 | 0.8856 |
| f1 | 0.9201 | 0.8876 | 0.8823 |
| precision | 0.9156 | 0.8845 | 0.8798 |
| recall | 0.9245 | 0.8907 | 0.8849 |
| roc_auc | 0.9789 | 0.9543 | 0.9512 |
| log_loss | 0.2134 | 0.2876 | 0.3012 |
Run Configuration
+
+
+
+
+ | Key | Value |
|---|---|
| Predictor type | MultiModalPredictor |
| Framework | AutoGluon Multimodal |
| Model architecture | timm_image=resnet50, hf_text=bert-base-uncased |
| Modalities & Inputs | Images + Tabular |
| Label column | target |
| Image columns | image_path |
| Tabular columns | 15 |
| Presets | medium_quality |
| Eval metric | accuracy |
| Decision threshold calibration | enabled |
| Decision threshold (Test only) | 0.500 |
| Seed | 42 |
| time limit(s) | 3600 |
Class Balance (Train Full)
+
+
+
+
+ Class Balance (Train Full)
+| Class | Count | Percent |
|---|---|---|
| 0 | 1245 | 45.23% |
| 1 | 1508 | 54.77% |
+
+
+
+ Train/Validation Performance Summary
+ +
+
+
+
+
+ | Metric | Train | Validation |
|---|---|---|
| accuracy | 0.9234 | 0.8912 |
| f1 | 0.9201 | 0.8876 |
| precision | 0.9156 | 0.8845 |
| recall | 0.9245 | 0.8907 |
| roc_auc | 0.9789 | 0.9543 |
| log_loss | 0.2134 | 0.2876 |
Learning Curves — Label Accuracy
+
+
+
+
+ Learning Curves — Label Loss
+
+
+
+
+
+
+
+
+
+
+
+ Test Performance Summary
+| Metric | +Test | +
|---|---|
| accuracy | 0.8856 |
| f1 | 0.8823 |
| precision | 0.8798 |
| recall | 0.8849 |
| roc_auc | 0.9512 |
| log_loss | 0.3012 |
| specificity (TNR) | 0.8765 |
| sensitivity (Sensitivity/TPR) | 0.8923 |
Confusion Matrix
+
+
+
+
+ Per-Class Metrics
+
+
+
+
+ ROC Curve
+
+
+
+
+ Precision–Recall Curve
+
+
+
+
+ Threshold Plot
+
+
+
+
+
+
+
+
+
+
+
+
+ Feature Importance
+
+
+ Permutation importance is not supported for MultiModalPredictor in this tool. For tabular-only runs, this section shows permutation importance.
+Modalities & Inputs
+
+
+ Modalities & Inputs
+This run used MultiModalPredictor (images + tabular).
+Label column: target
+Image column: image_path
+Tabular columns: 15
+
+ -
+
- feature_1 +
- feature_2 +
- feature_3 +
- feature_4 +
- feature_5 +
- feature_6 +
- feature_7 +
- feature_8 +
- feature_9 +
- feature_10 +
- feature_11 +
- feature_12 +
- feature_13 +
- feature_14 +
- feature_15 +