comparison training_pipeline.py @ 8:a48e750cfd25 draft default tip

planemo upload for repository https://github.com/goeckslab/gleam.git commit c8a7fef0c54c269afd6c6bdf035af1a7574d11cb

7:ed2fefc8d892

    label_col: str,
    problem_type: str,
    eval_metric: Optional[str],
    threshold_test: Optional[float],
    df_test_external: Optional[pd.DataFrame] = None,
) -> Tuple[Dict[str, Dict[str, float]], Dict[str, Dict[str, float]]]:
    """
    Returns (raw_metrics, ag_scores_by_split)
      - raw_metrics: our transparent suite (threshold applied to Test/External Test only inside metrics_logic)
      - ag_scores_by_split: AutoGluon's evaluate() per split for the chosen eval_metric (or default)
    """

    df_test_effective = df_test_external if df_test_external is not None else df_test
    if df_test_effective is not None and len(df_test_effective):
        ag_by_split["Test"] = ag_evaluate_safely(predictor, df_test_effective, metrics=metrics_req)

    # Transparent suite (threshold on Test handled inside metrics_logic)
    _, raw_metrics = evaluate_all_transparency(
        predictor=predictor,
        train_df=df_train,
        val_df=df_val,
        test_df=df_test_effective,
        target_col=label_col,
        problem_type=problem_type,
        threshold=threshold_test,
    )

    if df_test_external is not None and df_test_external is not df_test and len(df_test_external):
        raw_metrics["Test (external)"] = compute_metrics_for_split(
            predictor, df_test_external, label_col, problem_type, threshold=threshold_test
        )
        ag_by_split["Test (external)"] = ag_evaluate_safely(predictor, df_test_external, metrics=metrics_req)

    return raw_metrics, ag_by_split


def fit_summary_safely(predictor) -> Optional[dict]:
    """Get fit summary without printing misleading one-liners."""
    with suppress_stdout_stderr():

8:a48e750cfd25

    label_col: str,
    problem_type: str,
    eval_metric: Optional[str],
    threshold_test: Optional[float],
    df_test_external: Optional[pd.DataFrame] = None,
) -> Tuple[Dict[str, Dict[str, float]], Dict[str, Dict[str, float]], Dict[str, dict]]:
    """
    Returns (raw_metrics, ag_scores_by_split)
      - raw_metrics: our transparent suite (threshold applied to Test/External Test only inside metrics_logic)
      - ag_scores_by_split: AutoGluon's evaluate() per split for the chosen eval_metric (or default)
    """

    df_test_effective = df_test_external if df_test_external is not None else df_test
    if df_test_effective is not None and len(df_test_effective):
        ag_by_split["Test"] = ag_evaluate_safely(predictor, df_test_effective, metrics=metrics_req)

    # Transparent suite (threshold on Test handled inside metrics_logic)
    _, raw_metrics, roc_curves = evaluate_all_transparency(
        predictor=predictor,
        train_df=df_train,
        val_df=df_val,
        test_df=df_test_effective,
        target_col=label_col,
        problem_type=problem_type,
        threshold=threshold_test,
    )

    if df_test_external is not None and df_test_external is not df_test and len(df_test_external):
        ext_metrics, ext_curve = compute_metrics_for_split(
            predictor,
            df_test_external,
            label_col,
            problem_type,
            threshold=threshold_test,
            return_curve=True,
        )
        raw_metrics["Test (external)"] = ext_metrics
        if ext_curve:
            roc_curves["Test (external)"] = ext_curve
        ag_by_split["Test (external)"] = ag_evaluate_safely(predictor, df_test_external, metrics=metrics_req)

    return raw_metrics, ag_by_split, roc_curves


def fit_summary_safely(predictor) -> Optional[dict]:
    """Get fit summary without printing misleading one-liners."""
    with suppress_stdout_stderr():