-
-
-"""
- modal_css = """
-
-"""
- modal_js = """
-
-"""
- return modal_css + modal_html + modal_js
-
-
def format_config_table_html(
config: dict,
split_info: Optional[str] = None,
training_progress: dict = None,
) -> str:
display_keys = [
+ "task_type",
"model_name",
"epochs",
"batch_size",
@@ -287,6 +68,8 @@
for key in display_keys:
val = config.get(key, "N/A")
+ if key == "task_type":
+ val = val.title() if isinstance(val, str) else val
if key == "batch_size":
if val is not None:
val = int(val)
@@ -348,6 +131,18 @@
f""
)
+ aug_cfg = config.get("augmentation")
+ if aug_cfg:
+ types = [str(a.get("type", "")) for a in aug_cfg]
+ aug_val = ", ".join(types)
+ rows.append(
+ ""
"Model trained using Ludwig.
"
"If want to learn more about Ludwig default settings,"
- "please check the their "
+ "please check their "
"website(ludwig.ai)."
"
" ) @@ -380,6 +175,8 @@ def detect_output_type(test_stats): """Detects if the output type is 'binary' or 'category' based on test statistics.""" label_stats = test_stats.get("label", {}) + if "mean_squared_error" in label_stats: + return "regression" per_class = label_stats.get("per_class_stats", {}) if len(per_class) == 2: return "binary" @@ -420,6 +217,24 @@ "specificity": get_last_value(label_stats, "specificity"), "roc_auc": get_last_value(label_stats, "roc_auc"), } + elif output_type == "regression": + metrics[split] = { + "loss": get_last_value(label_stats, "loss"), + "mean_absolute_error": get_last_value( + label_stats, "mean_absolute_error" + ), + "mean_absolute_percentage_error": get_last_value( + label_stats, "mean_absolute_percentage_error" + ), + "mean_squared_error": get_last_value(label_stats, "mean_squared_error"), + "root_mean_squared_error": get_last_value( + label_stats, "root_mean_squared_error" + ), + "root_mean_squared_percentage_error": get_last_value( + label_stats, "root_mean_squared_percentage_error" + ), + "r2": get_last_value(label_stats, "r2"), + } else: metrics[split] = { "accuracy": get_last_value(label_stats, "accuracy"), @@ -565,7 +380,9 @@ return html -def format_test_merged_stats_table_html(test_metrics: Dict[str, Optional[float]]) -> str: +def format_test_merged_stats_table_html( + test_metrics: Dict[str, Optional[float]], +) -> str: """Formats an HTML table for test metrics.""" rows = [] for key in sorted(test_metrics.keys()): @@ -598,63 +415,6 @@ return html -def build_tabbed_html(metrics_html: str, train_val_html: str, test_html: str) -> str: - return f""" - -
-
- Config & Results Summary
- Train/Validation Results
- Test Results
-
- {metrics_html}
-
-
- {train_val_html}
-
-
- {test_html}
-
-
-"""
-
-
def split_data_0_2(
df: pd.DataFrame,
split_column: str,
@@ -727,16 +487,15 @@
) -> None:
...
- def generate_plots(
- self,
- output_dir: Path
- ) -> None:
+ def generate_plots(self, output_dir: Path) -> None:
...
def generate_html_report(
self,
title: str,
- output_dir: str
+ output_dir: str,
+ config: Dict[str, Any],
+ split_info: str,
) -> Path:
...
@@ -749,23 +508,21 @@
config_params: Dict[str, Any],
split_config: Dict[str, Any],
) -> str:
- """Build and serialize the Ludwig YAML configuration."""
logger.info("LudwigDirectBackend: Preparing YAML configuration.")
model_name = config_params.get("model_name", "resnet18")
use_pretrained = config_params.get("use_pretrained", False)
fine_tune = config_params.get("fine_tune", False)
+ if use_pretrained:
+ trainable = bool(fine_tune)
+ else:
+ trainable = True
epochs = config_params.get("epochs", 10)
batch_size = config_params.get("batch_size")
num_processes = config_params.get("preprocessing_num_processes", 1)
early_stop = config_params.get("early_stop", None)
learning_rate = config_params.get("learning_rate")
learning_rate = "auto" if learning_rate is None else float(learning_rate)
- trainable = fine_tune or (not use_pretrained)
- if not use_pretrained and not trainable:
- logger.warning("trainable=False; use_pretrained=False is ignored.")
- logger.warning("Setting trainable=True to train the model from scratch.")
- trainable = True
raw_encoder = MODEL_ENCODER_TEMPLATES.get(model_name, model_name)
if isinstance(raw_encoder, dict):
encoder_config = {
@@ -779,39 +536,68 @@
batch_size_cfg = batch_size or "auto"
label_column_path = config_params.get("label_column_data_path")
+ label_series = None
if label_column_path is not None and Path(label_column_path).exists():
try:
label_series = pd.read_csv(label_column_path)[LABEL_COLUMN_NAME]
- num_unique_labels = label_series.nunique()
except Exception as e:
- logger.warning(
- f"Could not determine label cardinality, defaulting to 'binary': {e}"
- )
- num_unique_labels = 2
+ logger.warning(f"Could not read label column for task detection: {e}")
+
+ if (
+ label_series is not None
+ and ptypes.is_numeric_dtype(label_series.dtype)
+ and label_series.nunique() > 10
+ ):
+ task_type = "regression"
else:
- logger.warning(
- "label_column_data_path not provided, defaulting to 'binary'"
+ task_type = "classification"
+
+ config_params["task_type"] = task_type
+
+ image_feat: Dict[str, Any] = {
+ "name": IMAGE_PATH_COLUMN_NAME,
+ "type": "image",
+ "encoder": encoder_config,
+ }
+ if config_params.get("augmentation") is not None:
+ image_feat["augmentation"] = config_params["augmentation"]
+
+ if task_type == "regression":
+ output_feat = {
+ "name": LABEL_COLUMN_NAME,
+ "type": "number",
+ "decoder": {"type": "regressor"},
+ "loss": {"type": "mean_squared_error"},
+ "evaluation": {
+ "metrics": [
+ "mean_squared_error",
+ "mean_absolute_error",
+ "r2",
+ ]
+ },
+ }
+ val_metric = config_params.get("validation_metric", "mean_squared_error")
+
+ else:
+ num_unique_labels = (
+ label_series.nunique() if label_series is not None else 2
)
- num_unique_labels = 2
-
- output_type = "binary" if num_unique_labels == 2 else "category"
+ output_type = "binary" if num_unique_labels == 2 else "category"
+ output_feat = {"name": LABEL_COLUMN_NAME, "type": output_type}
+ val_metric = None
conf: Dict[str, Any] = {
"model_type": "ecd",
- "input_features": [
- {
- "name": IMAGE_PATH_COLUMN_NAME,
- "type": "image",
- "encoder": encoder_config,
- }
- ],
- "output_features": [{"name": LABEL_COLUMN_NAME, "type": output_type}],
+ "input_features": [image_feat],
+ "output_features": [output_feat],
"combiner": {"type": "concat"},
"trainer": {
"epochs": epochs,
"early_stop": early_stop,
"batch_size": batch_size_cfg,
"learning_rate": learning_rate,
+ # only set validation_metric for regression
+ **({"validation_metric": val_metric} if val_metric else {}),
},
"preprocessing": {
"split": split_config,
@@ -876,7 +662,7 @@
)
raise
- def get_training_process(self, output_dir) -> float:
+ def get_training_process(self, output_dir) -> Optional[Dict[str, Any]]:
"""Retrieve the learning rate used in the most recent Ludwig run."""
output_dir = Path(output_dir)
exp_dirs = sorted(
@@ -1000,11 +786,12 @@
viz_registry = get_visualizations_registry()
for viz_name, viz_func in viz_registry.items():
- viz_dir_plot = None
if viz_name in train_plots:
viz_dir_plot = train_viz
elif viz_name in test_plots:
viz_dir_plot = test_viz
+ else:
+ continue
try:
viz_func(
@@ -1040,6 +827,7 @@
report_name = title.lower().replace(" ", "_") + "_report.html"
report_path = cwd / report_name
output_dir = Path(output_dir)
+ output_type = None
exp_dirs = sorted(
output_dir.glob("experiment_run*"),
@@ -1059,7 +847,6 @@
metrics_html = ""
train_val_metrics_html = ""
test_metrics_html = ""
-
try:
train_stats_path = exp_dir / "training_statistics.json"
test_stats_path = exp_dir / TEST_STATISTICS_FILE_NAME
@@ -1069,18 +856,14 @@
with open(test_stats_path) as f:
test_stats = json.load(f)
output_type = detect_output_type(test_stats)
- all_metrics = extract_metrics_from_json(
- train_stats,
- test_stats,
- output_type,
- )
metrics_html = format_stats_table_html(train_stats, test_stats)
train_val_metrics_html = format_train_val_stats_table_html(
- train_stats,
- test_stats,
+ train_stats, test_stats
)
test_metrics_html = format_test_merged_stats_table_html(
- all_metrics["test"],
+ extract_metrics_from_json(train_stats, test_stats, output_type)[
+ "test"
+ ]
)
except Exception as e:
logger.warning(
@@ -1090,11 +873,15 @@
config_html = ""
training_progress = self.get_training_process(output_dir)
try:
- config_html = format_config_table_html(config, split_info, training_progress)
+ config_html = format_config_table_html(
+ config, split_info, training_progress
+ )
except Exception as e:
logger.warning(f"Could not load config for HTML report: {e}")
- def render_img_section(title: str, dir_path: Path, output_type: str = None) -> str:
+ def render_img_section(
+ title: str, dir_path: Path, output_type: str = None
+ ) -> str:
if not dir_path.exists():
return f"{title}
Directory not found.
" @@ -1141,11 +928,7 @@ img_names[fname] for fname in display_order if fname in img_names ] remaining = sorted( - [ - img - for img in img_names.values() - if img.name not in display_order - ] + [img for img in img_names.values() if img.name not in display_order] ) imgs = ordered_imgs + remaining @@ -1173,46 +956,61 @@ section_html += "" return section_html - button_html = """ - -- - """ - tab1_content = button_html + config_html + metrics_html - tab2_content = ( - button_html - + train_val_metrics_html - + render_img_section("Training & Validation Visualizations", train_viz_dir) + tab1_content = config_html + metrics_html + + tab2_content = train_val_metrics_html + render_img_section( + "Training & Validation Visualizations", train_viz_dir ) + + # --- Predictions vs Ground Truth table --- + preds_section = "" + parquet_path = exp_dir / PREDICTIONS_PARQUET_FILE_NAME + if parquet_path.exists(): + try: + # 1) load predictions from Parquet + df_preds = pd.read_parquet(parquet_path).reset_index(drop=True) + # assume the column containing your model's prediction is named "prediction" + # or contains that substring: + pred_col = next( + (c for c in df_preds.columns if "prediction" in c.lower()), + None, + ) + if pred_col is None: + raise ValueError("No prediction column found in Parquet output") + df_pred = df_preds[[pred_col]].rename(columns={pred_col: "prediction"}) + + # 2) load ground truth for the test split from prepared CSV + df_all = pd.read_csv(config["label_column_data_path"]) + df_gt = df_all[df_all[SPLIT_COLUMN_NAME] == 2][ + LABEL_COLUMN_NAME + ].reset_index(drop=True) + + # 3) concatenate side‐by‐side + df_table = pd.concat([df_gt, df_pred], axis=1) + df_table.columns = [LABEL_COLUMN_NAME, "prediction"] + + # 4) render as HTML + preds_html = df_table.to_html(index=False, classes="predictions-table") + preds_section = ( + "
Predictions vs. Ground Truth
" + ""
+ + preds_html
+ + "
"
+ )
+ except Exception as e:
+ logger.warning(f"Could not build Predictions vs GT table: {e}")
+ # Test tab = Metrics + Preds table + Visualizations
+
tab3_content = (
- button_html
- + test_metrics_html
+ test_metrics_html
+ + preds_section
+ render_img_section("Test Visualizations", test_viz_dir, output_type)
)
+ # assemble the tabs and help modal
tabbed_html = build_tabbed_html(tab1_content, tab2_content, tab3_content)
modal_html = get_metrics_help_modal()
- html += tabbed_html + modal_html
- html += get_html_closing()
+ html += tabbed_html + modal_html + get_html_closing()
try:
with open(report_path, "w") as f:
@@ -1263,7 +1061,7 @@
logger.error("Error extracting zip file", exc_info=True)
raise
- def _prepare_data(self) -> Tuple[Path, Dict[str, Any]]:
+ def _prepare_data(self) -> Tuple[Path, Dict[str, Any], str]:
"""Load CSV, update image paths, handle splits, and write prepared CSV."""
if not self.temp_dir or not self.image_extract_dir:
raise RuntimeError("Temp dirs not initialized before data prep.")
@@ -1302,8 +1100,9 @@
f"for train/val/test."
)
- final_csv = TEMP_CSV_FILENAME
+ final_csv = self.temp_dir / TEMP_CSV_FILENAME
try:
+
df.to_csv(final_csv, index=False)
logger.info(f"Saved prepared data to {final_csv}")
except Exception:
@@ -1312,7 +1111,9 @@
return final_csv, split_config, split_info
- def _process_fixed_split(self, df: pd.DataFrame) -> Dict[str, Any]:
+ def _process_fixed_split(
+ self, df: pd.DataFrame
+ ) -> Tuple[pd.DataFrame, Dict[str, Any], str]:
"""Process a fixed split column (0=train,1=val,2=test)."""
logger.info(f"Fixed split column '{SPLIT_COLUMN_NAME}' detected.")
try:
@@ -1384,6 +1185,7 @@
"random_seed": self.args.random_seed,
"early_stop": self.args.early_stop,
"label_column_data_path": csv_path,
+ "augmentation": self.args.augmentation,
}
yaml_str = self.backend.prepare_config(backend_args, split_cfg)
@@ -1422,6 +1224,29 @@
return None
+def aug_parse(aug_string: str):
+ """
+ Parse comma-separated augmentation keys into Ludwig augmentation dicts.
+ Raises ValueError on unknown key.
+ """
+ mapping = {
+ "random_horizontal_flip": {"type": "random_horizontal_flip"},
+ "random_vertical_flip": {"type": "random_vertical_flip"},
+ "random_rotate": {"type": "random_rotate", "degree": 10},
+ "random_blur": {"type": "random_blur", "kernel_size": 3},
+ "random_brightness": {"type": "random_brightness", "min": 0.5, "max": 2.0},
+ "random_contrast": {"type": "random_contrast", "min": 0.5, "max": 2.0},
+ }
+ aug_list = []
+ for tok in aug_string.split(","):
+ key = tok.strip()
+ if key not in mapping:
+ valid = ", ".join(mapping.keys())
+ raise ValueError(f"Unknown augmentation '{key}'. Valid choices: {valid}")
+ aug_list.append(mapping[key])
+ return aug_list
+
+
class SplitProbAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
train, val, test = values
@@ -1508,7 +1333,10 @@
metavar=("train", "val", "test"),
action=SplitProbAction,
default=[0.7, 0.1, 0.2],
- help="Random split proportions (e.g., 0.7 0.1 0.2). Only used if no split column.",
+ help=(
+ "Random split proportions (e.g., 0.7 0.1 0.2)."
+ "Only used if no split column."
+ ),
)
parser.add_argument(
"--random-seed",
@@ -1522,6 +1350,17 @@
default=None,
help="Learning rate. If not provided, Ludwig will auto-select it.",
)
+ parser.add_argument(
+ "--augmentation",
+ type=str,
+ default=None,
+ help=(
+ "Comma-separated list (in order) of any of: "
+ "random_horizontal_flip, random_vertical_flip, random_rotate, "
+ "random_blur, random_brightness, random_contrast. "
+ "E.g. --augmentation random_horizontal_flip,random_rotate"
+ ),
+ )
args = parser.parse_args()
@@ -1531,6 +1370,12 @@
parser.error(f"CSV not found: {args.csv_file}")
if not args.image_zip.is_file():
parser.error(f"ZIP not found: {args.image_zip}")
+ if args.augmentation is not None:
+ try:
+ augmentation_setup = aug_parse(args.augmentation)
+ setattr(args, "augmentation", augmentation_setup)
+ except ValueError as e:
+ parser.error(str(e))
backend_instance = LudwigDirectBackend()
orchestrator = WorkflowOrchestrator(args, backend_instance)
diff -r 39202fe5cf97 -r 186424a7eca7 test-data/age_regression.zip
Binary file test-data/age_regression.zip has changed
diff -r 39202fe5cf97 -r 186424a7eca7 test-data/expected_regression.html
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/expected_regression.html Thu Jul 03 20:43:24 2025 +0000
@@ -0,0 +1,276 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff -r 39202fe5cf97 -r 186424a7eca7 test-data/utkface_labels.csv
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/utkface_labels.csv Thu Jul 03 20:43:24 2025 +0000
@@ -0,0 +1,13 @@
+image_path,label,split
+1_0_4_20161221195047839.jpg.chip.jpg,1,0
+4_0_0_20170110213253207.jpg.chip.jpg,4,0
+18_1_0_20170109212647587.jpg.chip.jpg,18,0
+24_1_2_20170116164628750.jpg.chip.jpg,24,0
+26_1_0_20170116164911648.jpg.chip.jpg,26,0
+28_1_0_20170109141748286.jpg.chip.jpg,28,0
+31_0_4_20170120133240958.jpg.chip.jpg,31,0
+35_0_0_20170104202556995.jpg.chip.jpg,35,0
+42_0_2_20170117130543345.jpg.chip.jpg,42,1
+52_0_1_20170117175021585.jpg.chip.jpg,52,1
+62_1_3_20170109132000815.jpg.chip.jpg,62,2
+116_1_2_20170112220255503.jpg.chip.jpg,116,2
diff -r 39202fe5cf97 -r 186424a7eca7 utils.py
--- a/utils.py Wed Jul 02 18:59:10 2025 +0000
+++ b/utils.py Thu Jul 03 20:43:24 2025 +0000
@@ -155,3 +155,199 @@
if isinstance(json_data, str):
json_data = json.loads(json_data)
return json_to_nested_html_table(json_data)
+
+
+def build_tabbed_html(metrics_html: str, train_val_html: str, test_html: str) -> str:
+ return f"""
+
+
+Image Classification Results
+ + +
+
+
+Config & Results Summary
+ Train/Validation Results
+ Test Results
+
+
+
+
+
+Training Setup
| Parameter | Value |
|---|---|
| Task Type | Regression |
| Model Name | resnet18 |
| Epochs | 10 |
| Batch Size | Auto-selected batch size by Ludwig: 1 |
| Fine Tune | True |
| Use Pretrained | True |
| Learning Rate | Auto-selected learning rate by Ludwig: 1e-05 Based on model architecture and training setup (e.g., fine-tuning). See Ludwig Trainer Parameters for details. |
| Random Seed | 42 |
| Early Stop | 5 |
| Data Split | Used user-defined split column from CSV. |
Model trained using Ludwig.
If want to learn more about Ludwig default settings,please check their website(ludwig.ai).
Model Performance Summary
| Metric | Train | Validation | Test |
|---|---|---|---|
| Loss | 420.7510 | 2060.3052 | 8205.5977 |
| Mean Absolute Error | 17.3023 | 45.1572 | 86.0225 |
| Mean Absolute % Error | 0.6416 | 0.9613 | 0.9580 |
| Mean Squared Error | 420.7510 | 2060.3052 | 8205.5977 |
| R² Score | -2.1257 | -81.4122 | -10.2560 |
| Root Mean Squared Error | 20.5122 | 45.3906 | 90.5848 |
| Root Mean Squared % Error | 0.6416 | 0.9613 | 0.9580 |
+
+
+
+Train/Validation Performance Summary
| Metric | Train | Validation |
|---|---|---|
| Loss | 420.7510 | 2060.3052 |
| Mean Absolute Error | 17.3023 | 45.1572 |
| Mean Absolute % Error | 0.6416 | 0.9613 |
| Mean Squared Error | 420.7510 | 2060.3052 |
| R² Score | -2.1257 | -81.4122 |
| Root Mean Squared Error | 20.5122 | 45.3906 |
| Root Mean Squared % Error | 0.6416 | 0.9613 |
Training & Validation Visualizations
Learning Curves Label Loss
+
+
+
+
+
+
+
+
+
+Test Performance Summary
| Metric | Test |
|---|---|
| Loss | 8205.5977 |
| Mean Absolute Error | 86.0225 |
| Mean Absolute % Error | 0.9580 |
| Mean Squared Error | 8205.5977 |
| R² Score | -10.2560 |
| Root Mean Squared Error | 90.5848 |
| Root Mean Squared % Error | 0.9580 |
Predictions vs. Ground Truth
| label | +prediction | +
|---|---|
| 62 | +4.362794 | +
| 116 | +1.592189 | +
Test Visualizations
Compare Performance Label
+
+
+
+
+
+
+
+
+Config & Results Summary
+ Train/Validation Results
+ Test Results
+
+
+
+ {metrics_html}
+
+
+ {train_val_html}
+
+
+ {test_html}
+
+
+
+"""
+
+
+def get_metrics_help_modal() -> str:
+ modal_html = """
+
+
+
+"""
+ modal_css = """
+
+"""
+ modal_js = """
+
+"""
+ return modal_css + modal_html + modal_js