"""
Tool to train a PyTorch-based ``CTLearnModel`` on R1/DL1a data using the ``DLDataReader`` and ``PyTorchDataLoader``.
"""
__all__ = ["TrainCTLearnPyTorchModel"]
import os
import time
import torch
import torch.nn as nn
import torchmetrics
from ctapipe.core.tool import ToolConfigurationError
from ctapipe.core.traits import Bool, Int
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from ctlearn.core.model import CTLearnModel
from ctlearn.core.pytorch.dataset import PyTorchDataset
from ctlearn.tools.train_model import TrainCTLearnModel
from ctlearn.tools.utils import model_conv_backend, validate_conv_backend
[docs]
class TrainCTLearnPyTorchModel(TrainCTLearnModel):
"""
Tool to train a ``~ctlearn.core.model.CTLearnModel`` PyTorch-based model on R1/DL1a data.
The tool trains a CTLearn PyTorch-based model on the input data (R1 calibrated waveforms or DL1a images) and
saves the trained model in the output directory. The input data is loaded from the input directories
for signal and background events using the ``~dl1_data_handler.reader.DLDataReader`` and
``~dl1_data_handler.loader.DLDataLoader``. The tool supports the following reconstruction tasks:
- Classification of the primary particle type (gamma/proton)
- Regression of the primary particle energy
- Regression of the primary particle arrival direction based on the offsets in camera coordinates
- Regression of the primary particle arrival direction based on the offsets in sky coordinates
"""
name = "ctlearn-train-pytorch-model"
description = __doc__
examples = """
To train a PyTorch-based CTLearn model for the classification of the primary particle type:
> ctlearn-train-pytorch-model \
--signal /path/to/your/gammas_dl1_dir/ \
--pattern-signal "gamma_*_run1.dl1.h5" \
--pattern-signal "gamma_*_run10.dl1.h5" \
--background /path/to/your/protons_dl1_dir/ \
--pattern-background "proton_*_run1.dl1.h5" \
--pattern-background "proton_*_run10.dl1.h5" \
--output /path/to/your/type/ \
--reco type \
To train a PyTorch-based CTLearn model for the regression of the primary particle energy:
> ctlearn-train-pytorch-model \
--signal /path/to/your/gammas_dl1_dir/ \
--pattern-signal "gamma_*_run1.dl1.h5" \
--pattern-signal "gamma_*_run10.dl1.h5" \
--output /path/to/your/energy/ \
--reco energy \
To train a PyTorch-based CTLearn model for the regression of the primary particle
arrival direction based on the offsets in camera coordinates:
> ctlearn-train-pytorch-model \
--signal /path/to/your/gammas_dl1_dir/ \
--pattern-signal "gamma_*_run1.dl1.h5" \
--pattern-signal "gamma_*_run10.dl1.h5" \
--output /path/to/your/direction/ \
--reco cameradirection \
To train a PyTorch-based CTLearn model for the regression of the primary particle
arrival direction based on the offsets in sky coordinates:
> ctlearn-train-pytorch-model \
--signal /path/to/your/gammas_dl1_dir/ \
--pattern-signal "gamma_*_run1.dl1.h5" \
--pattern-signal "gamma_*_run10.dl1.h5" \
--output /path/to/your/direction/ \
--reco skydirection \
"""
num_workers = Int(
default_value=0,
allow_none=False,
help="Number of worker processes to use for data loading (0 means data will be loaded in the main process).",
).tag(config=True)
persistent_workers = Bool(
default_value=False,
allow_none=False,
help="Set whether to keep data loader worker processes alive between epochs.",
).tag(config=True)
[docs]
def setup_framework(self):
"""
Initialize the backend-specific parameters (e.g., PyTorch or Keras devices and distributed strategies).
This method ensures that the hardware accelerators (like GPUs) are properly allocated
and configures things like precision mixed-training and multi-processing workers.
"""
if self.persistent_workers and self.num_workers == 0:
raise ToolConfigurationError(
"Cannot set persistent_workers=True when num_workers=0. "
"PyTorch requires num_workers > 0 to keep worker processes alive. "
"Please set num_workers > 0 or persistent_workers=False."
)
# Determine available hardware device (Multi-GPU / Single GPU / CPU)
if torch.cuda.is_available():
self.device = torch.device("cuda")
self.num_devices = torch.cuda.device_count()
self.log.info("Using CUDA device(s). Available GPUs: %d", self.num_devices)
else:
self.device = torch.device("cpu")
self.num_devices = 1
self.log.info("Using CPU device.")
# Set seed globally for reproducibility across random operations
torch.manual_seed(self.random_seed)
# Create a dedicated Generator for the DataLoader
g = torch.Generator()
g.manual_seed(self.random_seed)
dataloader_kwargs = {
"batch_size": self.batch_size * self.num_devices,
"num_workers": self.num_workers,
"persistent_workers": self.persistent_workers,
"pin_memory": torch.cuda.is_available(),
}
if self.num_workers > 0:
dataloader_kwargs["multiprocessing_context"] = "fork"
# Init the PyTorchDataLoader
self.training_dataset = PyTorchDataset(
DLDataReader=self.dl1dh_reader,
indices=self.training_indices,
tasks=self.reco_tasks,
sort_by_intensity=self.sort_by_intensity,
stack_telescope_images=self.stack_telescope_images,
)
self.training_loader = DataLoader(
dataset=self.training_dataset,
shuffle=True, # Enables shuffling
generator=g, # Controls the shuffling seed deterministically
**dataloader_kwargs,
)
self.validation_dataset = PyTorchDataset(
DLDataReader=self.dl1dh_reader,
indices=self.validation_indices,
tasks=self.reco_tasks,
sort_by_intensity=self.sort_by_intensity,
stack_telescope_images=self.stack_telescope_images,
)
self.validation_loader = DataLoader(
dataset=self.validation_dataset,
shuffle=False, # Disables shuffling
**dataloader_kwargs,
)
# Set up TensorBoard writers for train and validation and CSV logging path
self.train_writer = SummaryWriter(
log_dir=os.path.join(self.output_dir, "train")
)
self.val_writer = SummaryWriter(
log_dir=os.path.join(self.output_dir, "validation")
)
self.csv_log_path = os.path.join(self.output_dir, "training_log.csv")
# Initialize TorchMetrics according to reconstruction tasks
self.train_metrics = self._get_task_metrics()
self.val_metrics = self._get_task_metrics()
# Build CSV dynamic header matching Keras logger format
self.csv_headers = ["epoch"]
if "type" in self.reco_tasks:
self.csv_headers.extend(["accuracy", "auc"])
if "energy" in self.reco_tasks:
self.csv_headers.append("mae_energy")
if "cameradirection" in self.reco_tasks:
self.csv_headers.append("mae_cameradirection")
if "skydirection" in self.reco_tasks:
self.csv_headers.append("mae_skydirection")
self.csv_headers.append("loss")
# Add validation metrics to header
if "type" in self.reco_tasks:
self.csv_headers.extend(["val_accuracy", "val_auc"])
if "energy" in self.reco_tasks:
self.csv_headers.append("val_mae_energy")
if "cameradirection" in self.reco_tasks:
self.csv_headers.append("val_mae_cameradirection")
if "skydirection" in self.reco_tasks:
self.csv_headers.append("val_mae_skydirection")
self.csv_headers.append("val_loss")
# Write CSV header if file doesn't exist
if not os.path.exists(self.csv_log_path):
with open(self.csv_log_path, "w") as f:
f.write(",".join(self.csv_headers) + "\n")
[docs]
def start(self):
"""
Execute the core logic of the tool.
Depending on the tool, this method either orchestrates the training and validation
loops across epochs, or it iterates through the input dataset to generate
and save model predictions to the output file.
"""
self.log.info("Setting up the PyTorch model.")
base_model = CTLearnModel.from_name(
f"PyTorch{self.model_type}",
input_shape=self.training_dataset.input_shape,
tasks=self.reco_tasks,
parent=self,
).model
# Validate that the image mapper(s) and the model's conv backend
# agree. This inspects the built model's layers rather than reading
# the conv_backend trait, because LoadedModel wraps an already
# trained model whose trait keeps its default value regardless of
# what that model was actually built with.
validate_conv_backend(
self.dl1dh_reader.image_mappers, model_conv_backend(base_model)
)
base_model.to(self.device)
if self.device.type == "cuda" and self.num_devices > 1:
self.model = nn.DataParallel(base_model)
else:
self.model = base_model
optimizers = {
"Adadelta": lambda params: torch.optim.Adadelta(
params, lr=self.learning_rate
),
"Adam": lambda params: torch.optim.Adam(
params, lr=self.learning_rate, eps=self.adam_epsilon
),
"RMSProp": lambda params: torch.optim.RMSprop(
params, lr=self.learning_rate
),
"SGD": lambda params: torch.optim.SGD(params, lr=self.learning_rate),
}
self.opt = optimizers[self.optimizer["name"]](self.model.parameters())
# Setup Learning Rate Scheduler
self.scheduler = None
if self.lr_reducing is not None:
self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
self.opt,
mode="min",
factor=self.lr_reducing["factor"],
patience=self.lr_reducing["patience"],
threshold=self.lr_reducing["min_delta"],
min_lr=self.lr_reducing["min_lr"],
)
self.loss_fns = self._get_loss_functions()
self.log.info("Training and evaluating...")
best_val_loss = float("inf")
patience_counter = 0
best_model_weights = None
model_save_path = os.path.join(self.output_dir, "ctlearn_model.pth")
state_dict_save_path = os.path.join(self.output_dir, "ctlearn_state_dict.pth")
for epoch_idx in range(self.n_epochs):
epoch_start = time.perf_counter()
train_loss, train_metric_vals = self._train_epoch()
val_loss, val_metric_vals = self._validate_epoch()
epoch_duration = time.perf_counter() - epoch_start
epoch_metrics = [
("loss", train_loss),
*train_metric_vals.items(),
("val_loss", val_loss),
]
epoch_metrics.extend(
(f"val_{name}", value) for name, value in val_metric_vals.items()
)
metrics_summary = " - ".join(
f"{name}: {value:.4f}" for name, value in epoch_metrics
)
self.log.info(
"Epoch %d/%d - %s - time: %.2fs",
epoch_idx + 1,
self.n_epochs,
metrics_summary,
epoch_duration,
)
# Record CSV Row (0-indexed epochs matching Keras CSVLogger)
row_dict = {"epoch": epoch_idx, "loss": train_loss, "val_loss": val_loss}
for k, v in train_metric_vals.items():
row_dict[k] = v
for k, v in val_metric_vals.items():
row_dict[f"val_{k}"] = v
row_str = ",".join(str(row_dict[col]) for col in self.csv_headers)
with open(self.csv_log_path, "a") as f:
f.write(row_str + "\n")
# TensorBoard metrics training and validation
self.train_writer.add_scalar("epoch_loss", train_loss, epoch_idx)
for k, v in train_metric_vals.items():
self.train_writer.add_scalar(f"epoch_{k}", v, epoch_idx)
self.train_writer.flush()
self.val_writer.add_scalar("epoch_loss", val_loss, epoch_idx)
for k, v in val_metric_vals.items():
self.val_writer.add_scalar(f"epoch_{k}", v, epoch_idx)
self.val_writer.flush()
# Unwrap model if wrapped with DataParallel / DistributedDataParallel
unwrapped_model = (
self.model.module
if isinstance(
self.model, (nn.DataParallel, nn.parallel.DistributedDataParallel)
)
else self.model
)
# Checkpoint saving & Early stopping logic
previous_best_val_loss = best_val_loss
validation_improved = val_loss < best_val_loss
if validation_improved:
best_val_loss = val_loss
patience_counter = 0
state_dict = (
self.model.module.state_dict()
if isinstance(self.model, nn.DataParallel)
else self.model.state_dict()
)
best_model_weights = state_dict
previous_loss = (
"None"
if previous_best_val_loss == float("inf")
else f"{previous_best_val_loss:.5f}"
)
self.log.info(
"Epoch %d: val_loss improved from %s to %.5f, saving model to %s",
epoch_idx + 1,
previous_loss,
val_loss,
model_save_path,
)
if self.save_best_validation_only:
# Save state_dict (tensors only)
torch.save(state_dict, state_dict_save_path)
# Save full model and config
checkpoint = {
"model": unwrapped_model,
"model_state_dict": state_dict,
"ctlearn_config": dict(self.config) if self.config else {},
}
torch.save(checkpoint, model_save_path)
self.log.info(
"Epoch %d: finished saving model to %s",
epoch_idx + 1,
model_save_path,
)
else:
patience_counter += 1
self.log.info(
"Epoch %d: val_loss did not improve from %.5f",
epoch_idx + 1,
best_val_loss,
)
if not self.save_best_validation_only:
state_dict = (
self.model.module.state_dict()
if isinstance(self.model, nn.DataParallel)
else self.model.state_dict()
)
# Save state_dict (tensors only)
torch.save(state_dict, state_dict_save_path)
# Save full model and config
checkpoint = {
"model": unwrapped_model,
"model_state_dict": state_dict,
"ctlearn_config": dict(self.config) if self.config else {},
}
torch.save(checkpoint, model_save_path)
self.log.info(
"Epoch %d: finished saving model to %s",
epoch_idx + 1,
model_save_path,
)
if self.scheduler is not None:
self.scheduler.step(val_loss)
if self.early_stopping is not None:
if patience_counter >= self.early_stopping["patience"]:
self.log.info("Early stopping triggered at epoch %d.", epoch_idx)
if (
self.early_stopping["restore_best_weights"]
and best_model_weights is not None
):
unwrapped_model = (
self.model.module
if isinstance(self.model, nn.DataParallel)
else self.model
)
unwrapped_model.load_state_dict(best_model_weights)
break
# Close the TensorBoard writers
self.train_writer.close()
self.val_writer.close()
self.log.info("Training and evaluating finished successfully!")
def _train_epoch(self):
"""
Execute a single training epoch.
This method iterates over the training data loader, performs the forward pass, calculates the loss,
and updates the model weights using backpropagation.
Returns
-------
tuple
The average training loss and a dictionary of computed training metrics for the epoch.
"""
self.model.train()
for m in self.train_metrics.values():
m.reset()
total_loss = 0.0
total_samples = 0
for batch_x, batch_y in self.training_loader:
batch_x = self._to_device(batch_x)
batch_y = self._to_device(batch_y)
batch_size = (
batch_x.size(0)
if isinstance(batch_x, torch.Tensor)
else next(iter(batch_x.values())).size(0)
)
self.opt.zero_grad()
outputs, _ = self.model(batch_x)
loss = self._compute_combined_loss(outputs, batch_y)
loss.backward()
self.opt.step()
total_loss += loss.item() * batch_size
total_samples += batch_size
self._update_metrics(self.train_metrics, outputs, batch_y)
avg_loss = total_loss / total_samples
metric_results = {k: m.compute().item() for k, m in self.train_metrics.items()}
return avg_loss, metric_results
def _validate_epoch(self):
"""
Execute a single validation epoch.
This method iterates over the validation data loader in evaluation mode (no gradients),
computes the validation loss, and calculates the validation metrics.
Returns
-------
tuple
The average validation loss and a dictionary of computed validation metrics for the epoch.
"""
self.model.eval()
self._reset_metrics(self.val_metrics)
total_loss = 0.0
total_samples = 0
with torch.no_grad():
for batch_x, batch_y in self.validation_loader:
batch_x = self._to_device(batch_x)
batch_y = self._to_device(batch_y)
# Dynamically determine batch size (works for Tensors or dicts of Tensors)
batch_size = (
batch_x.size(0)
if isinstance(batch_x, torch.Tensor)
else next(iter(batch_x.values())).size(0)
)
outputs, _ = self.model(batch_x)
loss = self._compute_combined_loss(outputs, batch_y)
total_loss += loss.item() * batch_size
total_samples += batch_size
self._update_metrics(self.val_metrics, outputs, batch_y)
avg_loss = total_loss / total_samples if total_samples > 0 else 0.0
metric_results = self._compute_metrics(self.val_metrics)
return avg_loss, metric_results
# Helper methods to safely handle flat or task-nested metric dictionaries
def _reset_metrics(self, metrics):
"""
Reset all tracking metrics at the start of an epoch.
Parameters
----------
metrics : dict
A dictionary containing the TorchMetrics objects to reset.
"""
for m in metrics.values():
if isinstance(m, dict):
for sub_m in m.values():
sub_m.reset()
else:
m.reset()
def _compute_metrics(self, metrics):
"""
Compute the final values for all tracked metrics at the end of an epoch.
Parameters
----------
metrics : dict
A dictionary containing the tracked TorchMetrics objects.
Returns
-------
dict
A dictionary with the computed scalar values for each metric.
"""
results = {}
for k, v in metrics.items():
if isinstance(v, dict):
for sub_k, sub_v in v.items():
results[f"{k}_{sub_k}"] = sub_v.compute().item()
else:
results[k] = v.compute().item()
return results
def _get_task_metrics(self):
"""Instantiates TorchMetrics matching Keras metric definitions."""
metrics = {}
if "type" in self.reco_tasks:
num_classes = getattr(self.dl1dh_reader, "num_classes", 2)
metrics["accuracy"] = torchmetrics.Accuracy(
task="multiclass" if num_classes > 2 else "binary",
num_classes=num_classes,
).to(self.device)
metrics["auc"] = torchmetrics.AUROC(
task="multiclass" if num_classes > 2 else "binary",
num_classes=num_classes,
).to(self.device)
if "energy" in self.reco_tasks:
metrics["mae_energy"] = torchmetrics.MeanAbsoluteError().to(self.device)
if "cameradirection" in self.reco_tasks:
metrics["mae_cameradirection"] = torchmetrics.MeanAbsoluteError().to(
self.device
)
if "skydirection" in self.reco_tasks:
metrics["mae_skydirection"] = torchmetrics.MeanAbsoluteError().to(
self.device
)
return metrics
def _update_metrics(self, metrics, outputs, targets):
"""
Update the running metrics with the predictions and targets from the current batch.
Parameters
----------
metrics : dict
A dictionary of TorchMetrics objects.
outputs : dict or torch.Tensor
The predictions output by the model.
targets : dict or torch.Tensor
The ground truth targets.
"""
for task in self.reco_tasks:
out = outputs[task] if isinstance(outputs, dict) else outputs
tgt = targets[task] if isinstance(targets, dict) else targets
task_metrics = {
"type": {k: metrics[k] for k in ("accuracy", "auc") if k in metrics},
"energy": metrics.get("mae_energy"),
"cameradirection": metrics.get("mae_cameradirection"),
"skydirection": metrics.get("mae_skydirection"),
}[task]
if task == "type":
# Ensure target is 1D LongTensor for classification
if tgt.ndim > 1:
tgt = tgt.squeeze(-1)
tgt = tgt.long()
# Convert 2-class logits to class 1 probabilities for binary metrics
if out.ndim > 1 and out.shape[-1] > 1:
probs = torch.softmax(out, dim=-1)
out = probs[:, 1]
elif out.ndim > 1 and out.shape[-1] == 1:
out = torch.sigmoid(out.squeeze(-1))
else:
if out.ndim > 1 and out.shape[-1] == 1:
out = out.squeeze(-1)
if tgt.ndim > 1 and tgt.shape[-1] == 1:
tgt = tgt.squeeze(-1)
if isinstance(task_metrics, dict):
for metric_obj in task_metrics.values():
metric_obj.update(out, tgt)
else:
task_metrics.update(out, tgt)
def _compute_combined_loss(self, outputs, targets):
"""
Compute the total combined loss for all reconstruction tasks.
If the model is multi-task, this calculates the loss for each individual task (e.g., type, energy)
using their respective loss functions and sums them up.
Parameters
----------
outputs : dict or torch.Tensor
The predictions output by the model.
targets : dict or torch.Tensor
The ground truth targets.
Returns
-------
torch.Tensor
The scalar tensor representing the total combined loss.
"""
total_loss = 0.0
for task in self.reco_tasks:
task_output = outputs[task] if isinstance(outputs, dict) else outputs
task_target = targets[task] if isinstance(targets, dict) else targets
if task == "type":
if task_target.ndim > 1:
task_target = task_target.squeeze(-1)
task_target = task_target.long()
else:
if task_output.ndim > 1 and task_output.shape[-1] == 1:
task_output = task_output.squeeze(-1)
if task_target.ndim > 1 and task_target.shape[-1] == 1:
task_target = task_target.squeeze(-1)
task_loss = self.loss_fns[task](task_output, task_target)
total_loss += task_loss
return total_loss
def _get_loss_functions(self):
"""
Instantiate and retrieve the appropriate PyTorch loss functions for each active reconstruction task.
Returns
-------
dict
A dictionary mapping task names to their corresponding nn.Module loss functions.
"""
loss_fns = {}
if "type" in self.reco_tasks:
weight = None
if self.dl1dh_reader.class_weight is not None:
class_weights = self.dl1dh_reader.class_weight
# Convert dict to a list ordered by class index
if isinstance(class_weights, dict):
class_weights = [
class_weights[k] for k in sorted(class_weights.keys())
]
weight = torch.tensor(
class_weights,
dtype=torch.float32,
device=self.device,
)
loss_fns["type"] = torch.nn.CrossEntropyLoss(weight=weight)
if "energy" in self.reco_tasks:
loss_fns["energy"] = nn.L1Loss()
if "cameradirection" in self.reco_tasks:
loss_fns["cameradirection"] = nn.L1Loss()
if "skydirection" in self.reco_tasks:
loss_fns["skydirection"] = nn.L1Loss()
return loss_fns
def _to_device(self, data):
"""
Recursively move tensors or collections of tensors to the target hardware device (CPU/GPU).
Parameters
----------
data : torch.Tensor or dict or list
The data structure containing tensors to be moved.
Returns
-------
torch.Tensor or dict or list
The data structure with all tensors located on the target device.
"""
if isinstance(data, torch.Tensor):
return data.to(self.device)
elif isinstance(data, dict):
return {k: self._to_device(v) for k, v in data.items()}
elif isinstance(data, list):
return [self._to_device(v) for v in data]
return data
def main():
"""
Main entry point for the command-line tool.
This function instantiates the Tool class and invokes its `run()` method,
which sequentially executes the `setup()`, `start()`, and `finish()` methods.
"""
tool = TrainCTLearnPyTorchModel()
tool.run()
if __name__ == "__main__":
main()