Source code for ctlearn.tools.pytorch.train_model

"""
Tool to train a PyTorch-based ``CTLearnModel`` on R1/DL1a data using the ``DLDataReader`` and ``PyTorchDataLoader``.
"""

__all__ = ["TrainCTLearnPyTorchModel"]

import os
import time

import torch
import torch.nn as nn
import torchmetrics
from ctapipe.core.tool import ToolConfigurationError
from ctapipe.core.traits import Bool, Int
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

from ctlearn.core.model import CTLearnModel
from ctlearn.core.pytorch.dataset import PyTorchDataset
from ctlearn.tools.train_model import TrainCTLearnModel
from ctlearn.tools.utils import model_conv_backend, validate_conv_backend


[docs] class TrainCTLearnPyTorchModel(TrainCTLearnModel): """ Tool to train a ``~ctlearn.core.model.CTLearnModel`` PyTorch-based model on R1/DL1a data. The tool trains a CTLearn PyTorch-based model on the input data (R1 calibrated waveforms or DL1a images) and saves the trained model in the output directory. The input data is loaded from the input directories for signal and background events using the ``~dl1_data_handler.reader.DLDataReader`` and ``~dl1_data_handler.loader.DLDataLoader``. The tool supports the following reconstruction tasks: - Classification of the primary particle type (gamma/proton) - Regression of the primary particle energy - Regression of the primary particle arrival direction based on the offsets in camera coordinates - Regression of the primary particle arrival direction based on the offsets in sky coordinates """ name = "ctlearn-train-pytorch-model" description = __doc__ examples = """ To train a PyTorch-based CTLearn model for the classification of the primary particle type: > ctlearn-train-pytorch-model \ --signal /path/to/your/gammas_dl1_dir/ \ --pattern-signal "gamma_*_run1.dl1.h5" \ --pattern-signal "gamma_*_run10.dl1.h5" \ --background /path/to/your/protons_dl1_dir/ \ --pattern-background "proton_*_run1.dl1.h5" \ --pattern-background "proton_*_run10.dl1.h5" \ --output /path/to/your/type/ \ --reco type \ To train a PyTorch-based CTLearn model for the regression of the primary particle energy: > ctlearn-train-pytorch-model \ --signal /path/to/your/gammas_dl1_dir/ \ --pattern-signal "gamma_*_run1.dl1.h5" \ --pattern-signal "gamma_*_run10.dl1.h5" \ --output /path/to/your/energy/ \ --reco energy \ To train a PyTorch-based CTLearn model for the regression of the primary particle arrival direction based on the offsets in camera coordinates: > ctlearn-train-pytorch-model \ --signal /path/to/your/gammas_dl1_dir/ \ --pattern-signal "gamma_*_run1.dl1.h5" \ --pattern-signal "gamma_*_run10.dl1.h5" \ --output /path/to/your/direction/ \ --reco cameradirection \ To train a PyTorch-based CTLearn model for the regression of the primary particle arrival direction based on the offsets in sky coordinates: > ctlearn-train-pytorch-model \ --signal /path/to/your/gammas_dl1_dir/ \ --pattern-signal "gamma_*_run1.dl1.h5" \ --pattern-signal "gamma_*_run10.dl1.h5" \ --output /path/to/your/direction/ \ --reco skydirection \ """ num_workers = Int( default_value=0, allow_none=False, help="Number of worker processes to use for data loading (0 means data will be loaded in the main process).", ).tag(config=True) persistent_workers = Bool( default_value=False, allow_none=False, help="Set whether to keep data loader worker processes alive between epochs.", ).tag(config=True)
[docs] def setup_framework(self): """ Initialize the backend-specific parameters (e.g., PyTorch or Keras devices and distributed strategies). This method ensures that the hardware accelerators (like GPUs) are properly allocated and configures things like precision mixed-training and multi-processing workers. """ if self.persistent_workers and self.num_workers == 0: raise ToolConfigurationError( "Cannot set persistent_workers=True when num_workers=0. " "PyTorch requires num_workers > 0 to keep worker processes alive. " "Please set num_workers > 0 or persistent_workers=False." ) # Determine available hardware device (Multi-GPU / Single GPU / CPU) if torch.cuda.is_available(): self.device = torch.device("cuda") self.num_devices = torch.cuda.device_count() self.log.info("Using CUDA device(s). Available GPUs: %d", self.num_devices) else: self.device = torch.device("cpu") self.num_devices = 1 self.log.info("Using CPU device.") # Set seed globally for reproducibility across random operations torch.manual_seed(self.random_seed) # Create a dedicated Generator for the DataLoader g = torch.Generator() g.manual_seed(self.random_seed) dataloader_kwargs = { "batch_size": self.batch_size * self.num_devices, "num_workers": self.num_workers, "persistent_workers": self.persistent_workers, "pin_memory": torch.cuda.is_available(), } if self.num_workers > 0: dataloader_kwargs["multiprocessing_context"] = "fork" # Init the PyTorchDataLoader self.training_dataset = PyTorchDataset( DLDataReader=self.dl1dh_reader, indices=self.training_indices, tasks=self.reco_tasks, sort_by_intensity=self.sort_by_intensity, stack_telescope_images=self.stack_telescope_images, ) self.training_loader = DataLoader( dataset=self.training_dataset, shuffle=True, # Enables shuffling generator=g, # Controls the shuffling seed deterministically **dataloader_kwargs, ) self.validation_dataset = PyTorchDataset( DLDataReader=self.dl1dh_reader, indices=self.validation_indices, tasks=self.reco_tasks, sort_by_intensity=self.sort_by_intensity, stack_telescope_images=self.stack_telescope_images, ) self.validation_loader = DataLoader( dataset=self.validation_dataset, shuffle=False, # Disables shuffling **dataloader_kwargs, ) # Set up TensorBoard writers for train and validation and CSV logging path self.train_writer = SummaryWriter( log_dir=os.path.join(self.output_dir, "train") ) self.val_writer = SummaryWriter( log_dir=os.path.join(self.output_dir, "validation") ) self.csv_log_path = os.path.join(self.output_dir, "training_log.csv") # Initialize TorchMetrics according to reconstruction tasks self.train_metrics = self._get_task_metrics() self.val_metrics = self._get_task_metrics() # Build CSV dynamic header matching Keras logger format self.csv_headers = ["epoch"] if "type" in self.reco_tasks: self.csv_headers.extend(["accuracy", "auc"]) if "energy" in self.reco_tasks: self.csv_headers.append("mae_energy") if "cameradirection" in self.reco_tasks: self.csv_headers.append("mae_cameradirection") if "skydirection" in self.reco_tasks: self.csv_headers.append("mae_skydirection") self.csv_headers.append("loss") # Add validation metrics to header if "type" in self.reco_tasks: self.csv_headers.extend(["val_accuracy", "val_auc"]) if "energy" in self.reco_tasks: self.csv_headers.append("val_mae_energy") if "cameradirection" in self.reco_tasks: self.csv_headers.append("val_mae_cameradirection") if "skydirection" in self.reco_tasks: self.csv_headers.append("val_mae_skydirection") self.csv_headers.append("val_loss") # Write CSV header if file doesn't exist if not os.path.exists(self.csv_log_path): with open(self.csv_log_path, "w") as f: f.write(",".join(self.csv_headers) + "\n")
[docs] def start(self): """ Execute the core logic of the tool. Depending on the tool, this method either orchestrates the training and validation loops across epochs, or it iterates through the input dataset to generate and save model predictions to the output file. """ self.log.info("Setting up the PyTorch model.") base_model = CTLearnModel.from_name( f"PyTorch{self.model_type}", input_shape=self.training_dataset.input_shape, tasks=self.reco_tasks, parent=self, ).model # Validate that the image mapper(s) and the model's conv backend # agree. This inspects the built model's layers rather than reading # the conv_backend trait, because LoadedModel wraps an already # trained model whose trait keeps its default value regardless of # what that model was actually built with. validate_conv_backend( self.dl1dh_reader.image_mappers, model_conv_backend(base_model) ) base_model.to(self.device) if self.device.type == "cuda" and self.num_devices > 1: self.model = nn.DataParallel(base_model) else: self.model = base_model optimizers = { "Adadelta": lambda params: torch.optim.Adadelta( params, lr=self.learning_rate ), "Adam": lambda params: torch.optim.Adam( params, lr=self.learning_rate, eps=self.adam_epsilon ), "RMSProp": lambda params: torch.optim.RMSprop( params, lr=self.learning_rate ), "SGD": lambda params: torch.optim.SGD(params, lr=self.learning_rate), } self.opt = optimizers[self.optimizer["name"]](self.model.parameters()) # Setup Learning Rate Scheduler self.scheduler = None if self.lr_reducing is not None: self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( self.opt, mode="min", factor=self.lr_reducing["factor"], patience=self.lr_reducing["patience"], threshold=self.lr_reducing["min_delta"], min_lr=self.lr_reducing["min_lr"], ) self.loss_fns = self._get_loss_functions() self.log.info("Training and evaluating...") best_val_loss = float("inf") patience_counter = 0 best_model_weights = None model_save_path = os.path.join(self.output_dir, "ctlearn_model.pth") state_dict_save_path = os.path.join(self.output_dir, "ctlearn_state_dict.pth") for epoch_idx in range(self.n_epochs): epoch_start = time.perf_counter() train_loss, train_metric_vals = self._train_epoch() val_loss, val_metric_vals = self._validate_epoch() epoch_duration = time.perf_counter() - epoch_start epoch_metrics = [ ("loss", train_loss), *train_metric_vals.items(), ("val_loss", val_loss), ] epoch_metrics.extend( (f"val_{name}", value) for name, value in val_metric_vals.items() ) metrics_summary = " - ".join( f"{name}: {value:.4f}" for name, value in epoch_metrics ) self.log.info( "Epoch %d/%d - %s - time: %.2fs", epoch_idx + 1, self.n_epochs, metrics_summary, epoch_duration, ) # Record CSV Row (0-indexed epochs matching Keras CSVLogger) row_dict = {"epoch": epoch_idx, "loss": train_loss, "val_loss": val_loss} for k, v in train_metric_vals.items(): row_dict[k] = v for k, v in val_metric_vals.items(): row_dict[f"val_{k}"] = v row_str = ",".join(str(row_dict[col]) for col in self.csv_headers) with open(self.csv_log_path, "a") as f: f.write(row_str + "\n") # TensorBoard metrics training and validation self.train_writer.add_scalar("epoch_loss", train_loss, epoch_idx) for k, v in train_metric_vals.items(): self.train_writer.add_scalar(f"epoch_{k}", v, epoch_idx) self.train_writer.flush() self.val_writer.add_scalar("epoch_loss", val_loss, epoch_idx) for k, v in val_metric_vals.items(): self.val_writer.add_scalar(f"epoch_{k}", v, epoch_idx) self.val_writer.flush() # Unwrap model if wrapped with DataParallel / DistributedDataParallel unwrapped_model = ( self.model.module if isinstance( self.model, (nn.DataParallel, nn.parallel.DistributedDataParallel) ) else self.model ) # Checkpoint saving & Early stopping logic previous_best_val_loss = best_val_loss validation_improved = val_loss < best_val_loss if validation_improved: best_val_loss = val_loss patience_counter = 0 state_dict = ( self.model.module.state_dict() if isinstance(self.model, nn.DataParallel) else self.model.state_dict() ) best_model_weights = state_dict previous_loss = ( "None" if previous_best_val_loss == float("inf") else f"{previous_best_val_loss:.5f}" ) self.log.info( "Epoch %d: val_loss improved from %s to %.5f, saving model to %s", epoch_idx + 1, previous_loss, val_loss, model_save_path, ) if self.save_best_validation_only: # Save state_dict (tensors only) torch.save(state_dict, state_dict_save_path) # Save full model and config checkpoint = { "model": unwrapped_model, "model_state_dict": state_dict, "ctlearn_config": dict(self.config) if self.config else {}, } torch.save(checkpoint, model_save_path) self.log.info( "Epoch %d: finished saving model to %s", epoch_idx + 1, model_save_path, ) else: patience_counter += 1 self.log.info( "Epoch %d: val_loss did not improve from %.5f", epoch_idx + 1, best_val_loss, ) if not self.save_best_validation_only: state_dict = ( self.model.module.state_dict() if isinstance(self.model, nn.DataParallel) else self.model.state_dict() ) # Save state_dict (tensors only) torch.save(state_dict, state_dict_save_path) # Save full model and config checkpoint = { "model": unwrapped_model, "model_state_dict": state_dict, "ctlearn_config": dict(self.config) if self.config else {}, } torch.save(checkpoint, model_save_path) self.log.info( "Epoch %d: finished saving model to %s", epoch_idx + 1, model_save_path, ) if self.scheduler is not None: self.scheduler.step(val_loss) if self.early_stopping is not None: if patience_counter >= self.early_stopping["patience"]: self.log.info("Early stopping triggered at epoch %d.", epoch_idx) if ( self.early_stopping["restore_best_weights"] and best_model_weights is not None ): unwrapped_model = ( self.model.module if isinstance(self.model, nn.DataParallel) else self.model ) unwrapped_model.load_state_dict(best_model_weights) break # Close the TensorBoard writers self.train_writer.close() self.val_writer.close() self.log.info("Training and evaluating finished successfully!")
def _train_epoch(self): """ Execute a single training epoch. This method iterates over the training data loader, performs the forward pass, calculates the loss, and updates the model weights using backpropagation. Returns ------- tuple The average training loss and a dictionary of computed training metrics for the epoch. """ self.model.train() for m in self.train_metrics.values(): m.reset() total_loss = 0.0 total_samples = 0 for batch_x, batch_y in self.training_loader: batch_x = self._to_device(batch_x) batch_y = self._to_device(batch_y) batch_size = ( batch_x.size(0) if isinstance(batch_x, torch.Tensor) else next(iter(batch_x.values())).size(0) ) self.opt.zero_grad() outputs, _ = self.model(batch_x) loss = self._compute_combined_loss(outputs, batch_y) loss.backward() self.opt.step() total_loss += loss.item() * batch_size total_samples += batch_size self._update_metrics(self.train_metrics, outputs, batch_y) avg_loss = total_loss / total_samples metric_results = {k: m.compute().item() for k, m in self.train_metrics.items()} return avg_loss, metric_results def _validate_epoch(self): """ Execute a single validation epoch. This method iterates over the validation data loader in evaluation mode (no gradients), computes the validation loss, and calculates the validation metrics. Returns ------- tuple The average validation loss and a dictionary of computed validation metrics for the epoch. """ self.model.eval() self._reset_metrics(self.val_metrics) total_loss = 0.0 total_samples = 0 with torch.no_grad(): for batch_x, batch_y in self.validation_loader: batch_x = self._to_device(batch_x) batch_y = self._to_device(batch_y) # Dynamically determine batch size (works for Tensors or dicts of Tensors) batch_size = ( batch_x.size(0) if isinstance(batch_x, torch.Tensor) else next(iter(batch_x.values())).size(0) ) outputs, _ = self.model(batch_x) loss = self._compute_combined_loss(outputs, batch_y) total_loss += loss.item() * batch_size total_samples += batch_size self._update_metrics(self.val_metrics, outputs, batch_y) avg_loss = total_loss / total_samples if total_samples > 0 else 0.0 metric_results = self._compute_metrics(self.val_metrics) return avg_loss, metric_results # Helper methods to safely handle flat or task-nested metric dictionaries def _reset_metrics(self, metrics): """ Reset all tracking metrics at the start of an epoch. Parameters ---------- metrics : dict A dictionary containing the TorchMetrics objects to reset. """ for m in metrics.values(): if isinstance(m, dict): for sub_m in m.values(): sub_m.reset() else: m.reset() def _compute_metrics(self, metrics): """ Compute the final values for all tracked metrics at the end of an epoch. Parameters ---------- metrics : dict A dictionary containing the tracked TorchMetrics objects. Returns ------- dict A dictionary with the computed scalar values for each metric. """ results = {} for k, v in metrics.items(): if isinstance(v, dict): for sub_k, sub_v in v.items(): results[f"{k}_{sub_k}"] = sub_v.compute().item() else: results[k] = v.compute().item() return results def _get_task_metrics(self): """Instantiates TorchMetrics matching Keras metric definitions.""" metrics = {} if "type" in self.reco_tasks: num_classes = getattr(self.dl1dh_reader, "num_classes", 2) metrics["accuracy"] = torchmetrics.Accuracy( task="multiclass" if num_classes > 2 else "binary", num_classes=num_classes, ).to(self.device) metrics["auc"] = torchmetrics.AUROC( task="multiclass" if num_classes > 2 else "binary", num_classes=num_classes, ).to(self.device) if "energy" in self.reco_tasks: metrics["mae_energy"] = torchmetrics.MeanAbsoluteError().to(self.device) if "cameradirection" in self.reco_tasks: metrics["mae_cameradirection"] = torchmetrics.MeanAbsoluteError().to( self.device ) if "skydirection" in self.reco_tasks: metrics["mae_skydirection"] = torchmetrics.MeanAbsoluteError().to( self.device ) return metrics def _update_metrics(self, metrics, outputs, targets): """ Update the running metrics with the predictions and targets from the current batch. Parameters ---------- metrics : dict A dictionary of TorchMetrics objects. outputs : dict or torch.Tensor The predictions output by the model. targets : dict or torch.Tensor The ground truth targets. """ for task in self.reco_tasks: out = outputs[task] if isinstance(outputs, dict) else outputs tgt = targets[task] if isinstance(targets, dict) else targets task_metrics = { "type": {k: metrics[k] for k in ("accuracy", "auc") if k in metrics}, "energy": metrics.get("mae_energy"), "cameradirection": metrics.get("mae_cameradirection"), "skydirection": metrics.get("mae_skydirection"), }[task] if task == "type": # Ensure target is 1D LongTensor for classification if tgt.ndim > 1: tgt = tgt.squeeze(-1) tgt = tgt.long() # Convert 2-class logits to class 1 probabilities for binary metrics if out.ndim > 1 and out.shape[-1] > 1: probs = torch.softmax(out, dim=-1) out = probs[:, 1] elif out.ndim > 1 and out.shape[-1] == 1: out = torch.sigmoid(out.squeeze(-1)) else: if out.ndim > 1 and out.shape[-1] == 1: out = out.squeeze(-1) if tgt.ndim > 1 and tgt.shape[-1] == 1: tgt = tgt.squeeze(-1) if isinstance(task_metrics, dict): for metric_obj in task_metrics.values(): metric_obj.update(out, tgt) else: task_metrics.update(out, tgt) def _compute_combined_loss(self, outputs, targets): """ Compute the total combined loss for all reconstruction tasks. If the model is multi-task, this calculates the loss for each individual task (e.g., type, energy) using their respective loss functions and sums them up. Parameters ---------- outputs : dict or torch.Tensor The predictions output by the model. targets : dict or torch.Tensor The ground truth targets. Returns ------- torch.Tensor The scalar tensor representing the total combined loss. """ total_loss = 0.0 for task in self.reco_tasks: task_output = outputs[task] if isinstance(outputs, dict) else outputs task_target = targets[task] if isinstance(targets, dict) else targets if task == "type": if task_target.ndim > 1: task_target = task_target.squeeze(-1) task_target = task_target.long() else: if task_output.ndim > 1 and task_output.shape[-1] == 1: task_output = task_output.squeeze(-1) if task_target.ndim > 1 and task_target.shape[-1] == 1: task_target = task_target.squeeze(-1) task_loss = self.loss_fns[task](task_output, task_target) total_loss += task_loss return total_loss def _get_loss_functions(self): """ Instantiate and retrieve the appropriate PyTorch loss functions for each active reconstruction task. Returns ------- dict A dictionary mapping task names to their corresponding nn.Module loss functions. """ loss_fns = {} if "type" in self.reco_tasks: weight = None if self.dl1dh_reader.class_weight is not None: class_weights = self.dl1dh_reader.class_weight # Convert dict to a list ordered by class index if isinstance(class_weights, dict): class_weights = [ class_weights[k] for k in sorted(class_weights.keys()) ] weight = torch.tensor( class_weights, dtype=torch.float32, device=self.device, ) loss_fns["type"] = torch.nn.CrossEntropyLoss(weight=weight) if "energy" in self.reco_tasks: loss_fns["energy"] = nn.L1Loss() if "cameradirection" in self.reco_tasks: loss_fns["cameradirection"] = nn.L1Loss() if "skydirection" in self.reco_tasks: loss_fns["skydirection"] = nn.L1Loss() return loss_fns def _to_device(self, data): """ Recursively move tensors or collections of tensors to the target hardware device (CPU/GPU). Parameters ---------- data : torch.Tensor or dict or list The data structure containing tensors to be moved. Returns ------- torch.Tensor or dict or list The data structure with all tensors located on the target device. """ if isinstance(data, torch.Tensor): return data.to(self.device) elif isinstance(data, dict): return {k: self._to_device(v) for k, v in data.items()} elif isinstance(data, list): return [self._to_device(v) for v in data] return data
def main(): """ Main entry point for the command-line tool. This function instantiates the Tool class and invokes its `run()` method, which sequentially executes the `setup()`, `start()`, and `finish()` methods. """ tool = TrainCTLearnPyTorchModel() tool.run() if __name__ == "__main__": main()