Source code for ctlearn.core.keras.model

"""
This module defines the ``CTLearnModel`` classes, which holds the basic functionality for creating a Keras model to be used in CTLearn.
"""

__all__ = [
    "build_fully_connect_keras_head",
    "KerasConvBackend",
    "KerasSingleCNN",
    "KerasResNet",
    "KerasLoadedModel",
]

from functools import partial

import keras
import keras_hexagdly as hgly

from ctlearn.core.keras.attention import (
    channel_squeeze_excite_block,
    dual_squeeze_excite_block,
    spatial_squeeze_excite_block,
)
from ctlearn.core.model import (
    ConvBackend,
    LoadedModel,
    ResNet,
    SingleCNN,
)


[docs] def build_fully_connect_keras_head(inputs, layers, activation_function, tasks): """ Build the fully connected head for the Keras-based CTLearn model. Function to build the fully connected head of the Keras-based CTLearn model using the specified parameters. Parameters ---------- inputs : keras.layers.Layer Keras layer of the model. layers : dict Dictionary containing the number of neurons (as value) in the fully connected head for each task (as key). activation_function : dict Dictionary containing the activation function (as value) for the fully connected head for each task (as key). tasks : list List of tasks to build the head for. Returns ------- logits : dict Dictionary containing the logits for each task. """ logits = {} for task in tasks: x = inputs for i, units in enumerate(layers[task]): if i != len(layers[task]) - 1: x = keras.layers.Dense( units=units, activation=activation_function[task], name=f"fc_{task}_{i + 1}", )(x) else: x = keras.layers.Dense(units=units, name=task)(x) logits[task] = keras.layers.Softmax()(x) if task == "type" else x # Temp fix till keras support class weights for multiple outputs or I wrote custom loss # https://github.com/keras-team/keras/issues/11735 if len(tasks) == 1 and tasks[0] == "type": logits = logits[tasks[0]] return logits
# The hexagonal layers read ``kernel_size`` as a square-kernel width, like the # square layers do, instead of counting hexagonal rings (their default): 3 is # the 7-cell hexagon and 1 a plain 1x1. The same configuration then describes # comparable models on both grids. _HexConv2d = partial(hgly.Conv2d, kernel_size_unit="pixels") _HexMaxPool2d = partial(hgly.MaxPool2d, kernel_size_unit="pixels") _HexAvgPool2d = partial(hgly.AvgPool2d, kernel_size_unit="pixels")
[docs] class KerasConvBackend(ConvBackend): """Select Keras layers for square or hexagonal model backbones. The ``conv_backend`` trait selects native Keras layers for square-mapped inputs or ``keras_hexagdly`` layers for hex-addressed inputs. """ @property def _conv_layer(self): """Convolution layer class for ``conv_backend``. Both are built with the same keyword arguments, and ``kernel_size`` has the same meaning for both: a square-kernel width. The hex grid is always padded (``"same"``), including where the square call leaves Keras' default ``"valid"``. """ return _HexConv2d if self.conv_backend == "hexagdly" else keras.layers.Conv2D @property def _pool_layer(self): """Max-pooling layer class for ``conv_backend``.""" return ( _HexMaxPool2d if self.conv_backend == "hexagdly" else keras.layers.MaxPool2D ) @property def _avg_pool_layer(self): """Average-pooling layer class for ``conv_backend``.""" return ( _HexAvgPool2d if self.conv_backend == "hexagdly" else keras.layers.AveragePooling2D )
[docs] class KerasSingleCNN(SingleCNN, KerasConvBackend): """ ``SingleCNN`` is a simple convolutional neural network model. This class extends the functionality of ``CTLearnModel`` by implementing methods to build a simple convolutional neural network model. """ def __init__( self, input_shape, tasks, config=None, parent=None, **kwargs, ): """ Initialize the KerasSingleCNN model component. Parameters ---------- input_shape : tuple Input image tensor dimensions (height, width, channels). tasks : list of str List of target prediction tasks. config : traitlets.config.Config, optional Configuration specified by config file or cmdline arguments. parent : ctapipe.core.Component or ctapipe.core.Tool, optional Parent component in hierarchy. **kwargs : dict, optional Additional keyword arguments. """ super().__init__( tasks=tasks, config=config, parent=parent, **kwargs, ) # Build the ResNet model backbone self.backbone_model, self.input_layer = self._build_backbone(input_shape) backbone_output = self.backbone_model(self.input_layer) # Build the fully connected head depending on the tasks self.logits = build_fully_connect_keras_head( backbone_output, self.head_layers, self.head_activation_function, tasks ) # Build the full pipeline model∫ self.model = keras.Model(self.input_layer, self.logits, name="CTLearn_model") def _build_backbone(self, input_shape): """ Build the SingleCNN model backbone. Function to build the backbone of the SingleCNN model using the specified parameters. Parameters ---------- input_shape : tuple Shape of the input data (batch_size, height, width, channels). Returns ------- backbone_model : keras.Model Keras model object representing the backbone of the SingleCNN model. network_input : keras.Input Keras input layer object for the backbone model. """ # Define the input layer from the input shape network_input = keras.Input(shape=input_shape) # Get model arcihtecture parameters for the backbone filters_list = [layer["filters"] for layer in self.architecture] kernel_sizes = [layer["kernel_size"] for layer in self.architecture] numbers_list = [layer["number"] for layer in self.architecture] x = network_input if self.batchnorm: x = keras.layers.BatchNormalization(momentum=0.99)(x) for i, (filters, kernel_size, number) in enumerate( zip(filters_list, kernel_sizes, numbers_list) ): for nr in range(number): x = self._conv_layer( filters=filters, kernel_size=kernel_size, padding="same", activation="relu", name=f"{self.backbone_name}_conv_{i + 1}_{nr + 1}", )(x) if self.pooling_type is not None: if self.pooling_type == "max": x = self._pool_layer( pool_size=self.pooling_parameters["size"], strides=self.pooling_parameters["strides"], name=f"{self.backbone_name}_pool_{i + 1}", )(x) elif self.pooling_type == "average": x = self._avg_pool_layer( pool_size=self.pooling_parameters["size"], strides=self.pooling_parameters["strides"], name=f"{self.backbone_name}_pool_{i + 1}", )(x) if self.batchnorm: x = keras.layers.BatchNormalization(momentum=0.99)(x) # bottleneck layer -- a 1x1 conv is a pure per-cell channel projection # with no spatial mixing. It is built with the backend's conv class like # every other conv: kernel_size=1 is a plain 1x1 for both backends. if self.bottleneck_filters is not None: x = self._conv_layer( filters=self.bottleneck_filters, kernel_size=1, padding="same", activation="relu", name=f"{self.backbone_name}_bottleneck", )(x) if self.batchnorm: x = keras.layers.BatchNormalization(momentum=0.99)(x) # Attention mechanism if self.attention is not None: if self.attention["mechanism"] == "Dual-SE": x = dual_squeeze_excite_block( x, self.attention["reduction_ratio"], name=f"{self.backbone_name}_dse", ) elif self.attention["mechanism"] == "Channel-SE": x = channel_squeeze_excite_block( x, self.attention["reduction_ratio"], name=f"{self.backbone_name}_cse", ) elif self.attention["mechanism"] == "Spatial-SE": x = spatial_squeeze_excite_block(x, name=f"{self.backbone_name}_sse") # Apply global average pooling as the final layer of the backbone network_output = keras.layers.GlobalAveragePooling2D( name=self.backbone_name + "_global_avgpool" )(x) # Create the backbone model backbone_model = keras.Model( network_input, network_output, name=self.backbone_name ) return backbone_model, network_input
[docs] class KerasResNet(ResNet, KerasConvBackend): """ ``ResNet`` is a residual neural network model. This class extends the functionality of ``CTLearnModel`` by implementing methods to build a residual neural network model. """ def __init__( self, input_shape, tasks, config=None, parent=None, **kwargs, ): """ Initialize the KerasResNet model component. Parameters ---------- input_shape : tuple Input image tensor dimensions (height, width, channels). tasks : list of str List of target prediction tasks. config : traitlets.config.Config, optional Configuration specified by config file or cmdline arguments. parent : ctapipe.core.Component or ctapipe.core.Tool, optional Parent component in hierarchy. **kwargs : dict, optional Additional keyword arguments. """ super().__init__( tasks=tasks, config=config, parent=parent, **kwargs, ) # Build the ResNet model backbone self.backbone_model, self.input_layer = self._build_backbone(input_shape) backbone_output = self.backbone_model(self.input_layer) # Build the fully connected head depending on the tasks self.logits = build_fully_connect_keras_head( backbone_output, self.head_layers, self.head_activation_function, tasks ) self.model = keras.Model(self.input_layer, self.logits, name="CTLearn_model") def _build_backbone(self, input_shape): """ Build the ResNet model backbone. Function to build the backbone of the ResNet model using the specified parameters. Parameters ---------- input_shape : tuple Shape of the input data (batch_size, height, width, channels). Returns ------- backbone_model : keras.Model Keras model object representing the ResNet backbone. network_input : keras.Input Keras input layer object for the backbone model. """ # Define the input layer from the input shape network_input = keras.Input(shape=input_shape) x = network_input # Apply initial padding if specified if self.init_padding > 0: x = keras.layers.ZeroPadding2D( padding=self.init_padding, name=self.backbone_name + "_padding", )(x) # Apply initial convolutional layer if specified if self.init_layer is not None: x = self._conv_layer( filters=self.init_layer["filters"], kernel_size=self.init_layer["kernel_size"], strides=self.init_layer["strides"], name=self.backbone_name + "_conv1_conv", )(x) # Apply max pooling if specified if self.init_max_pool is not None: x = self._pool_layer( pool_size=self.init_max_pool["size"], strides=self.init_max_pool["strides"], name=self.backbone_name + "_pool1_pool", )(x) # Build the residual blocks engine_output = self._stacked_res_blocks( x, architecture=self.architecture, residual_block_type=self.residual_block_type, attention=self.attention, name=self.backbone_name, ) # Apply global average pooling as the final layer of the backbone network_output = keras.layers.GlobalAveragePooling2D( name=self.backbone_name + "_global_avgpool" )(engine_output) # Create the backbone model backbone_model = keras.Model( network_input, network_output, name=self.backbone_name ) return backbone_model, network_input def _stacked_res_blocks( self, inputs, architecture, residual_block_type, attention, name=None ): """ Build a stack of residual blocks for the CTLearn model. This function constructs a stack of residual blocks, which are used to build the backbone of the CTLearn model. Each residual block consists of a series of convolutional layers with skip connections. Parameters ---------- inputs : keras.layers.Layer Input Keras layer to the residual blocks. architecture : list of dict List of dictionaries containing the architecture of the ResNet model, which includes: - Number of filters for the convolutional layers in the residual blocks. - Number of residual blocks to stack. residual_block_type : str Type of residual block to use. Options are 'basic' or 'bottleneck'. attention : dict Dictionary containing the configuration parameters for the attention mechanism. name : str, optional Label for the model. Returns ------- x : keras.layers.Layer Output Keras layer after passing through the stack of residual blocks. """ # Get hyperparameters for the model architecture filters_list = [layer["filters"] for layer in architecture] blocks_list = [layer["blocks"] for layer in architecture] # Build the ResNet model x = self._stack_fn( inputs, filters_list[0], blocks_list[0], residual_block_type, stride=1, attention=attention, name=name + "_conv2", ) for i, (filters, blocks) in enumerate(zip(filters_list[1:], blocks_list[1:])): x = self._stack_fn( x, filters, blocks, residual_block_type, attention=attention, name=name + "_conv" + str(i + 3), ) return x def _stack_fn( self, inputs, filters, blocks, residual_block_type, stride=2, attention=None, name=None, ): """ Stack residual blocks for the CTLearn model. This function constructs a stack of residual blocks, which are used to build the backbone of the CTLearn model. Each residual block can be of different types (e.g., basic or bottleneck) and can include attention mechanisms. Parameters ---------- inputs : keras.layers.Layer Input tensor to the residual blocks. filters : int Number of filters for the bottleneck layer in a block. blocks : int Number of residual blocks to stack. residual_block_type : str Type of residual block ('basic' or 'bottleneck'). stride : int, optional Stride for the first layer in the first block. Default is 2. attention : dict, optional Configuration parameters for the attention mechanism. Default is None. name : str, optional Label for the stack. Default is None. Returns ------- keras.layers.Layer Output tensor for the stacked blocks. """ res_blocks = { "basic": self._basic_residual_block, "bottleneck": self._bottleneck_residual_block, } x = res_blocks[residual_block_type]( inputs, filters, stride=stride, attention=attention, name=name + "_block1", ) for i in range(2, blocks + 1): x = res_blocks[residual_block_type]( x, filters, conv_shortcut=False, attention=attention, name=name + "_block" + str(i), ) return x def _basic_residual_block( self, inputs, filters, kernel_size=3, stride=1, conv_shortcut=True, attention=None, name=None, ): """ Build a basic residual block for the CTLearn model. This function constructs a basic residual block, which is a fundamental building block of ResNet architectures. The block consists of two convolutional layers with an optional convolutional shortcut, and can include attention mechanisms. Parameters ---------- inputs : keras.layers.Layer Input tensor to the residual block. filters : int Number of filters for the convolutional layers. kernel_size : int, optional Size of the convolutional kernel. Default is 3. stride : int, optional Stride for the convolutional layers. Default is 1. conv_shortcut : bool, optional Whether to use a convolutional layer for the shortcut connection. Default is True. attention : dict, optional Configuration parameters for the attention mechanism. Default is None. name : str, optional Name for the residual block. Default is None. Returns ------- keras.layers.Layer Output tensor after applying the residual block. """ if conv_shortcut: shortcut = self._conv_layer( filters=filters, kernel_size=1, strides=stride, name=name + "_0_conv" )(inputs) else: shortcut = inputs x = self._conv_layer( filters=filters, kernel_size=kernel_size, strides=stride, padding="same", activation="relu", name=name + "_1_conv", )(inputs) x = self._conv_layer( filters=filters, kernel_size=kernel_size, padding="same", activation="relu", name=name + "_2_conv", )(x) # Attention mechanism if attention is not None: if attention["mechanism"] == "Dual-SE": x = dual_squeeze_excite_block( x, attention["reduction_ratio"], name=name + "_dse" ) elif attention["mechanism"] == "Channel-SE": x = channel_squeeze_excite_block( x, attention["reduction_ratio"], name=name + "_cse" ) elif attention["mechanism"] == "Spatial-SE": x = spatial_squeeze_excite_block(x, name=name + "_sse") x = keras.layers.Add(name=name + "_add")([x, shortcut]) x = keras.layers.ReLU(name=name + "_out")(x) return x def _bottleneck_residual_block( self, inputs, filters, kernel_size=3, stride=1, conv_shortcut=True, attention=None, name=None, ): """ Build a bottleneck residual block for the CTLearn model. This function constructs a bottleneck residual block, which is a fundamental building block of ResNet architectures. The block consists of three convolutional layers: a 1x1 convolution to reduce dimensionality, a 3x3 convolution for main computation, and another 1x1 convolution to restore dimensionality. It also includes an optional shortcut connection and can include attention mechanisms. Parameters ---------- inputs : keras.layers.Layer Input tensor to the residual block. filters : int Number of filters for the convolutional layers. kernel_size : int, optional Size of the convolutional kernel. Default is 3. stride : int, optional Stride for the convolutional layers. Default is 1. conv_shortcut : bool, optional Whether to use a convolutional layer for the shortcut connection. Default is True. attention : dict, optional Configuration parameters for the attention mechanism. Default is None. name : str, optional Name for the residual block. Default is None. Returns ------- output : keras.layers.Layer Output layer of the residual block. """ if conv_shortcut: shortcut = self._conv_layer( filters=4 * filters, kernel_size=1, strides=stride, name=name + "_0_conv", )(inputs) else: shortcut = inputs x = self._conv_layer( filters=filters, kernel_size=1, strides=stride, activation="relu", name=name + "_1_conv", )(inputs) x = self._conv_layer( filters=filters, kernel_size=kernel_size, padding="same", activation="relu", name=name + "_2_conv", )(x) x = self._conv_layer(filters=4 * filters, kernel_size=1, name=name + "_3_conv")( x ) # Attention mechanism if attention is not None: if attention["mechanism"] == "Dual-SE": x = dual_squeeze_excite_block( x, attention["reduction_ratio"], name=name + "_dse" ) elif attention["mechanism"] == "Channel-SE": x = channel_squeeze_excite_block( x, attention["reduction_ratio"], name=name + "_cse" ) elif attention["mechanism"] == "Spatial-SE": x = spatial_squeeze_excite_block(x, name=name + "_sse") x = keras.layers.Add(name=name + "_add")([x, shortcut]) x = keras.layers.ReLU(name=name + "_out")(x) return x
[docs] class KerasLoadedModel(LoadedModel): """ ``LoadedModel`` is a pre-trained Keras model. This class extends the functionality of ``CTLearnModel`` by implementing methods to load a pre-trained Keras model. The model can be used as a backbone for the CTLearn model. """ def __init__( self, input_shape, tasks, config=None, parent=None, **kwargs, ): """ Initialize the KerasLoadedModel component. Parameters ---------- input_shape : tuple Input image tensor dimensions (height, width, channels). tasks : list of str List of target prediction tasks. config : traitlets.config.Config, optional Configuration specified by config file or cmdline arguments. parent : ctapipe.core.Component or ctapipe.core.Tool, optional Parent component in hierarchy. **kwargs : dict, optional Additional keyword arguments. """ super().__init__( tasks=tasks, config=config, parent=parent, **kwargs, ) # Load the model from the specified path self.model = keras.saving.load_model(self.load_model_from) # Build the ResNet model backbone self.backbone_model, self.input_layer = self._build_backbone(input_shape) # Load the fully connected head from the loaded model or build a new one if self.overwrite_head: backbone_output = self.backbone_model(self.input_layer) # Build the fully connected head depending on the tasks self.logits = build_fully_connect_keras_head( backbone_output, self.head_layers, self.head_activation_function, tasks ) self.model = keras.Model( self.input_layer, self.logits, name="CTLearn_model" ) def _build_backbone(self, input_shape): """ Build the LoadedModel backbone. Function to build the backbone of the LoadedModel using the specified parameters. Parameters ---------- input_shape : tuple Shape of the input data (batch_size, height, width, channels). Returns ------- backbone_model : keras.Model Keras model object representing the LoadedModel backbone. network_input : keras.Input Keras input layer object for the backbone model. """ # Define the input layer from the input shape network_input = keras.Input(shape=input_shape) # Set the backbone model to be trainable or not for layer in self.model.layers: if layer.name.endswith("_block"): backbone_layer = self.model.get_layer(layer.name) self.backbone_name = backbone_layer.name backbone_layer.trainable = self.trainable_backbone network_output = backbone_layer(network_input) # Create the backbone model backbone_model = keras.Model( network_input, network_output, name=self.backbone_name ) return backbone_model, network_input