"""
This module defines the ``CTLearnModel`` classes, which holds the basic functionality for creating a Keras model to be used in CTLearn.
"""
__all__ = [
"build_fully_connect_keras_head",
"KerasConvBackend",
"KerasSingleCNN",
"KerasResNet",
"KerasLoadedModel",
]
from functools import partial
import keras
import keras_hexagdly as hgly
from ctlearn.core.keras.attention import (
channel_squeeze_excite_block,
dual_squeeze_excite_block,
spatial_squeeze_excite_block,
)
from ctlearn.core.model import (
ConvBackend,
LoadedModel,
ResNet,
SingleCNN,
)
[docs]
def build_fully_connect_keras_head(inputs, layers, activation_function, tasks):
"""
Build the fully connected head for the Keras-based CTLearn model.
Function to build the fully connected head of the Keras-based CTLearn model using the specified parameters.
Parameters
----------
inputs : keras.layers.Layer
Keras layer of the model.
layers : dict
Dictionary containing the number of neurons (as value) in the fully connected head for each task (as key).
activation_function : dict
Dictionary containing the activation function (as value) for the fully connected head for each task (as key).
tasks : list
List of tasks to build the head for.
Returns
-------
logits : dict
Dictionary containing the logits for each task.
"""
logits = {}
for task in tasks:
x = inputs
for i, units in enumerate(layers[task]):
if i != len(layers[task]) - 1:
x = keras.layers.Dense(
units=units,
activation=activation_function[task],
name=f"fc_{task}_{i + 1}",
)(x)
else:
x = keras.layers.Dense(units=units, name=task)(x)
logits[task] = keras.layers.Softmax()(x) if task == "type" else x
# Temp fix till keras support class weights for multiple outputs or I wrote custom loss
# https://github.com/keras-team/keras/issues/11735
if len(tasks) == 1 and tasks[0] == "type":
logits = logits[tasks[0]]
return logits
# The hexagonal layers read ``kernel_size`` as a square-kernel width, like the
# square layers do, instead of counting hexagonal rings (their default): 3 is
# the 7-cell hexagon and 1 a plain 1x1. The same configuration then describes
# comparable models on both grids.
_HexConv2d = partial(hgly.Conv2d, kernel_size_unit="pixels")
_HexMaxPool2d = partial(hgly.MaxPool2d, kernel_size_unit="pixels")
_HexAvgPool2d = partial(hgly.AvgPool2d, kernel_size_unit="pixels")
[docs]
class KerasConvBackend(ConvBackend):
"""Select Keras layers for square or hexagonal model backbones.
The ``conv_backend`` trait selects native Keras layers for square-mapped
inputs or ``keras_hexagdly`` layers for hex-addressed inputs.
"""
@property
def _conv_layer(self):
"""Convolution layer class for ``conv_backend``.
Both are built with the same keyword arguments, and ``kernel_size`` has
the same meaning for both: a square-kernel width. The hex grid is
always padded (``"same"``), including where the square call leaves
Keras' default ``"valid"``.
"""
return _HexConv2d if self.conv_backend == "hexagdly" else keras.layers.Conv2D
@property
def _pool_layer(self):
"""Max-pooling layer class for ``conv_backend``."""
return (
_HexMaxPool2d if self.conv_backend == "hexagdly" else keras.layers.MaxPool2D
)
@property
def _avg_pool_layer(self):
"""Average-pooling layer class for ``conv_backend``."""
return (
_HexAvgPool2d
if self.conv_backend == "hexagdly"
else keras.layers.AveragePooling2D
)
[docs]
class KerasSingleCNN(SingleCNN, KerasConvBackend):
"""
``SingleCNN`` is a simple convolutional neural network model.
This class extends the functionality of ``CTLearnModel`` by implementing
methods to build a simple convolutional neural network model.
"""
def __init__(
self,
input_shape,
tasks,
config=None,
parent=None,
**kwargs,
):
"""
Initialize the KerasSingleCNN model component.
Parameters
----------
input_shape : tuple
Input image tensor dimensions (height, width, channels).
tasks : list of str
List of target prediction tasks.
config : traitlets.config.Config, optional
Configuration specified by config file or cmdline arguments.
parent : ctapipe.core.Component or ctapipe.core.Tool, optional
Parent component in hierarchy.
**kwargs : dict, optional
Additional keyword arguments.
"""
super().__init__(
tasks=tasks,
config=config,
parent=parent,
**kwargs,
)
# Build the ResNet model backbone
self.backbone_model, self.input_layer = self._build_backbone(input_shape)
backbone_output = self.backbone_model(self.input_layer)
# Build the fully connected head depending on the tasks
self.logits = build_fully_connect_keras_head(
backbone_output, self.head_layers, self.head_activation_function, tasks
)
# Build the full pipeline model∫
self.model = keras.Model(self.input_layer, self.logits, name="CTLearn_model")
def _build_backbone(self, input_shape):
"""
Build the SingleCNN model backbone.
Function to build the backbone of the SingleCNN model using the specified parameters.
Parameters
----------
input_shape : tuple
Shape of the input data (batch_size, height, width, channels).
Returns
-------
backbone_model : keras.Model
Keras model object representing the backbone of the SingleCNN model.
network_input : keras.Input
Keras input layer object for the backbone model.
"""
# Define the input layer from the input shape
network_input = keras.Input(shape=input_shape)
# Get model arcihtecture parameters for the backbone
filters_list = [layer["filters"] for layer in self.architecture]
kernel_sizes = [layer["kernel_size"] for layer in self.architecture]
numbers_list = [layer["number"] for layer in self.architecture]
x = network_input
if self.batchnorm:
x = keras.layers.BatchNormalization(momentum=0.99)(x)
for i, (filters, kernel_size, number) in enumerate(
zip(filters_list, kernel_sizes, numbers_list)
):
for nr in range(number):
x = self._conv_layer(
filters=filters,
kernel_size=kernel_size,
padding="same",
activation="relu",
name=f"{self.backbone_name}_conv_{i + 1}_{nr + 1}",
)(x)
if self.pooling_type is not None:
if self.pooling_type == "max":
x = self._pool_layer(
pool_size=self.pooling_parameters["size"],
strides=self.pooling_parameters["strides"],
name=f"{self.backbone_name}_pool_{i + 1}",
)(x)
elif self.pooling_type == "average":
x = self._avg_pool_layer(
pool_size=self.pooling_parameters["size"],
strides=self.pooling_parameters["strides"],
name=f"{self.backbone_name}_pool_{i + 1}",
)(x)
if self.batchnorm:
x = keras.layers.BatchNormalization(momentum=0.99)(x)
# bottleneck layer -- a 1x1 conv is a pure per-cell channel projection
# with no spatial mixing. It is built with the backend's conv class like
# every other conv: kernel_size=1 is a plain 1x1 for both backends.
if self.bottleneck_filters is not None:
x = self._conv_layer(
filters=self.bottleneck_filters,
kernel_size=1,
padding="same",
activation="relu",
name=f"{self.backbone_name}_bottleneck",
)(x)
if self.batchnorm:
x = keras.layers.BatchNormalization(momentum=0.99)(x)
# Attention mechanism
if self.attention is not None:
if self.attention["mechanism"] == "Dual-SE":
x = dual_squeeze_excite_block(
x,
self.attention["reduction_ratio"],
name=f"{self.backbone_name}_dse",
)
elif self.attention["mechanism"] == "Channel-SE":
x = channel_squeeze_excite_block(
x,
self.attention["reduction_ratio"],
name=f"{self.backbone_name}_cse",
)
elif self.attention["mechanism"] == "Spatial-SE":
x = spatial_squeeze_excite_block(x, name=f"{self.backbone_name}_sse")
# Apply global average pooling as the final layer of the backbone
network_output = keras.layers.GlobalAveragePooling2D(
name=self.backbone_name + "_global_avgpool"
)(x)
# Create the backbone model
backbone_model = keras.Model(
network_input, network_output, name=self.backbone_name
)
return backbone_model, network_input
[docs]
class KerasResNet(ResNet, KerasConvBackend):
"""
``ResNet`` is a residual neural network model.
This class extends the functionality of ``CTLearnModel`` by implementing
methods to build a residual neural network model.
"""
def __init__(
self,
input_shape,
tasks,
config=None,
parent=None,
**kwargs,
):
"""
Initialize the KerasResNet model component.
Parameters
----------
input_shape : tuple
Input image tensor dimensions (height, width, channels).
tasks : list of str
List of target prediction tasks.
config : traitlets.config.Config, optional
Configuration specified by config file or cmdline arguments.
parent : ctapipe.core.Component or ctapipe.core.Tool, optional
Parent component in hierarchy.
**kwargs : dict, optional
Additional keyword arguments.
"""
super().__init__(
tasks=tasks,
config=config,
parent=parent,
**kwargs,
)
# Build the ResNet model backbone
self.backbone_model, self.input_layer = self._build_backbone(input_shape)
backbone_output = self.backbone_model(self.input_layer)
# Build the fully connected head depending on the tasks
self.logits = build_fully_connect_keras_head(
backbone_output, self.head_layers, self.head_activation_function, tasks
)
self.model = keras.Model(self.input_layer, self.logits, name="CTLearn_model")
def _build_backbone(self, input_shape):
"""
Build the ResNet model backbone.
Function to build the backbone of the ResNet model using the specified parameters.
Parameters
----------
input_shape : tuple
Shape of the input data (batch_size, height, width, channels).
Returns
-------
backbone_model : keras.Model
Keras model object representing the ResNet backbone.
network_input : keras.Input
Keras input layer object for the backbone model.
"""
# Define the input layer from the input shape
network_input = keras.Input(shape=input_shape)
x = network_input
# Apply initial padding if specified
if self.init_padding > 0:
x = keras.layers.ZeroPadding2D(
padding=self.init_padding,
name=self.backbone_name + "_padding",
)(x)
# Apply initial convolutional layer if specified
if self.init_layer is not None:
x = self._conv_layer(
filters=self.init_layer["filters"],
kernel_size=self.init_layer["kernel_size"],
strides=self.init_layer["strides"],
name=self.backbone_name + "_conv1_conv",
)(x)
# Apply max pooling if specified
if self.init_max_pool is not None:
x = self._pool_layer(
pool_size=self.init_max_pool["size"],
strides=self.init_max_pool["strides"],
name=self.backbone_name + "_pool1_pool",
)(x)
# Build the residual blocks
engine_output = self._stacked_res_blocks(
x,
architecture=self.architecture,
residual_block_type=self.residual_block_type,
attention=self.attention,
name=self.backbone_name,
)
# Apply global average pooling as the final layer of the backbone
network_output = keras.layers.GlobalAveragePooling2D(
name=self.backbone_name + "_global_avgpool"
)(engine_output)
# Create the backbone model
backbone_model = keras.Model(
network_input, network_output, name=self.backbone_name
)
return backbone_model, network_input
def _stacked_res_blocks(
self, inputs, architecture, residual_block_type, attention, name=None
):
"""
Build a stack of residual blocks for the CTLearn model.
This function constructs a stack of residual blocks, which are used to build the backbone of the CTLearn model.
Each residual block consists of a series of convolutional layers with skip connections.
Parameters
----------
inputs : keras.layers.Layer
Input Keras layer to the residual blocks.
architecture : list of dict
List of dictionaries containing the architecture of the ResNet model, which includes:
- Number of filters for the convolutional layers in the residual blocks.
- Number of residual blocks to stack.
residual_block_type : str
Type of residual block to use. Options are 'basic' or 'bottleneck'.
attention : dict
Dictionary containing the configuration parameters for the attention mechanism.
name : str, optional
Label for the model.
Returns
-------
x : keras.layers.Layer
Output Keras layer after passing through the stack of residual blocks.
"""
# Get hyperparameters for the model architecture
filters_list = [layer["filters"] for layer in architecture]
blocks_list = [layer["blocks"] for layer in architecture]
# Build the ResNet model
x = self._stack_fn(
inputs,
filters_list[0],
blocks_list[0],
residual_block_type,
stride=1,
attention=attention,
name=name + "_conv2",
)
for i, (filters, blocks) in enumerate(zip(filters_list[1:], blocks_list[1:])):
x = self._stack_fn(
x,
filters,
blocks,
residual_block_type,
attention=attention,
name=name + "_conv" + str(i + 3),
)
return x
def _stack_fn(
self,
inputs,
filters,
blocks,
residual_block_type,
stride=2,
attention=None,
name=None,
):
"""
Stack residual blocks for the CTLearn model.
This function constructs a stack of residual blocks, which are used to build the backbone of the CTLearn model.
Each residual block can be of different types (e.g., basic or bottleneck) and can include attention mechanisms.
Parameters
----------
inputs : keras.layers.Layer
Input tensor to the residual blocks.
filters : int
Number of filters for the bottleneck layer in a block.
blocks : int
Number of residual blocks to stack.
residual_block_type : str
Type of residual block ('basic' or 'bottleneck').
stride : int, optional
Stride for the first layer in the first block. Default is 2.
attention : dict, optional
Configuration parameters for the attention mechanism. Default is None.
name : str, optional
Label for the stack. Default is None.
Returns
-------
keras.layers.Layer
Output tensor for the stacked blocks.
"""
res_blocks = {
"basic": self._basic_residual_block,
"bottleneck": self._bottleneck_residual_block,
}
x = res_blocks[residual_block_type](
inputs,
filters,
stride=stride,
attention=attention,
name=name + "_block1",
)
for i in range(2, blocks + 1):
x = res_blocks[residual_block_type](
x,
filters,
conv_shortcut=False,
attention=attention,
name=name + "_block" + str(i),
)
return x
def _basic_residual_block(
self,
inputs,
filters,
kernel_size=3,
stride=1,
conv_shortcut=True,
attention=None,
name=None,
):
"""
Build a basic residual block for the CTLearn model.
This function constructs a basic residual block, which is a fundamental building block
of ResNet architectures. The block consists of two convolutional layers with an optional
convolutional shortcut, and can include attention mechanisms.
Parameters
----------
inputs : keras.layers.Layer
Input tensor to the residual block.
filters : int
Number of filters for the convolutional layers.
kernel_size : int, optional
Size of the convolutional kernel. Default is 3.
stride : int, optional
Stride for the convolutional layers. Default is 1.
conv_shortcut : bool, optional
Whether to use a convolutional layer for the shortcut connection. Default is True.
attention : dict, optional
Configuration parameters for the attention mechanism. Default is None.
name : str, optional
Name for the residual block. Default is None.
Returns
-------
keras.layers.Layer
Output tensor after applying the residual block.
"""
if conv_shortcut:
shortcut = self._conv_layer(
filters=filters, kernel_size=1, strides=stride, name=name + "_0_conv"
)(inputs)
else:
shortcut = inputs
x = self._conv_layer(
filters=filters,
kernel_size=kernel_size,
strides=stride,
padding="same",
activation="relu",
name=name + "_1_conv",
)(inputs)
x = self._conv_layer(
filters=filters,
kernel_size=kernel_size,
padding="same",
activation="relu",
name=name + "_2_conv",
)(x)
# Attention mechanism
if attention is not None:
if attention["mechanism"] == "Dual-SE":
x = dual_squeeze_excite_block(
x, attention["reduction_ratio"], name=name + "_dse"
)
elif attention["mechanism"] == "Channel-SE":
x = channel_squeeze_excite_block(
x, attention["reduction_ratio"], name=name + "_cse"
)
elif attention["mechanism"] == "Spatial-SE":
x = spatial_squeeze_excite_block(x, name=name + "_sse")
x = keras.layers.Add(name=name + "_add")([x, shortcut])
x = keras.layers.ReLU(name=name + "_out")(x)
return x
def _bottleneck_residual_block(
self,
inputs,
filters,
kernel_size=3,
stride=1,
conv_shortcut=True,
attention=None,
name=None,
):
"""
Build a bottleneck residual block for the CTLearn model.
This function constructs a bottleneck residual block, which is a fundamental building block of
ResNet architectures. The block consists of three convolutional layers: a 1x1 convolution to reduce
dimensionality, a 3x3 convolution for main computation, and another 1x1 convolution to restore dimensionality.
It also includes an optional shortcut connection and can include attention mechanisms.
Parameters
----------
inputs : keras.layers.Layer
Input tensor to the residual block.
filters : int
Number of filters for the convolutional layers.
kernel_size : int, optional
Size of the convolutional kernel. Default is 3.
stride : int, optional
Stride for the convolutional layers. Default is 1.
conv_shortcut : bool, optional
Whether to use a convolutional layer for the shortcut connection. Default is True.
attention : dict, optional
Configuration parameters for the attention mechanism. Default is None.
name : str, optional
Name for the residual block. Default is None.
Returns
-------
output : keras.layers.Layer
Output layer of the residual block.
"""
if conv_shortcut:
shortcut = self._conv_layer(
filters=4 * filters,
kernel_size=1,
strides=stride,
name=name + "_0_conv",
)(inputs)
else:
shortcut = inputs
x = self._conv_layer(
filters=filters,
kernel_size=1,
strides=stride,
activation="relu",
name=name + "_1_conv",
)(inputs)
x = self._conv_layer(
filters=filters,
kernel_size=kernel_size,
padding="same",
activation="relu",
name=name + "_2_conv",
)(x)
x = self._conv_layer(filters=4 * filters, kernel_size=1, name=name + "_3_conv")(
x
)
# Attention mechanism
if attention is not None:
if attention["mechanism"] == "Dual-SE":
x = dual_squeeze_excite_block(
x, attention["reduction_ratio"], name=name + "_dse"
)
elif attention["mechanism"] == "Channel-SE":
x = channel_squeeze_excite_block(
x, attention["reduction_ratio"], name=name + "_cse"
)
elif attention["mechanism"] == "Spatial-SE":
x = spatial_squeeze_excite_block(x, name=name + "_sse")
x = keras.layers.Add(name=name + "_add")([x, shortcut])
x = keras.layers.ReLU(name=name + "_out")(x)
return x
[docs]
class KerasLoadedModel(LoadedModel):
"""
``LoadedModel`` is a pre-trained Keras model.
This class extends the functionality of ``CTLearnModel`` by implementing
methods to load a pre-trained Keras model. The model can be used as a backbone
for the CTLearn model.
"""
def __init__(
self,
input_shape,
tasks,
config=None,
parent=None,
**kwargs,
):
"""
Initialize the KerasLoadedModel component.
Parameters
----------
input_shape : tuple
Input image tensor dimensions (height, width, channels).
tasks : list of str
List of target prediction tasks.
config : traitlets.config.Config, optional
Configuration specified by config file or cmdline arguments.
parent : ctapipe.core.Component or ctapipe.core.Tool, optional
Parent component in hierarchy.
**kwargs : dict, optional
Additional keyword arguments.
"""
super().__init__(
tasks=tasks,
config=config,
parent=parent,
**kwargs,
)
# Load the model from the specified path
self.model = keras.saving.load_model(self.load_model_from)
# Build the ResNet model backbone
self.backbone_model, self.input_layer = self._build_backbone(input_shape)
# Load the fully connected head from the loaded model or build a new one
if self.overwrite_head:
backbone_output = self.backbone_model(self.input_layer)
# Build the fully connected head depending on the tasks
self.logits = build_fully_connect_keras_head(
backbone_output, self.head_layers, self.head_activation_function, tasks
)
self.model = keras.Model(
self.input_layer, self.logits, name="CTLearn_model"
)
def _build_backbone(self, input_shape):
"""
Build the LoadedModel backbone.
Function to build the backbone of the LoadedModel using the specified parameters.
Parameters
----------
input_shape : tuple
Shape of the input data (batch_size, height, width, channels).
Returns
-------
backbone_model : keras.Model
Keras model object representing the LoadedModel backbone.
network_input : keras.Input
Keras input layer object for the backbone model.
"""
# Define the input layer from the input shape
network_input = keras.Input(shape=input_shape)
# Set the backbone model to be trainable or not
for layer in self.model.layers:
if layer.name.endswith("_block"):
backbone_layer = self.model.get_layer(layer.name)
self.backbone_name = backbone_layer.name
backbone_layer.trainable = self.trainable_backbone
network_output = backbone_layer(network_input)
# Create the backbone model
backbone_model = keras.Model(
network_input, network_output, name=self.backbone_name
)
return backbone_model, network_input