Source code for lis2.pipeline

import copy
import os
from pathlib import Path
from pprint import pprint
from typing import Union, Any, Optional

import astropy.io.fits as fits

from configurable import Configurable, Schema, Config

from .controller import FoldsController
from .segmenter import Segmenter
from .trainer import Trainer
from .utils.distributed import get_device_ids, setup, cleanup, synchronize


[docs] class KfoldsTrainingPipeline(Configurable): """ A pipeline for training models using K-Fold cross-validation. This class handles the configuration and execution of training across multiple folds, including dataset parsing, training, and inference. Configuration: - **run_name** (str): The name of the training run. - **inference_source** (Union[Path, str], optional): The path to the inference data source. - **train_output_dir** (Union[Path, str]): The directory where training outputs are saved. - **nb_folds** (int): The number of folds for cross-validation. - **data** (Config): The dataset configuration. - **trainer** (Config): The trainer configuration. - **model** (Config): The model configuration. - **gpus** (Union[int, list[int], str], optional): The GPU IDs to use for training. If 'auto', the pipeline will use all available GPUs. Default: 'auto'. Example Configuration: .. code-block:: yaml run_name: "example_run" inference_source: "data/test.fits" train_output_dir: "output" nb_folds: 5 data: dataset_path: "data/train.fits" type: "fits" controller: type: "kfold" nb_folds: 5 trainset: type: "fits" transform: "default" batch_size: 32 validset: type: "fits" transform: "default" batch_size: 32 testset: type: "fits" transform: "default" batch_size: 32 trainer: type: "default" epochs: 10 batch_size: 32 learning_rate: 0.001 model: type: "default" name: "example_model" """ aliases = ["kfold_pipeline"] config_schema = { "run_name": Schema(str), "inference_source": Schema(Path, optional=True), "train_output_dir": Schema(Path), "nb_folds": Schema(int, default=1), "data": Schema(type=Config), "trainer": Schema(type=Config), "model": Schema(type=Config), "gpus": Schema(Union[int, list[int], str], default="auto"), } def __init__(self) -> None: """ Initializes the KfoldsTrainingPipeline with configuration parsing and setup. """ self.data = DataConfig.from_config(self.data) ( self.folds_controller_config, self.train_config, self.valid_config, self.test_config, ) = self.data.parse_datasets_config() self.folds_controller = FoldsController.from_config( self.folds_controller_config ) self.trainer["output_dir"] = self.train_output_dir os.makedirs(self.train_output_dir, exist_ok=True) self.save_dict_to_yaml( self.config, Path(self.train_output_dir) / "config_pipeline.yaml" ) self.gpus = get_device_ids(self.gpus)
[docs] def preconditions(self) -> None: """ Checks preconditions for the inference source. """ if self.inference_source is not None: self.inference_source = ( Path(self.inference_source) if isinstance(self.inference_source, str) else self.inference_source ) assert ( self.inference_source.exists() ), f"{self.inference_source} does not exist" assert ( self.inference_source.suffix == ".fits" ), f"{self.inference_source} is not a FITS file"
[docs] def run_training(self) -> None: """ Executes training across multiple folds defined by the folds controller. This method iterates through each fold, updating dataset configurations and trainer settings accordingly. It trains a model for each fold and logs the results. If inference is enabled, predictions from each fold are aggregated and saved as a FITS file. Steps: 1. Iterate through the predefined folds. 2. Update dataset configurations for training, validation, and testing. 3. Configure and initialize a trainer for the current fold. 4. Train the model and log the results. 5. If inference is enabled, generate and aggregate predictions across folds. 6. Save the aggregated predictions to a FITS file. Returns: None """ splits = self.folds_controller.splits fold_assignments = self.folds_controller.fold_assignments aggregated_predictions = None for fold_index, split in enumerate(splits): self.logger.info( f"## Running training on fold {fold_index + 1}/{len(splits)}\n" ) train_split, valid_split, test_split = split self.data.update_configs_for_fold( fold_assignments, train_split, valid_split, test_split ) trainer_config = copy.deepcopy(self.trainer) trainer_config.update( { "run_name": f"{self.run_name}_fold_{fold_index}", "name": f"trainer_fold_{fold_index}", "train_dataset": self.data.trainset, "val_dataset": self.data.validset, "test_dataset": self.data.testset, "model": self.model, } ) training_results = self.safe_train(trainer_config) if self.inference_source is not None: fold_predictions = self.run_inference( training_results["final_model_path"], self.data.testset ) if aggregated_predictions is None: aggregated_predictions = fold_predictions else: aggregated_predictions += fold_predictions if aggregated_predictions is not None: self.logger.info( "Final aggregated predictions saved to aggregated_predictions.fits" ) fits.writeto( Path(self.train_output_dir) / "aggregated_predictions.fits", data=aggregated_predictions, header=fits.getheader(self.inference_source), overwrite=True, )
[docs] def run_inference( self, model_snapshot: str, test_config: dict[str, Any] ) -> Optional[Any]: """ Runs inference using the trained model snapshot on the test dataset. Args: model_snapshot (str): Path to the trained model snapshot. test_config (dict[str, Any]): Configuration for the test dataset. Returns: Optional[Any]: Inference results, if available. """ inference_config = { "model_snapshot": model_snapshot, "source": self.inference_source, "dataset": test_config, } segmenter = Segmenter.from_config(inference_config) results = segmenter.segment() return results
[docs] def safe_train(self, trainer_config): """ Execute training in a safe environment with optional multi-GPU support. Only returns the result from the main GPU process. Args: trainer_config (dict): Trainer configuration. Returns: main_result: Result from the main GPU training process. """ import torch.multiprocessing as mp mp.set_start_method("spawn", force=True) if self.gpus is not None and len(self.gpus) > 1: world_size = len(self.gpus) self.logger.info(f"Starting parallel training on {world_size} GPU(s).") ctx = mp.get_context("spawn") result_queue = ctx.Queue() processes = [] for idx, gpu in enumerate(self.gpus): self.logger.debug(f"Launching training process on GPU {gpu}.") p = ctx.Process( target=train_process, args=(idx, world_size, result_queue, trainer_config, self.gpus), ) p.start() processes.append(p) main_result = None if not result_queue.empty(): main_result = result_queue.get() for p in processes: p.join() self.logger.debug(f"Process {p.pid} has finished.") self.logger.info("Parallel training completed.") return main_result else: trainer = Trainer.from_config( trainer_config, force_device=self.gpus[0] if self.gpus is not None else None, ) return trainer.train()
[docs] def finish(self): self.folds_controller.close() import gc import time import logging if hasattr(self, "logger"): self._close_logger() logging.shutdown() gc.collect() time.sleep(5)
def _close_logger(self): for h in self.logger.handlers[:]: try: h.flush() h.close() except Exception: pass self.logger.removeHandler(h)
class GridSearchPipeline(KfoldsTrainingPipeline): """ Configuration: - **run_name** (str): The name of the training run. - **inference_source** (Union[Path, str], optional): The path to the inference data source. - **train_output_dir** (Union[Path, str]): The directory where training outputs are saved. - **nb_folds** (int): The number of folds for cross-validation. - **data** (Config): The dataset configuration. - **gridsearch** (Config): The gridsearch trainer configuration. - **trainer** (Config): The trainer configuration. - **model** (Config): The model configuration. - **gpus** (Union[int, list[int], str], optional): The GPU IDs to use for training. If 'auto', the pipeline will use all available GPUs. """ aliases = ["gridsearch_pipeline"] config_schema = { **KfoldsTrainingPipeline.config_schema, "gridsearch": Schema(type=Config), } def run_training(self) -> None: splits = self.folds_controller.splits fold_assignments = self.folds_controller.fold_assignments for fold_index, split in enumerate(splits): self.logger.info( f"## Running training on fold {fold_index + 1}/{len(splits)}\n" ) train_split, valid_split, test_split = split self.data.update_configs_for_fold( fold_assignments, train_split, valid_split, test_split ) gridsearch_trainer = self.gridsearch["trainer"] trainer_config = copy.deepcopy(self.trainer) list_lr = gridsearch_trainer.get("optimizer", {}).get( "lr", [trainer_config["optimizer"]["lr"]] ) list_batch_size = gridsearch_trainer.get("batch_size", [256]) for lr in list_lr: for batch_size in list_batch_size: opt_config = trainer_config["optimizer"].copy() opt_config["lr"] = lr trainer_config.update( { "run_name": f"{self.run_name}_fold_{fold_index}_{lr=}_{batch_size=}", "name": f"trainer_grid_fold_{fold_index}_{lr=}_{batch_size=}", "train_dataset": self.data.trainset, "val_dataset": self.data.validset, "test_dataset": self.data.testset, "model": self.model, "batch_size": batch_size, "optimizer": opt_config, } ) training_results = self.safe_train(trainer_config) self.logger.info( f"Training results for configuration {lr=}, {batch_size=}:" ) pprint(training_results) # region Utils class DataConfig(Configurable): """ Configuration for the datasets used in the training pipeline. Attributes: dataset_path (Union[Path, str]): Path to the dataset. type (str): Type of the dataset. controller (dict[str, Any]): Configuration for the folds controller. trainset (dict[str, Any]): Configuration for the training dataset. validset (dict[str, Any]): Configuration for the validation dataset. testset (dict[str, Any]): Configuration for the test dataset. Methods: preconditions(): Checks preconditions for the dataset configuration. parse_datasets_config(): Parses the dataset configuration for training, validation, and testing. update_configs_for_fold(): Updates the configurations for a specific fold. """ config_schema = { "dataset_path": Schema(Union[Path, str]), "type": Schema(str), "controller": Schema(type=Config), "trainset": Schema(type=Config), "validset": Schema(type=Config), "testset": Schema(type=Config), } def preconditions(self) -> None: """ Checks preconditions for the dataset configuration. """ assert Path( self.dataset_path ).exists(), f"Dataset path {self.dataset_path} does not exist" def parse_datasets_config( self, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any]]: """ Parses the dataset configuration for training, validation, and testing. Returns: tuple[dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any]]: Parsed configurations for the folds controller, train, validation, and test datasets. """ train_config = self.trainset valid_config = self.validset test_config = self.testset folds_controller_config = self.controller dataset_type = self.type dataset_path = self.dataset_path for config in [train_config, valid_config, test_config]: config["type"] = dataset_type config["dataset_path"] = dataset_path folds_controller_config["dataset_path"] = dataset_path return folds_controller_config, train_config, valid_config, test_config def update_configs_for_fold( self, fold_assignments: dict[str, Any], train_split: Any, valid_split: Any, test_split: Any, ) -> None: """ Updates the configurations for a specific fold. Args: fold_assignments (dict[str, Any]): Fold assignments. train_split (Any): Training split. valid_split (Any): Validation split. test_split (Any): Test split. """ for dataset, split in zip( [self.trainset, self.validset, self.testset], [train_split, valid_split, test_split], ): dataset.update( { "fold_assignments": fold_assignments, "fold_list": split, } ) def train_process(rank, world_size, result_queue, trainer_config, gpus): """ Train process function. Args: rank (int): Rank of the process. world_size (int): Total number of processes. result_queue (Queue): Queue to store the training result. trainer_config (dict): Trainer configuration. gpus (list): List of GPU IDs. """ setup(rank, world_size) trainer = Trainer.from_config( trainer_config, force_device=f"cuda:{gpus[rank]}", multi_gpu=True ) synchronize() result = trainer.train() result_queue.put(result) synchronize() cleanup() # endregion