Source code for lis2.utils.distributed

import os

from torch import distributed as dist


[docs] def get_rank(): """ Get the rank of the current process. Returns: str: 'cpu' if CUDA is not available, 'cuda' if there is a single GPU, or 'cuda:{rank}' for distributed GPUs. """ if not torch.cuda.is_available(): return "cpu" if torch.cuda.device_count() < 2: return "cuda" return "cuda:" + str(dist.get_rank())
[docs] def get_rank_num(): """ Get the rank number of the current process. Returns: int: 0 if CUDA is not available or there is a single GPU, or the rank number for distributed GPUs. """ if not torch.cuda.is_available(): return 0 if torch.cuda.device_count() < 2: return 0 return dist.get_rank()
[docs] def is_main_gpu(): """ Check if the current process is running on the main GPU. Returns: bool: True if CUDA is not available or there is a single GPU, or if the current process is the main process in distributed training. """ if not torch.cuda.is_available(): return True if torch.cuda.device_count() < 2: return True return dist.get_rank() == 0
[docs] def synchronize(): """ Synchronize all processes in distributed training. This function is a barrier that ensures all processes reach this point before proceeding. """ if not dist.is_available(): return if not dist.is_initialized(): return world_size = dist.get_world_size() if world_size == 1: return dist.barrier()
[docs] def get_world_size(): """ Get the total number of processes in distributed training. Returns: int: 1 if CUDA is not available or not in distributed mode, or the total number of processes in distributed training. """ if not dist.is_available(): return 1 if not dist.is_initialized(): return 1 return dist.get_world_size()
[docs] def reduce_sum(value): """ Perform distributed sum reduction on the input tensor. Args: tensor (torch.Tensor): Input tensor to be summed across all processes. Returns: torch.Tensor: Resulting tensor after the sum reduction. """ if not dist.is_available(): return value if not dist.is_initialized(): return value if not isinstance(value, torch.Tensor): tensor = torch.tensor(value) tensor = value.clone() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tensor = tensor.to(device) dist.all_reduce(tensor, op=dist.ReduceOp.SUM) return tensor
import socket
[docs] def find_free_port(): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("", 0)) return s.getsockname()[1]
[docs] def setup(rank: int, world_size: int) -> None: """Initializes the distributed process group.""" try: import idr_torch os.environ["MASTER_ADDR"] = str(idr_torch.master_addr) os.environ["MASTER_PORT"] = str(idr_torch.master_port) except ImportError: os.environ["MASTER_ADDR"] = "localhost" os.environ["CUDA_LAUNCH_BLOCKING"] = "1" if not dist.is_initialized(): dist.init_process_group( backend="nccl", init_method="env://", rank=rank, world_size=world_size )
[docs] def cleanup(): synchronize() dist.destroy_process_group()
import torch
[docs] def get_device_ids(gpus): """ Returns the appropriate list of GPU IDs or 'cpu' based on the input argument. Args: gpus (int, list of int, str): - An integer representing a single GPU ID. - A list of integers representing multiple GPU IDs. - The string 'auto' to use all available GPUs. Returns: list of int or str: List of GPU IDs or 'cpu' if no GPUs are available. Raises: ValueError: If the `gpus` argument is not valid (not an int, list of ints, or 'auto'). IndexError: If the GPU ID in the list exceeds the available GPU count. """ if gpus == "auto": if torch.cuda.is_available(): if "CUDA_VISIBLE_DEVICES" in os.environ: visible_devices = os.environ["CUDA_VISIBLE_DEVICES"].split(",") return [int(dev) for dev in visible_devices if dev.isdigit()] else: return list(range(torch.cuda.device_count())) else: return ["cpu"] elif isinstance(gpus, int): if gpus < 0: raise ValueError("GPU ID cannot be negative.") if torch.cuda.is_available() and gpus <= torch.cuda.device_count(): os.environ["CUDA_VISIBLE_DEVICES"] = gpus return [gpus] else: return ["cpu"] elif isinstance(gpus, list): if not all(isinstance(g, int) for g in gpus): raise ValueError("All elements in the GPU list must be integers.") if any(g < 0 for g in gpus): raise ValueError("GPU IDs cannot be negative.") if torch.cuda.is_available() and all( g < torch.cuda.device_count() for g in gpus ): os.environ["CUDA_VISIBLE_DEVICES"] = gpus return gpus else: return ["cpu"] else: raise ValueError( "Invalid argument for `gpus`. Must be an integer, a list of integers, or 'auto'." )