"""Custom inference handler for HuggingFace Inference Endpoints.""" import base64 import binascii import os from pathlib import Path from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from .alignment import QwenForcedAligner from .asr_modeling import ASRModel from .asr_pipeline import ASRPipeline from .diarization import NemotronDiarizer from .diarization import get_device as _best_device else: try: # For remote execution, imports are relative from .alignment import QwenForcedAligner from .asr_modeling import ASRModel from .asr_pipeline import ASRPipeline from .diarization import NemotronDiarizer from .diarization import get_device as _best_device except ImportError: # For local execution, imports are not relative from alignment import QwenForcedAligner from asr_modeling import ASRModel from asr_pipeline import ASRPipeline from diarization import NemotronDiarizer from diarization import get_device as _best_device def decode_inputs(inputs: Any) -> Any: """A JSON request's base64 audio as bytes; paths, raw bytes and arrays unchanged. The toolkit hands `inputs` over as raw bytes for an `audio/*` body, but a JSON body can only carry audio as a base64 string, which the pipeline would otherwise read as a file path. """ if not isinstance(inputs, str) or Path(inputs).exists(): return inputs try: return base64.b64decode(inputs, validate=True) except (binascii.Error, ValueError): return inputs class EndpointHandler: """HuggingFace Inference Endpoints handler for ASR model. Handles model loading, warmup, and inference requests for deployment on HuggingFace Inference Endpoints or similar services. """ def __init__(self, path: str = ""): """Initialize the endpoint handler. Args: path: Path to model directory or HuggingFace model ID """ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # `ASRModel.from_pretrained` constructs its own submodules and forwards # **kwargs to `ASRModel.__init__`, which discards them -- no loader # reads `device_map`, `torch_dtype` or `low_cpu_mem_usage`, and it sets # `_is_loading_from_pretrained` precisely to keep `device_map="auto"` # out of the sub-model loaders. Passing them here did nothing, so the # model stayed on CPU and a GPU Inference Endpoint silently decoded on # CPU. Place it explicitly instead. dtype comes from # `config.model_dtype` and the attention backend from # `config.attn_implementation`, which already downgrades FA2 when # flash_attn is missing. self.model = ASRModel.from_pretrained(path) self.device = _best_device() # PreTrainedModel.to is functools.wraps'd, which pyright cannot bind as a method. self.model.to(self.device) # pyright: ignore[reportArgumentType] self.model.eval() self.pipe = ASRPipeline( model=self.model, feature_extractor=self.model.feature_extractor, tokenizer=self.model.tokenizer, device=self.device, ) # Load the forced aligner (return_timestamps) and Nemotron diarizer # (return_speakers) at boot, as the Spaces demo does, so the first # request that asks for them doesn't pay the download and load. QwenForcedAligner.get_instance() NemotronDiarizer.get_instance() def __call__(self, data: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]: """Process an inference request. Args: data: Request data containing 'inputs' (audio path, bytes or base64 string) and optional 'parameters' for the pipeline, e.g. `return_timestamps`, `return_speakers`, `num_speakers`, `max_speakers` -- the same options as the Spaces demo Returns: Transcription result with 'text' key; 'words' with timestamps, plus speaker labels and 'speaker_segments' when diarizing """ inputs = data.get("inputs") if inputs is None: msg = "Missing 'inputs' in request data" raise ValueError(msg) # Pass through any parameters from request, let model config provide defaults params = data.get("parameters", {}) result: dict[str, Any] | list[dict[str, Any]] = self.pipe(decode_inputs(inputs), **params) return result