audio_utils.py 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237
  1. # Copyright 2023 The HuggingFace Inc. team and the librosa & torchaudio authors.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. """
  15. Audio processing functions to extract features from audio waveforms. This code is pure numpy to support all frameworks
  16. and remove unnecessary dependencies.
  17. """
  18. import base64
  19. import importlib
  20. import io
  21. import os
  22. import warnings
  23. from collections.abc import Sequence
  24. from io import BytesIO
  25. from typing import TYPE_CHECKING, Any, Union
  26. import httpx
  27. import numpy as np
  28. from packaging import version
  29. from .utils import (
  30. is_librosa_available,
  31. is_numpy_array,
  32. is_soundfile_available,
  33. is_torch_tensor,
  34. is_torchcodec_available,
  35. requires_backends,
  36. )
  37. if TYPE_CHECKING:
  38. import torch
  39. if is_soundfile_available():
  40. import soundfile as sf
  41. if is_librosa_available():
  42. import librosa
  43. # TODO: @eustlb, we actually don't need librosa but soxr is installed with librosa
  44. import soxr
  45. if is_torchcodec_available():
  46. TORCHCODEC_VERSION = version.parse(importlib.metadata.version("torchcodec"))
  47. AudioInput = Union[np.ndarray, "torch.Tensor", Sequence[np.ndarray], Sequence["torch.Tensor"]]
  48. def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None) -> np.ndarray:
  49. """
  50. Loads `audio` to an np.ndarray object.
  51. Args:
  52. audio (`str` or `np.ndarray`):
  53. The audio to be loaded to the numpy array format.
  54. sampling_rate (`int`, *optional*, defaults to 16000):
  55. The sampling rate to be used when loading the audio. It should be same as the
  56. sampling rate the model you will be using further was trained with.
  57. timeout (`float`, *optional*):
  58. The timeout value in seconds for the URL request.
  59. Returns:
  60. `np.ndarray`: A numpy array representing the audio.
  61. """
  62. if isinstance(audio, str):
  63. # Try to load with `torchcodec` but do not enforce users to install it. If not found
  64. # fallback to `librosa`. If using an audio-only model, most probably `torchcodec` won't be
  65. # needed. Do not raise any errors if not installed or versions do not match
  66. if is_torchcodec_available() and version.parse("0.3.0") <= TORCHCODEC_VERSION:
  67. audio = load_audio_torchcodec(audio, sampling_rate=sampling_rate)
  68. else:
  69. audio = load_audio_librosa(audio, sampling_rate=sampling_rate, timeout=timeout)
  70. elif not isinstance(audio, np.ndarray):
  71. raise TypeError(
  72. "Incorrect format used for `audio`. Should be an url linking to an audio, a local path, or numpy array."
  73. )
  74. return audio
  75. def load_audio_torchcodec(audio: str | np.ndarray, sampling_rate=16000) -> np.ndarray:
  76. """
  77. Loads `audio` to an np.ndarray object using `torchcodec`.
  78. Args:
  79. audio (`str` or `np.ndarray`):
  80. The audio to be loaded to the numpy array format.
  81. sampling_rate (`int`, *optional*, defaults to 16000):
  82. The sampling rate to be used when loading the audio. It should be same as the
  83. sampling rate the model you will be using further was trained with.
  84. Returns:
  85. `np.ndarray`: A numpy array representing the audio.
  86. """
  87. # Lazy import so that issues in torchcodec compatibility don't crash the whole library
  88. requires_backends(load_audio_torchcodec, ["torchcodec"])
  89. from torchcodec.decoders import AudioDecoder
  90. # Set `num_channels` to `1` which is what most models expects and the default in librosa
  91. decoder = AudioDecoder(audio, sample_rate=sampling_rate, num_channels=1)
  92. audio = decoder.get_all_samples().data[0].numpy() # NOTE: feature extractors don't accept torch tensors
  93. return audio
  94. def load_audio_librosa(audio: str | np.ndarray, sampling_rate=16000, timeout=None) -> np.ndarray:
  95. """
  96. Loads `audio` to an np.ndarray object using `librosa`.
  97. Args:
  98. audio (`str` or `np.ndarray`):
  99. The audio to be loaded to the numpy array format.
  100. sampling_rate (`int`, *optional*, defaults to 16000):
  101. The sampling rate to be used when loading the audio. It should be same as the
  102. sampling rate the model you will be using further was trained with.
  103. timeout (`float`, *optional*):
  104. The timeout value in seconds for the URL request.
  105. Returns:
  106. `np.ndarray`: A numpy array representing the audio.
  107. """
  108. requires_backends(load_audio_librosa, ["librosa"])
  109. # Load audio from URL (e.g https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/translate_to_chinese.wav)
  110. if audio.startswith("http://") or audio.startswith("https://"):
  111. audio = librosa.load(
  112. BytesIO(httpx.get(audio, follow_redirects=True, timeout=timeout).content), sr=sampling_rate
  113. )[0]
  114. elif os.path.isfile(audio):
  115. audio = librosa.load(audio, sr=sampling_rate)[0]
  116. return audio
  117. def load_audio_as(
  118. audio: str,
  119. return_format: str,
  120. timeout: int | None = None,
  121. force_mono: bool = False,
  122. sampling_rate: int | None = None,
  123. ) -> str | dict[str, Any] | io.BytesIO | None:
  124. """
  125. Load audio from either a local file path or URL and return in specified format.
  126. Args:
  127. audio (`str`): Either a local file path or a URL to an audio file
  128. return_format (`str`): Format to return the audio in:
  129. - "base64": Base64 encoded string
  130. - "dict": Dictionary with data and format
  131. - "buffer": BytesIO object
  132. timeout (`int`, *optional*): Timeout for URL requests in seconds
  133. force_mono (`bool`): Whether to convert stereo audio to mono
  134. sampling_rate (`int`, *optional*): If provided, the audio will be resampled to the specified sampling rate.
  135. Returns:
  136. `Union[str, Dict[str, Any], io.BytesIO, None]`:
  137. - `str`: Base64 encoded audio data (if return_format="base64")
  138. - `dict`: Dictionary with 'data' (base64 encoded audio data) and 'format' keys (if return_format="dict")
  139. - `io.BytesIO`: BytesIO object containing audio data (if return_format="buffer")
  140. """
  141. requires_backends(load_audio_as, ["librosa"])
  142. if return_format not in ["base64", "dict", "buffer"]:
  143. raise ValueError(f"Invalid return_format: {return_format}. Must be 'base64', 'dict', or 'buffer'")
  144. try:
  145. # Load audio bytes from URL or file
  146. audio_bytes = None
  147. if audio.startswith(("http://", "https://")):
  148. response = httpx.get(audio, follow_redirects=True, timeout=timeout)
  149. response.raise_for_status()
  150. audio_bytes = response.content
  151. elif os.path.isfile(audio):
  152. with open(audio, "rb") as audio_file:
  153. audio_bytes = audio_file.read()
  154. else:
  155. raise ValueError(f"File not found: {audio}")
  156. # Process audio data
  157. with io.BytesIO(audio_bytes) as audio_file:
  158. with sf.SoundFile(audio_file) as f:
  159. audio_array = f.read(dtype="float32")
  160. original_sr = f.samplerate
  161. audio_format = f.format
  162. if sampling_rate is not None and sampling_rate != original_sr:
  163. # Resample audio to target sampling rate
  164. audio_array = soxr.resample(audio_array, original_sr, sampling_rate, quality="HQ")
  165. else:
  166. sampling_rate = original_sr
  167. # Convert to mono if needed
  168. if force_mono and audio_array.ndim != 1:
  169. audio_array = audio_array.mean(axis=1)
  170. buffer = io.BytesIO()
  171. sf.write(buffer, audio_array, sampling_rate, format=audio_format.upper())
  172. buffer.seek(0)
  173. if return_format == "buffer":
  174. return buffer
  175. elif return_format == "base64":
  176. return base64.b64encode(buffer.read()).decode("utf-8")
  177. elif return_format == "dict":
  178. return {
  179. "data": base64.b64encode(buffer.read()).decode("utf-8"),
  180. "format": audio_format.lower(),
  181. }
  182. except Exception as e:
  183. raise ValueError(f"Error loading audio: {e}")
  184. def conv1d_output_length(module: "torch.nn.Conv1d", input_length: int) -> int:
  185. """
  186. Computes the output length of a 1D convolution layer according to torch's documentation:
  187. https://docs.pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
  188. """
  189. return int(
  190. (input_length + 2 * module.padding[0] - module.dilation[0] * (module.kernel_size[0] - 1) - 1)
  191. / module.stride[0]
  192. + 1
  193. )
  194. def is_valid_audio(audio):
  195. return is_numpy_array(audio) or is_torch_tensor(audio)
  196. def is_valid_list_of_audio(audio):
  197. return audio and all(is_valid_audio(audio_i) for audio_i in audio)
  198. def make_list_of_audio(
  199. audio: list[AudioInput] | AudioInput,
  200. ) -> AudioInput:
  201. """
  202. Ensure that the output is a list of audio.
  203. Args:
  204. audio (`Union[list[AudioInput], AudioInput]`):
  205. The input audio.
  206. Returns:
  207. list: A list of audio.
  208. """
  209. # If it's a list of audios, it's already in the right format
  210. if isinstance(audio, (list, tuple)) and is_valid_list_of_audio(audio):
  211. return audio
  212. # If it's a single audio, convert it to a list of
  213. if is_valid_audio(audio):
  214. return [audio]
  215. raise ValueError("Invalid input type. Must be a single audio or a list of audio")
  216. def hertz_to_mel(freq: float | np.ndarray, mel_scale: str = "htk") -> float | np.ndarray:
  217. """
  218. Convert frequency from hertz to mels.
  219. Args:
  220. freq (`float` or `np.ndarray`):
  221. The frequency, or multiple frequencies, in hertz (Hz).
  222. mel_scale (`str`, *optional*, defaults to `"htk"`):
  223. The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
  224. Returns:
  225. `float` or `np.ndarray`: The frequencies on the mel scale.
  226. """
  227. if mel_scale not in ["slaney", "htk", "kaldi"]:
  228. raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
  229. if mel_scale == "htk":
  230. return 2595.0 * np.log10(1.0 + (freq / 700.0))
  231. elif mel_scale == "kaldi":
  232. return 1127.0 * np.log(1.0 + (freq / 700.0))
  233. min_log_hertz = 1000.0
  234. min_log_mel = 15.0
  235. logstep = 27.0 / np.log(6.4)
  236. mels = 3.0 * freq / 200.0
  237. if isinstance(freq, np.ndarray):
  238. log_region = freq >= min_log_hertz
  239. mels[log_region] = min_log_mel + np.log(freq[log_region] / min_log_hertz) * logstep
  240. elif freq >= min_log_hertz:
  241. mels = min_log_mel + np.log(freq / min_log_hertz) * logstep
  242. return mels
  243. def mel_to_hertz(mels: float | np.ndarray, mel_scale: str = "htk") -> float | np.ndarray:
  244. """
  245. Convert frequency from mels to hertz.
  246. Args:
  247. mels (`float` or `np.ndarray`):
  248. The frequency, or multiple frequencies, in mels.
  249. mel_scale (`str`, *optional*, `"htk"`):
  250. The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
  251. Returns:
  252. `float` or `np.ndarray`: The frequencies in hertz.
  253. """
  254. if mel_scale not in ["slaney", "htk", "kaldi"]:
  255. raise ValueError('mel_scale should be one of "htk", "slaney" or "kaldi".')
  256. if mel_scale == "htk":
  257. return 700.0 * (np.power(10, mels / 2595.0) - 1.0)
  258. elif mel_scale == "kaldi":
  259. return 700.0 * (np.exp(mels / 1127.0) - 1.0)
  260. min_log_hertz = 1000.0
  261. min_log_mel = 15.0
  262. logstep = np.log(6.4) / 27.0
  263. freq = 200.0 * mels / 3.0
  264. if isinstance(mels, np.ndarray):
  265. log_region = mels >= min_log_mel
  266. freq[log_region] = min_log_hertz * np.exp(logstep * (mels[log_region] - min_log_mel))
  267. elif mels >= min_log_mel:
  268. freq = min_log_hertz * np.exp(logstep * (mels - min_log_mel))
  269. return freq
  270. def hertz_to_octave(freq: float | np.ndarray, tuning: float = 0.0, bins_per_octave: int = 12):
  271. """
  272. Convert frequency from hertz to fractional octave numbers.
  273. Adapted from *librosa*.
  274. Args:
  275. freq (`float` or `np.ndarray`):
  276. The frequency, or multiple frequencies, in hertz (Hz).
  277. tuning (`float`, defaults to `0.`):
  278. Tuning deviation from the Stuttgart pitch (A440) in (fractional) bins per octave.
  279. bins_per_octave (`int`, defaults to `12`):
  280. Number of bins per octave.
  281. Returns:
  282. `float` or `np.ndarray`: The frequencies on the octave scale.
  283. """
  284. stuttgart_pitch = 440.0 * 2.0 ** (tuning / bins_per_octave)
  285. octave = np.log2(freq / (float(stuttgart_pitch) / 16))
  286. return octave
  287. def _create_triangular_filter_bank(fft_freqs: np.ndarray, filter_freqs: np.ndarray) -> np.ndarray:
  288. """
  289. Creates a triangular filter bank.
  290. Adapted from *torchaudio* and *librosa*.
  291. Args:
  292. fft_freqs (`np.ndarray` of shape `(num_frequency_bins,)`):
  293. Discrete frequencies of the FFT bins in Hz.
  294. filter_freqs (`np.ndarray` of shape `(num_mel_filters,)`):
  295. Center frequencies of the triangular filters to create, in Hz.
  296. Returns:
  297. `np.ndarray` of shape `(num_frequency_bins, num_mel_filters)`
  298. """
  299. filter_diff = np.diff(filter_freqs)
  300. slopes = np.expand_dims(filter_freqs, 0) - np.expand_dims(fft_freqs, 1)
  301. down_slopes = -slopes[:, :-2] / filter_diff[:-1]
  302. up_slopes = slopes[:, 2:] / filter_diff[1:]
  303. return np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes))
  304. def chroma_filter_bank(
  305. num_frequency_bins: int,
  306. num_chroma: int,
  307. sampling_rate: int,
  308. tuning: float = 0.0,
  309. power: float | None = 2.0,
  310. weighting_parameters: tuple[float, float] | None = (5.0, 2.0),
  311. start_at_c_chroma: bool = True,
  312. ):
  313. """
  314. Creates a chroma filter bank, i.e a linear transformation to project spectrogram bins onto chroma bins.
  315. Adapted from *librosa*.
  316. Args:
  317. num_frequency_bins (`int`):
  318. Number of frequencies used to compute the spectrogram (should be the same as in `stft`).
  319. num_chroma (`int`):
  320. Number of chroma bins (i.e pitch classes).
  321. sampling_rate (`float`):
  322. Sample rate of the audio waveform.
  323. tuning (`float`):
  324. Tuning deviation from A440 in fractions of a chroma bin.
  325. power (`float`, *optional*, defaults to 2.0):
  326. If 12.0, normalizes each column with their L2 norm. If 1.0, normalizes each column with their L1 norm.
  327. weighting_parameters (`tuple[float, float]`, *optional*, defaults to `(5., 2.)`):
  328. If specified, apply a Gaussian weighting parameterized by the first element of the tuple being the center and
  329. the second element being the Gaussian half-width.
  330. start_at_c_chroma (`bool`, *optional*, defaults to `True`):
  331. If True, the filter bank will start at the 'C' pitch class. Otherwise, it will start at 'A'.
  332. Returns:
  333. `np.ndarray` of shape `(num_frequency_bins, num_chroma)`
  334. """
  335. # Get the FFT bins, not counting the DC component
  336. frequencies = np.linspace(0, sampling_rate, num_frequency_bins, endpoint=False)[1:]
  337. freq_bins = num_chroma * hertz_to_octave(frequencies, tuning=tuning, bins_per_octave=num_chroma)
  338. # make up a value for the 0 Hz bin = 1.5 octaves below bin 1
  339. # (so chroma is 50% rotated from bin 1, and bin width is broad)
  340. freq_bins = np.concatenate(([freq_bins[0] - 1.5 * num_chroma], freq_bins))
  341. bins_width = np.concatenate((np.maximum(freq_bins[1:] - freq_bins[:-1], 1.0), [1]))
  342. chroma_filters = np.subtract.outer(freq_bins, np.arange(0, num_chroma, dtype="d")).T
  343. num_chroma2 = np.round(float(num_chroma) / 2)
  344. # Project into range -num_chroma/2 .. num_chroma/2
  345. # add on fixed offset of 10*num_chroma to ensure all values passed to
  346. # rem are positive
  347. chroma_filters = np.remainder(chroma_filters + num_chroma2 + 10 * num_chroma, num_chroma) - num_chroma2
  348. # Gaussian bumps - 2*D to make them narrower
  349. chroma_filters = np.exp(-0.5 * (2 * chroma_filters / np.tile(bins_width, (num_chroma, 1))) ** 2)
  350. # normalize each column
  351. if power is not None:
  352. chroma_filters = chroma_filters / np.sum(chroma_filters**power, axis=0, keepdims=True) ** (1.0 / power)
  353. # Maybe apply scaling for fft bins
  354. if weighting_parameters is not None:
  355. center, half_width = weighting_parameters
  356. chroma_filters *= np.tile(
  357. np.exp(-0.5 * (((freq_bins / num_chroma - center) / half_width) ** 2)),
  358. (num_chroma, 1),
  359. )
  360. if start_at_c_chroma:
  361. chroma_filters = np.roll(chroma_filters, -3 * (num_chroma // 12), axis=0)
  362. # remove aliasing columns, copy to ensure row-contiguity
  363. return np.ascontiguousarray(chroma_filters[:, : int(1 + num_frequency_bins / 2)])
  364. def mel_filter_bank(
  365. num_frequency_bins: int,
  366. num_mel_filters: int,
  367. min_frequency: float,
  368. max_frequency: float,
  369. sampling_rate: int,
  370. norm: str | None = None,
  371. mel_scale: str = "htk",
  372. triangularize_in_mel_space: bool = False,
  373. ) -> np.ndarray:
  374. """
  375. Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and
  376. various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters
  377. are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these
  378. features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.
  379. Different banks of mel filters were introduced in the literature. The following variations are supported:
  380. - MFCC FB-20: introduced in 1980 by Davis and Mermelstein, it assumes a sampling frequency of 10 kHz and a speech
  381. bandwidth of `[0, 4600]` Hz.
  382. - MFCC FB-24 HTK: from the Cambridge HMM Toolkit (HTK) (1995) uses a filter bank of 24 filters for a speech
  383. bandwidth of `[0, 8000]` Hz. This assumes sampling rate ≥ 16 kHz.
  384. - MFCC FB-40: from the Auditory Toolbox for MATLAB written by Slaney in 1998, assumes a sampling rate of 16 kHz and
  385. speech bandwidth of `[133, 6854]` Hz. This version also includes area normalization.
  386. - HFCC-E FB-29 (Human Factor Cepstral Coefficients) of Skowronski and Harris (2004), assumes a sampling rate of
  387. 12.5 kHz and speech bandwidth of `[0, 6250]` Hz.
  388. This code is adapted from *torchaudio* and *librosa*. Note that the default parameters of torchaudio's
  389. `melscale_fbanks` implement the `"htk"` filters while librosa uses the `"slaney"` implementation.
  390. Args:
  391. num_frequency_bins (`int`):
  392. Number of frequency bins (should be the same as `n_fft // 2 + 1` where `n_fft` is the size of the Fourier Transform used to compute the spectrogram).
  393. num_mel_filters (`int`):
  394. Number of mel filters to generate.
  395. min_frequency (`float`):
  396. Lowest frequency of interest in Hz.
  397. max_frequency (`float`):
  398. Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.
  399. sampling_rate (`int`):
  400. Sample rate of the audio waveform.
  401. norm (`str`, *optional*):
  402. If `"slaney"`, divide the triangular mel weights by the width of the mel band (area normalization).
  403. mel_scale (`str`, *optional*, defaults to `"htk"`):
  404. The mel frequency scale to use, `"htk"`, `"kaldi"` or `"slaney"`.
  405. triangularize_in_mel_space (`bool`, *optional*, defaults to `False`):
  406. If this option is enabled, the triangular filter is applied in mel space rather than frequency space. This
  407. should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.
  408. Returns:
  409. `np.ndarray` of shape (`num_frequency_bins`, `num_mel_filters`): Triangular filter bank matrix. This is a
  410. projection matrix to go from a spectrogram to a mel spectrogram.
  411. """
  412. if norm is not None and norm != "slaney":
  413. raise ValueError('norm must be one of None or "slaney"')
  414. if num_frequency_bins < 2:
  415. raise ValueError(f"Require num_frequency_bins: {num_frequency_bins} >= 2")
  416. if min_frequency > max_frequency:
  417. raise ValueError(f"Require min_frequency: {min_frequency} <= max_frequency: {max_frequency}")
  418. # center points of the triangular mel filters
  419. mel_min = hertz_to_mel(min_frequency, mel_scale=mel_scale)
  420. mel_max = hertz_to_mel(max_frequency, mel_scale=mel_scale)
  421. mel_freqs = np.linspace(mel_min, mel_max, num_mel_filters + 2)
  422. filter_freqs = mel_to_hertz(mel_freqs, mel_scale=mel_scale)
  423. if triangularize_in_mel_space:
  424. # frequencies of FFT bins in Hz, but filters triangularized in mel space
  425. fft_bin_width = sampling_rate / ((num_frequency_bins - 1) * 2)
  426. fft_freqs = hertz_to_mel(fft_bin_width * np.arange(num_frequency_bins), mel_scale=mel_scale)
  427. filter_freqs = mel_freqs
  428. else:
  429. # frequencies of FFT bins in Hz
  430. fft_freqs = np.linspace(0, sampling_rate // 2, num_frequency_bins)
  431. mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs)
  432. if norm is not None and norm == "slaney":
  433. # Slaney-style mel is scaled to be approx constant energy per channel
  434. enorm = 2.0 / (filter_freqs[2 : num_mel_filters + 2] - filter_freqs[:num_mel_filters])
  435. mel_filters *= np.expand_dims(enorm, 0)
  436. if (mel_filters.max(axis=0) == 0.0).any():
  437. warnings.warn(
  438. "At least one mel filter has all zero values. "
  439. f"The value for `num_mel_filters` ({num_mel_filters}) may be set too high. "
  440. f"Or, the value for `num_frequency_bins` ({num_frequency_bins}) may be set too low."
  441. )
  442. return mel_filters
  443. def optimal_fft_length(window_length: int) -> int:
  444. """
  445. Finds the best FFT input size for a given `window_length`. This function takes a given window length and, if not
  446. already a power of two, rounds it up to the next power or two.
  447. The FFT algorithm works fastest when the length of the input is a power of two, which may be larger than the size
  448. of the window or analysis frame. For example, if the window is 400 samples, using an FFT input size of 512 samples
  449. is more optimal than an FFT size of 400 samples. Using a larger FFT size does not affect the detected frequencies,
  450. it simply gives a higher frequency resolution (i.e. the frequency bins are smaller).
  451. """
  452. return 2 ** int(np.ceil(np.log2(window_length)))
  453. def window_function(
  454. window_length: int,
  455. name: str = "hann",
  456. periodic: bool = True,
  457. frame_length: int | None = None,
  458. center: bool = True,
  459. ) -> np.ndarray:
  460. """
  461. Returns an array containing the specified window. This window is intended to be used with `stft`.
  462. The following window types are supported:
  463. - `"boxcar"`: a rectangular window
  464. - `"hamming"`: the Hamming window
  465. - `"hann"`: the Hann window
  466. - `"povey"`: the Povey window
  467. Args:
  468. window_length (`int`):
  469. The length of the window in samples.
  470. name (`str`, *optional*, defaults to `"hann"`):
  471. The name of the window function.
  472. periodic (`bool`, *optional*, defaults to `True`):
  473. Whether the window is periodic or symmetric.
  474. frame_length (`int`, *optional*):
  475. The length of the analysis frames in samples. Provide a value for `frame_length` if the window is smaller
  476. than the frame length, so that it will be zero-padded.
  477. center (`bool`, *optional*, defaults to `True`):
  478. Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.
  479. Returns:
  480. `np.ndarray` of shape `(window_length,)` or `(frame_length,)` containing the window.
  481. """
  482. length = window_length + 1 if periodic else window_length
  483. if name == "boxcar":
  484. window = np.ones(length)
  485. elif name in ["hamming", "hamming_window"]:
  486. window = np.hamming(length)
  487. elif name in ["hann", "hann_window"]:
  488. window = np.hanning(length)
  489. elif name == "povey":
  490. window = np.power(np.hanning(length), 0.85)
  491. else:
  492. raise ValueError(f"Unknown window function '{name}'")
  493. if periodic:
  494. window = window[:-1]
  495. if frame_length is None:
  496. return window
  497. if window_length > frame_length:
  498. raise ValueError(
  499. f"Length of the window ({window_length}) may not be larger than frame_length ({frame_length})"
  500. )
  501. padded_window = np.zeros(frame_length)
  502. offset = (frame_length - window_length) // 2 if center else 0
  503. padded_window[offset : offset + window_length] = window
  504. return padded_window
  505. # Note: This method processes a single waveform. For batch processing, use spectrogram_batch().
  506. def spectrogram(
  507. waveform: np.ndarray,
  508. window: np.ndarray,
  509. frame_length: int,
  510. hop_length: int,
  511. fft_length: int | None = None,
  512. power: float | None = 1.0,
  513. center: bool = True,
  514. pad_mode: str = "reflect",
  515. onesided: bool = True,
  516. dither: float = 0.0,
  517. preemphasis: float | None = None,
  518. mel_filters: np.ndarray | None = None,
  519. mel_floor: float = 1e-10,
  520. log_mel: str | None = None,
  521. reference: float = 1.0,
  522. min_value: float = 1e-10,
  523. db_range: float | None = None,
  524. remove_dc_offset: bool = False,
  525. dtype: np.dtype = np.float32,
  526. ) -> np.ndarray:
  527. """
  528. Calculates a spectrogram over one waveform using the Short-Time Fourier Transform.
  529. This function can create the following kinds of spectrograms:
  530. - amplitude spectrogram (`power = 1.0`)
  531. - power spectrogram (`power = 2.0`)
  532. - complex-valued spectrogram (`power = None`)
  533. - log spectrogram (use `log_mel` argument)
  534. - mel spectrogram (provide `mel_filters`)
  535. - log-mel spectrogram (provide `mel_filters` and `log_mel`)
  536. How this works:
  537. 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length
  538. - hop_length` samples.
  539. 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`.
  540. 3. The DFT is taken of each windowed frame.
  541. 4. The results are stacked into a spectrogram.
  542. We make a distinction between the following "blocks" of sample data, each of which may have a different lengths:
  543. - The analysis frame. This is the size of the time slices that the input waveform is split into.
  544. - The window. Each analysis frame is multiplied by the window to avoid spectral leakage.
  545. - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram.
  546. In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A
  547. padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame,
  548. typically the next power of two.
  549. Note: This function is not optimized for speed yet. It should be mostly compatible with `librosa.stft` and
  550. `torchaudio.functional.transforms.Spectrogram`, although it is more flexible due to the different ways spectrograms
  551. can be constructed.
  552. Args:
  553. waveform (`np.ndarray` of shape `(length,)`):
  554. The input waveform. This must be a single real-valued, mono waveform.
  555. window (`np.ndarray` of shape `(frame_length,)`):
  556. The windowing function to apply, including zero-padding if necessary. The actual window length may be
  557. shorter than `frame_length`, but we're assuming the array has already been zero-padded.
  558. frame_length (`int`):
  559. The length of the analysis frames in samples. With librosa this is always equal to `fft_length` but we also
  560. allow smaller sizes.
  561. hop_length (`int`):
  562. The stride between successive analysis frames in samples.
  563. fft_length (`int`, *optional*):
  564. The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have.
  565. For optimal speed, this should be a power of two. If `None`, uses `frame_length`.
  566. power (`float`, *optional*, defaults to 1.0):
  567. If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `None`, returns
  568. complex numbers.
  569. center (`bool`, *optional*, defaults to `True`):
  570. Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `False`, frame
  571. `t` will start at time `t * hop_length`.
  572. pad_mode (`str`, *optional*, defaults to `"reflect"`):
  573. Padding mode used when `center` is `True`. Possible values are: `"constant"` (pad with zeros), `"edge"`
  574. (pad with edge values), `"reflect"` (pads with mirrored values).
  575. onesided (`bool`, *optional*, defaults to `True`):
  576. If True, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1`
  577. frequency bins. If False, also computes the negative frequencies and returns `fft_length` frequency bins.
  578. dither (`float`, *optional*, defaults to 0.0):
  579. Adds dithering. In other words, adds a small Gaussian noise to each frame.
  580. E.g. use 4.0 to add dithering with a normal distribution centered
  581. around 0.0 with standard deviation 4.0, 0.0 means no dithering.
  582. Dithering has similar effect as `mel_floor`. It reduces the high log_mel_fbank
  583. values for signals with hard-zero sections, when VAD cutoff is present in the signal.
  584. preemphasis (`float`, *optional*)
  585. Coefficient for a low-pass filter that applies pre-emphasis before the DFT.
  586. mel_filters (`np.ndarray` of shape `(num_freq_bins, num_mel_filters)`, *optional*):
  587. The mel filter bank. If supplied, applies a this filter bank to create a mel spectrogram.
  588. mel_floor (`float`, *optional*, defaults to 1e-10):
  589. Minimum value of mel frequency banks.
  590. log_mel (`str`, *optional*):
  591. How to convert the spectrogram to log scale. Possible options are: `None` (don't convert), `"log"` (take
  592. the natural logarithm) `"log10"` (take the base-10 logarithm), `"dB"` (convert to decibels). Can only be
  593. used when `power` is not `None`.
  594. reference (`float`, *optional*, defaults to 1.0):
  595. Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
  596. the loudest part to 0 dB. Must be greater than zero.
  597. min_value (`float`, *optional*, defaults to `1e-10`):
  598. The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
  599. `log(0)`. For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an
  600. amplitude spectrogram, the value `1e-5` corresponds to -100 dB. Must be greater than zero.
  601. db_range (`float`, *optional*):
  602. Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
  603. peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
  604. remove_dc_offset (`bool`, *optional*):
  605. Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in
  606. order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters.
  607. dtype (`np.dtype`, *optional*, defaults to `np.float32`):
  608. Data type of the spectrogram tensor. If `power` is None, this argument is ignored and the dtype will be
  609. `np.complex64`.
  610. Returns:
  611. `nd.array` containing a spectrogram of shape `(num_frequency_bins, length)` for a regular spectrogram or shape
  612. `(num_mel_filters, length)` for a mel spectrogram.
  613. """
  614. window_length = len(window)
  615. if fft_length is None:
  616. fft_length = frame_length
  617. if frame_length > fft_length:
  618. raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})")
  619. if window_length != frame_length:
  620. raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})")
  621. if hop_length <= 0:
  622. raise ValueError("hop_length must be greater than zero")
  623. if waveform.ndim != 1:
  624. raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}")
  625. if np.iscomplexobj(waveform):
  626. raise ValueError("Complex-valued input waveforms are not currently supported")
  627. if power is None and mel_filters is not None:
  628. raise ValueError(
  629. "You have provided `mel_filters` but `power` is `None`. Mel spectrogram computation is not yet supported for complex-valued spectrogram."
  630. "Specify `power` to fix this issue."
  631. )
  632. # center pad the waveform
  633. if center:
  634. padding = [(int(frame_length // 2), int(frame_length // 2))]
  635. waveform = np.pad(waveform, padding, mode=pad_mode)
  636. # promote to float64, since np.fft uses float64 internally
  637. waveform = waveform.astype(np.float64)
  638. window = window.astype(np.float64)
  639. # split waveform into frames of frame_length size
  640. num_frames = int(1 + np.floor((waveform.size - frame_length) / hop_length))
  641. num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length
  642. spectrogram = np.empty((num_frames, num_frequency_bins), dtype=np.complex64)
  643. # rfft is faster than fft
  644. fft_func = np.fft.rfft if onesided else np.fft.fft
  645. buffer = np.zeros(fft_length)
  646. timestep = 0
  647. for frame_idx in range(num_frames):
  648. buffer[:frame_length] = waveform[timestep : timestep + frame_length]
  649. if dither != 0.0:
  650. buffer[:frame_length] += dither * np.random.randn(frame_length)
  651. if remove_dc_offset:
  652. buffer[:frame_length] = buffer[:frame_length] - buffer[:frame_length].mean()
  653. if preemphasis is not None:
  654. buffer[1:frame_length] -= preemphasis * buffer[: frame_length - 1]
  655. buffer[0] *= 1 - preemphasis
  656. buffer[:frame_length] *= window
  657. spectrogram[frame_idx] = fft_func(buffer)
  658. timestep += hop_length
  659. # note: ** is much faster than np.power
  660. if power is not None:
  661. spectrogram = np.abs(spectrogram, dtype=np.float64) ** power
  662. spectrogram = spectrogram.T
  663. if mel_filters is not None:
  664. spectrogram = np.maximum(mel_floor, np.dot(mel_filters.T, spectrogram))
  665. if power is not None and log_mel is not None:
  666. if log_mel == "log":
  667. spectrogram = np.log(spectrogram)
  668. elif log_mel == "log10":
  669. spectrogram = np.log10(spectrogram)
  670. elif log_mel == "dB":
  671. if power == 1.0:
  672. spectrogram = amplitude_to_db(spectrogram, reference, min_value, db_range)
  673. elif power == 2.0:
  674. spectrogram = power_to_db(spectrogram, reference, min_value, db_range)
  675. else:
  676. raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}")
  677. else:
  678. raise ValueError(f"Unknown log_mel option: {log_mel}")
  679. spectrogram = np.asarray(spectrogram, dtype)
  680. return spectrogram
  681. def spectrogram_batch(
  682. waveform_list: list[np.ndarray],
  683. window: np.ndarray,
  684. frame_length: int,
  685. hop_length: int,
  686. fft_length: int | None = None,
  687. power: float | None = 1.0,
  688. center: bool = True,
  689. pad_mode: str = "reflect",
  690. onesided: bool = True,
  691. dither: float = 0.0,
  692. preemphasis: float | None = None,
  693. mel_filters: np.ndarray | None = None,
  694. mel_floor: float = 1e-10,
  695. log_mel: str | None = None,
  696. reference: float = 1.0,
  697. min_value: float = 1e-10,
  698. db_range: float | None = None,
  699. remove_dc_offset: bool = False,
  700. dtype: np.dtype = np.float32,
  701. ) -> list[np.ndarray]:
  702. """
  703. Calculates spectrograms for a list of waveforms using the Short-Time Fourier Transform, optimized for batch processing.
  704. This function extends the capabilities of the `spectrogram` function to handle multiple waveforms efficiently by leveraging broadcasting.
  705. It supports generating various types of spectrograms:
  706. - amplitude spectrogram (`power = 1.0`)
  707. - power spectrogram (`power = 2.0`)
  708. - complex-valued spectrogram (`power = None`)
  709. - log spectrogram (use `log_mel` argument)
  710. - mel spectrogram (provide `mel_filters`)
  711. - log-mel spectrogram (provide `mel_filters` and `log_mel`)
  712. How this works:
  713. 1. The input waveform is split into frames of size `frame_length` that are partially overlapping by `frame_length
  714. - hop_length` samples.
  715. 2. Each frame is multiplied by the window and placed into a buffer of size `fft_length`.
  716. 3. The DFT is taken of each windowed frame.
  717. 4. The results are stacked into a spectrogram.
  718. We make a distinction between the following "blocks" of sample data, each of which may have a different lengths:
  719. - The analysis frame. This is the size of the time slices that the input waveform is split into.
  720. - The window. Each analysis frame is multiplied by the window to avoid spectral leakage.
  721. - The FFT input buffer. The length of this determines how many frequency bins are in the spectrogram.
  722. In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. A
  723. padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame,
  724. typically the next power of two.
  725. Note: This function is designed for efficient batch processing of multiple waveforms but retains compatibility with individual waveform processing methods like `librosa.stft`.
  726. Args:
  727. waveform_list (`list[np.ndarray]` with arrays of shape `(length,)`):
  728. The list of input waveforms, each a single-channel (mono) signal.
  729. window (`np.ndarray` of shape `(frame_length,)`):
  730. The windowing function to apply, including zero-padding if necessary.
  731. frame_length (`int`):
  732. The length of each frame for analysis.
  733. hop_length (`int`):
  734. The step size between successive frames.
  735. fft_length (`int`, *optional*):
  736. The size of the FFT buffer, defining frequency bin resolution.
  737. power (`float`, *optional*, defaults to 1.0):
  738. Determines the type of spectrogram: 1.0 for amplitude, 2.0 for power, None for complex.
  739. center (`bool`, *optional*, defaults to `True`):
  740. Whether to center-pad the waveform frames.
  741. pad_mode (`str`, *optional*, defaults to `"reflect"`):
  742. The padding strategy when `center` is `True`.
  743. onesided (`bool`, *optional*, defaults to `True`):
  744. If True, returns a one-sided spectrogram for real input signals.
  745. dither (`float`, *optional*, defaults to 0.0):
  746. Adds dithering. In other words, adds a small Gaussian noise to each frame.
  747. E.g. use 4.0 to add dithering with a normal distribution centered
  748. around 0.0 with standard deviation 4.0, 0.0 means no dithering.
  749. preemphasis (`float`, *optional*):
  750. Applies a pre-emphasis filter to each frame.
  751. mel_filters (`np.ndarray`, *optional*):
  752. Mel filter bank for converting to mel spectrogram.
  753. mel_floor (`float`, *optional*, defaults to 1e-10):
  754. Floor value for mel spectrogram to avoid log(0).
  755. log_mel (`str`, *optional*):
  756. Specifies log scaling strategy; options are None, "log", "log10", "dB".
  757. reference (`float`, *optional*, defaults to 1.0):
  758. Reference value for dB conversion in log_mel.
  759. min_value (`float`, *optional*, defaults to 1e-10):
  760. Minimum floor value for log scale conversions.
  761. db_range (`float`, *optional*):
  762. Dynamic range for dB scale spectrograms.
  763. remove_dc_offset (`bool`, *optional*):
  764. Whether to remove the DC offset from each frame.
  765. dtype (`np.dtype`, *optional*, defaults to `np.float32`):
  766. Data type of the output spectrogram.
  767. Returns:
  768. list[`np.ndarray`]: A list of spectrogram arrays, one for each input waveform.
  769. """
  770. window_length = len(window)
  771. if fft_length is None:
  772. fft_length = frame_length
  773. if frame_length > fft_length:
  774. raise ValueError(f"frame_length ({frame_length}) may not be larger than fft_length ({fft_length})")
  775. if window_length != frame_length:
  776. raise ValueError(f"Length of the window ({window_length}) must equal frame_length ({frame_length})")
  777. if hop_length <= 0:
  778. raise ValueError("hop_length must be greater than zero")
  779. # Check the dimensions of the waveform , and if waveform is complex
  780. for waveform in waveform_list:
  781. if waveform.ndim != 1:
  782. raise ValueError(f"Input waveform must have only one dimension, shape is {waveform.shape}")
  783. if np.iscomplexobj(waveform):
  784. raise ValueError("Complex-valued input waveforms are not currently supported")
  785. # Center pad the waveform
  786. if center:
  787. padding = [(int(frame_length // 2), int(frame_length // 2))]
  788. waveform_list = [
  789. np.pad(
  790. waveform,
  791. padding,
  792. mode=pad_mode,
  793. )
  794. for waveform in waveform_list
  795. ]
  796. original_waveform_lengths = [
  797. len(waveform) for waveform in waveform_list
  798. ] # these lengths will be used to remove padding later
  799. # Batch pad the waveform
  800. max_length = max(original_waveform_lengths)
  801. padded_waveform_batch = np.array(
  802. [
  803. np.pad(waveform, (0, max_length - len(waveform)), mode="constant", constant_values=0)
  804. for waveform in waveform_list
  805. ],
  806. dtype=dtype,
  807. )
  808. # Promote to float64, since np.fft uses float64 internally
  809. padded_waveform_batch = padded_waveform_batch.astype(np.float64)
  810. window = window.astype(np.float64)
  811. # Split waveform into frames of frame_length size
  812. num_frames = int(1 + np.floor((padded_waveform_batch.shape[1] - frame_length) / hop_length))
  813. # these lengths will be used to remove padding later
  814. true_num_frames = [int(1 + np.floor((length - frame_length) / hop_length)) for length in original_waveform_lengths]
  815. num_batches = padded_waveform_batch.shape[0]
  816. num_frequency_bins = (fft_length // 2) + 1 if onesided else fft_length
  817. spectrogram = np.empty((num_batches, num_frames, num_frequency_bins), dtype=np.complex64)
  818. # rfft is faster than fft
  819. fft_func = np.fft.rfft if onesided else np.fft.fft
  820. buffer = np.zeros((num_batches, fft_length))
  821. for frame_idx in range(num_frames):
  822. timestep = frame_idx * hop_length
  823. buffer[:, :frame_length] = padded_waveform_batch[:, timestep : timestep + frame_length]
  824. if dither != 0.0:
  825. buffer[:, :frame_length] += dither * np.random.randn(*buffer[:, :frame_length].shape)
  826. if remove_dc_offset:
  827. buffer[:, :frame_length] -= buffer[:, :frame_length].mean(axis=1, keepdims=True)
  828. if preemphasis is not None:
  829. buffer[:, 1:frame_length] -= preemphasis * buffer[:, : frame_length - 1]
  830. buffer[:, 0] *= 1 - preemphasis
  831. buffer[:, :frame_length] *= window
  832. spectrogram[:, frame_idx] = fft_func(buffer)
  833. # Note: ** is much faster than np.power
  834. if power is not None:
  835. spectrogram = np.abs(spectrogram, dtype=np.float64) ** power
  836. # Apply mel filters if provided
  837. if mel_filters is not None:
  838. result = np.tensordot(spectrogram, mel_filters.T, axes=([2], [1]))
  839. spectrogram = np.maximum(mel_floor, result)
  840. # Convert to log scale if specified
  841. if power is not None and log_mel is not None:
  842. if log_mel == "log":
  843. spectrogram = np.log(spectrogram)
  844. elif log_mel == "log10":
  845. spectrogram = np.log10(spectrogram)
  846. elif log_mel == "dB":
  847. if power == 1.0:
  848. spectrogram = amplitude_to_db_batch(spectrogram, reference, min_value, db_range)
  849. elif power == 2.0:
  850. spectrogram = power_to_db_batch(spectrogram, reference, min_value, db_range)
  851. else:
  852. raise ValueError(f"Cannot use log_mel option '{log_mel}' with power {power}")
  853. else:
  854. raise ValueError(f"Unknown log_mel option: {log_mel}")
  855. spectrogram = np.asarray(spectrogram, dtype)
  856. spectrogram_list = [spectrogram[i, : true_num_frames[i], :].T for i in range(len(true_num_frames))]
  857. return spectrogram_list
  858. def power_to_db(
  859. spectrogram: np.ndarray,
  860. reference: float = 1.0,
  861. min_value: float = 1e-10,
  862. db_range: float | None = None,
  863. ) -> np.ndarray:
  864. """
  865. Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`, using basic
  866. logarithm properties for numerical stability.
  867. The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a
  868. linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.
  869. This means that large variations in energy may not sound all that different if the sound is loud to begin with.
  870. This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.
  871. Based on the implementation of `librosa.power_to_db`.
  872. Args:
  873. spectrogram (`np.ndarray`):
  874. The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared!
  875. reference (`float`, *optional*, defaults to 1.0):
  876. Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
  877. the loudest part to 0 dB. Must be greater than zero.
  878. min_value (`float`, *optional*, defaults to `1e-10`):
  879. The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
  880. `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.
  881. db_range (`float`, *optional*):
  882. Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
  883. peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
  884. Returns:
  885. `np.ndarray`: the spectrogram in decibels
  886. """
  887. if reference <= 0.0:
  888. raise ValueError("reference must be greater than zero")
  889. if min_value <= 0.0:
  890. raise ValueError("min_value must be greater than zero")
  891. reference = max(min_value, reference)
  892. spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
  893. spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference))
  894. if db_range is not None:
  895. if db_range <= 0.0:
  896. raise ValueError("db_range must be greater than zero")
  897. spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None)
  898. return spectrogram
  899. def power_to_db_batch(
  900. spectrogram: np.ndarray,
  901. reference: float = 1.0,
  902. min_value: float = 1e-10,
  903. db_range: float | None = None,
  904. ) -> np.ndarray:
  905. """
  906. Converts a batch of power spectrograms to the decibel scale. This computes `10 * log10(spectrogram / reference)`,
  907. using basic logarithm properties for numerical stability.
  908. This function supports batch processing, where each item in the batch is an individual power (mel) spectrogram.
  909. Args:
  910. spectrogram (`np.ndarray`):
  911. The input batch of power (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape).
  912. Note that a power spectrogram has the amplitudes squared!
  913. reference (`float`, *optional*, defaults to 1.0):
  914. Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
  915. the loudest part to 0 dB. Must be greater than zero.
  916. min_value (`float`, *optional*, defaults to `1e-10`):
  917. The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
  918. `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.
  919. db_range (`float`, *optional*):
  920. Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
  921. peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
  922. Returns:
  923. `np.ndarray`: the batch of spectrograms in decibels
  924. """
  925. if reference <= 0.0:
  926. raise ValueError("reference must be greater than zero")
  927. if min_value <= 0.0:
  928. raise ValueError("min_value must be greater than zero")
  929. reference = max(min_value, reference)
  930. spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
  931. spectrogram = 10.0 * (np.log10(spectrogram) - np.log10(reference))
  932. if db_range is not None:
  933. if db_range <= 0.0:
  934. raise ValueError("db_range must be greater than zero")
  935. # Apply db_range clipping per batch item
  936. max_values = spectrogram.max(axis=(1, 2), keepdims=True)
  937. spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None)
  938. return spectrogram
  939. def amplitude_to_db(
  940. spectrogram: np.ndarray,
  941. reference: float = 1.0,
  942. min_value: float = 1e-5,
  943. db_range: float | None = None,
  944. ) -> np.ndarray:
  945. """
  946. Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`, using
  947. basic logarithm properties for numerical stability.
  948. The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a
  949. linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.
  950. This means that large variations in energy may not sound all that different if the sound is loud to begin with.
  951. This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.
  952. Args:
  953. spectrogram (`np.ndarray`):
  954. The input amplitude (mel) spectrogram.
  955. reference (`float`, *optional*, defaults to 1.0):
  956. Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
  957. the loudest part to 0 dB. Must be greater than zero.
  958. min_value (`float`, *optional*, defaults to `1e-5`):
  959. The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
  960. `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.
  961. db_range (`float`, *optional*):
  962. Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
  963. peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
  964. Returns:
  965. `np.ndarray`: the spectrogram in decibels
  966. """
  967. if reference <= 0.0:
  968. raise ValueError("reference must be greater than zero")
  969. if min_value <= 0.0:
  970. raise ValueError("min_value must be greater than zero")
  971. reference = max(min_value, reference)
  972. spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
  973. spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference))
  974. if db_range is not None:
  975. if db_range <= 0.0:
  976. raise ValueError("db_range must be greater than zero")
  977. spectrogram = np.clip(spectrogram, a_min=spectrogram.max() - db_range, a_max=None)
  978. return spectrogram
  979. def amplitude_to_db_batch(
  980. spectrogram: np.ndarray, reference: float = 1.0, min_value: float = 1e-5, db_range: float | None = None
  981. ) -> np.ndarray:
  982. """
  983. Converts a batch of amplitude spectrograms to the decibel scale. This computes `20 * log10(spectrogram / reference)`,
  984. using basic logarithm properties for numerical stability.
  985. The function supports batch processing, where each item in the batch is an individual amplitude (mel) spectrogram.
  986. Args:
  987. spectrogram (`np.ndarray`):
  988. The input batch of amplitude (mel) spectrograms. Expected shape is (batch_size, *spectrogram_shape).
  989. reference (`float`, *optional*, defaults to 1.0):
  990. Sets the input spectrogram value that corresponds to 0 dB. For example, use `np.max(spectrogram)` to set
  991. the loudest part to 0 dB. Must be greater than zero.
  992. min_value (`float`, *optional*, defaults to `1e-5`):
  993. The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking
  994. `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.
  995. db_range (`float`, *optional*):
  996. Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the
  997. peak value and the smallest value will never be more than 80 dB. Must be greater than zero.
  998. Returns:
  999. `np.ndarray`: the batch of spectrograms in decibels
  1000. """
  1001. if reference <= 0.0:
  1002. raise ValueError("reference must be greater than zero")
  1003. if min_value <= 0.0:
  1004. raise ValueError("min_value must be greater than zero")
  1005. reference = max(min_value, reference)
  1006. spectrogram = np.clip(spectrogram, a_min=min_value, a_max=None)
  1007. spectrogram = 20.0 * (np.log10(spectrogram) - np.log10(reference))
  1008. if db_range is not None:
  1009. if db_range <= 0.0:
  1010. raise ValueError("db_range must be greater than zero")
  1011. # Apply db_range clipping per batch item
  1012. max_values = spectrogram.max(axis=(1, 2), keepdims=True)
  1013. spectrogram = np.clip(spectrogram, a_min=max_values - db_range, a_max=None)
  1014. return spectrogram