depth.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598
  1. # LICENSE HEADER MANAGED BY add-license-header
  2. #
  3. # Copyright 2018 Kornia Team
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. #
  17. """Module containing operators to work on RGB-Depth images."""
  18. from __future__ import annotations
  19. from typing import Optional
  20. import torch
  21. import kornia.core as kornia_ops
  22. from kornia.core import Module, Tensor, tensor
  23. from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE
  24. from kornia.filters.sobel import spatial_gradient
  25. from kornia.utils import create_meshgrid
  26. from .camera import PinholeCamera, cam2pixel, pixel2cam, project_points, unproject_points
  27. from .conversions import normalize_pixel_coordinates, normalize_points_with_intrinsics
  28. from .linalg import convert_points_to_homogeneous, transform_points
  29. """Module containing operators to work on RGB-Depth images."""
  30. __all__ = [
  31. "DepthWarper",
  32. "depth_from_disparity",
  33. "depth_from_plane_equation",
  34. "depth_to_3d",
  35. "depth_to_3d_v2",
  36. "depth_to_normals",
  37. "depth_warp",
  38. "unproject_meshgrid",
  39. "warp_frame_depth",
  40. ]
  41. def unproject_meshgrid(
  42. height: int,
  43. width: int,
  44. camera_matrix: Tensor,
  45. normalize_points: bool = False,
  46. device: Optional[torch.device] = None,
  47. dtype: Optional[torch.dtype] = None,
  48. ) -> Tensor:
  49. """Compute a 3d point per pixel given its depth value and the camera intrinsics.
  50. .. tip::
  51. This function should be used in conjunction with :py:func:`kornia.geometry.depth.depth_to_3d_v2` to cache
  52. the meshgrid computation when warping multiple frames with the same camera intrinsics.
  53. Args:
  54. height: height of image.
  55. width: width of image.
  56. camera_matrix: tensor containing the camera intrinsics with shape :math:`(3, 3)`.
  57. normalize_points: whether to normalize the pointcloud. This must be set to `True` when the depth is
  58. represented as the Euclidean ray length from the camera position.
  59. device: device to place the result on.
  60. dtype: dtype of the result.
  61. Return:
  62. tensor with a 3d point per pixel of the same resolution as the input :math:`(*, H, W, 3)`.
  63. """
  64. KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"])
  65. # create base coordinates grid
  66. points_uv: Tensor = create_meshgrid(
  67. height, width, normalized_coordinates=False, device=device, dtype=dtype
  68. ).squeeze() # HxWx2
  69. # project pixels to camera frame
  70. camera_matrix_tmp: Tensor = camera_matrix[:, None, None] # Bx1x1x3x3
  71. points_xy = normalize_points_with_intrinsics(points_uv, camera_matrix_tmp) # HxWx2
  72. # unproject pixels to camera frame
  73. points_xyz = convert_points_to_homogeneous(points_xy) # HxWx3
  74. if normalize_points:
  75. points_xyz = kornia_ops.normalize(points_xyz, dim=-1, p=2)
  76. return points_xyz
  77. def depth_to_3d_v2(
  78. depth: Tensor, camera_matrix: Tensor, normalize_points: bool = False, xyz_grid: Optional[Tensor] = None
  79. ) -> Tensor:
  80. # NOTE: when this replaces the `depth_to_3d` behaviour, a deprecated function should be added here, instead
  81. # of just replace the other function.
  82. """Compute a 3d point per pixel given its depth value and the camera intrinsics.
  83. .. note::
  84. This is an alternative implementation of :py:func:`kornia.geometry.depth.depth_to_3d`
  85. that does not require the creation of a meshgrid.
  86. Args:
  87. depth: image tensor containing a depth value per pixel with shape :math:`(*, H, W)`.
  88. camera_matrix: tensor containing the camera intrinsics with shape :math:`(*, 3, 3)`.
  89. normalize_points: whether to normalise the pointcloud. This must be set to `True` when the depth is
  90. represented as the Euclidean ray length from the camera position.
  91. xyz_grid: explicit xyz point values.
  92. Return:
  93. tensor with a 3d point per pixel of the same resolution as the input :math:`(*, H, W, 3)`.
  94. Example:
  95. >>> depth = torch.rand(4, 4)
  96. >>> K = torch.eye(3).repeat(2,1,1)
  97. >>> depth_to_3d_v2(depth, K).shape
  98. torch.Size([2, 4, 4, 3])
  99. """
  100. KORNIA_CHECK_SHAPE(depth, ["*", "H", "W"])
  101. KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"])
  102. # create base grid if not provided
  103. height, width = depth.shape[-2:]
  104. points_xyz: Tensor = xyz_grid or unproject_meshgrid(
  105. height, width, camera_matrix, normalize_points, depth.device, depth.dtype
  106. )
  107. KORNIA_CHECK_SHAPE(points_xyz, ["*", "H", "W", "3"])
  108. return points_xyz * depth[..., None] # HxWx3
  109. def depth_to_3d(depth: Tensor, camera_matrix: Tensor, normalize_points: bool = False) -> Tensor:
  110. """Compute a 3d point per pixel given its depth value and the camera intrinsics.
  111. .. note::
  112. This is an alternative implementation of `depth_to_3d` that does not require the creation of a meshgrid.
  113. In future, we will support only this implementation.
  114. Args:
  115. depth: image tensor containing a depth value per pixel with shape :math:`(B, 1, H, W)`.
  116. camera_matrix: tensor containing the camera intrinsics with shape :math:`(B, 3, 3)`.
  117. normalize_points: whether to normalise the pointcloud. This must be set to `True` when the depth is
  118. represented as the Euclidean ray length from the camera position.
  119. Return:
  120. tensor with a 3d point per pixel of the same resolution as the input :math:`(B, 3, H, W)`.
  121. Example:
  122. >>> depth = torch.rand(1, 1, 4, 4)
  123. >>> K = torch.eye(3)[None]
  124. >>> depth_to_3d(depth, K).shape
  125. torch.Size([1, 3, 4, 4])
  126. """
  127. KORNIA_CHECK_IS_TENSOR(depth)
  128. KORNIA_CHECK_IS_TENSOR(camera_matrix)
  129. KORNIA_CHECK_SHAPE(depth, ["B", "1", "H", "W"])
  130. KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"])
  131. # create base coordinates grid
  132. _, _, height, width = depth.shape
  133. points_2d: Tensor = create_meshgrid(height, width, normalized_coordinates=False) # 1xHxWx2
  134. points_2d = points_2d.to(depth.device).to(depth.dtype)
  135. # depth should come in Bx1xHxW
  136. points_depth: Tensor = depth.permute(0, 2, 3, 1) # 1xHxWx1
  137. # project pixels to camera frame
  138. camera_matrix_tmp: Tensor = camera_matrix[:, None, None] # Bx1x1x3x3
  139. points_3d: Tensor = unproject_points(
  140. points_2d, points_depth, camera_matrix_tmp, normalize=normalize_points
  141. ) # BxHxWx3
  142. return points_3d.permute(0, 3, 1, 2) # Bx3xHxW
  143. def depth_to_normals(depth: Tensor, camera_matrix: Tensor, normalize_points: bool = False) -> Tensor:
  144. """Compute the normal surface per pixel.
  145. Args:
  146. depth: image tensor containing a depth value per pixel with shape :math:`(B, 1, H, W)`.
  147. camera_matrix: tensor containing the camera intrinsics with shape :math:`(B, 3, 3)`.
  148. normalize_points: whether to normalize the pointcloud. This must be set to `True` when the depth is
  149. represented as the Euclidean ray length from the camera position.
  150. Return:
  151. tensor with a normal surface vector per pixel of the same resolution as the input :math:`(B, 3, H, W)`.
  152. Example:
  153. >>> depth = torch.rand(1, 1, 4, 4)
  154. >>> K = torch.eye(3)[None]
  155. >>> depth_to_normals(depth, K).shape
  156. torch.Size([1, 3, 4, 4])
  157. """
  158. KORNIA_CHECK_IS_TENSOR(depth)
  159. KORNIA_CHECK_IS_TENSOR(camera_matrix)
  160. KORNIA_CHECK_SHAPE(depth, ["B", "1", "H", "W"])
  161. KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"])
  162. # compute the 3d points from depth
  163. xyz: Tensor = depth_to_3d(depth, camera_matrix, normalize_points) # Bx3xHxW
  164. # compute the pointcloud spatial gradients
  165. gradients: Tensor = spatial_gradient(xyz) # Bx3x2xHxW
  166. # compute normals
  167. a, b = gradients[:, :, 0], gradients[:, :, 1] # Bx3xHxW
  168. normals: Tensor = torch.linalg.cross(a, b, dim=1)
  169. return kornia_ops.normalize(normals, dim=1, p=2)
  170. def depth_from_plane_equation(
  171. plane_normals: Tensor, plane_offsets: Tensor, points_uv: Tensor, camera_matrix: Tensor, eps: float = 1e-8
  172. ) -> Tensor:
  173. """Compute depth values from plane equations and pixel coordinates.
  174. Args:
  175. plane_normals (Tensor): Plane normal vectors of shape (B, 3).
  176. plane_offsets (Tensor): Plane offsets of shape (B, 1).
  177. points_uv (Tensor): Pixel coordinates of shape (B, N, 2).
  178. camera_matrix (Tensor): Camera intrinsic matrix of shape (B, 3, 3).
  179. eps: epsilon for numerical stability.
  180. Returns:
  181. Tensor: Computed depth values at the given pixels, shape (B, N).
  182. """
  183. KORNIA_CHECK_SHAPE(plane_normals, ["B", "3"])
  184. KORNIA_CHECK_SHAPE(plane_offsets, ["B", "1"])
  185. KORNIA_CHECK_SHAPE(points_uv, ["B", "N", "2"])
  186. KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"])
  187. # Normalize pixel coordinates
  188. points_xy = normalize_points_with_intrinsics(points_uv, camera_matrix) # (B, N, 2)
  189. rays = convert_points_to_homogeneous(points_xy) # (B, N, 3)
  190. # Reshape plane normals to match rays
  191. plane_normals_exp = plane_normals.unsqueeze(1) # (B, 1, 3)
  192. # No need to unsqueeze plane_offsets; it is already (B, 1)
  193. # Compute the denominator of the depth equation
  194. denom = torch.sum(rays * plane_normals_exp, dim=-1) # (B, N)
  195. denom_abs = torch.abs(denom)
  196. zero_mask = denom_abs < eps
  197. denom = torch.where(zero_mask, eps * torch.sign(denom), denom)
  198. # Compute depth from plane equation
  199. depth = plane_offsets / denom # plane_offsets: (B, 1), denom: (B, N) -> depth: (B, N)
  200. return depth
  201. def warp_frame_depth(
  202. image_src: Tensor, depth_dst: Tensor, src_trans_dst: Tensor, camera_matrix: Tensor, normalize_points: bool = False
  203. ) -> Tensor:
  204. """Warp a tensor from a source to destination frame by the depth in the destination.
  205. Compute 3d points from the depth, transform them using given transformation, then project the point cloud to an
  206. image plane.
  207. Args:
  208. image_src: image tensor in the source frame with shape :math:`(B,D,H,W)`.
  209. depth_dst: depth tensor in the destination frame with shape :math:`(B,1,H,W)`.
  210. src_trans_dst: transformation matrix from destination to source with shape :math:`(B,4,4)`.
  211. camera_matrix: tensor containing the camera intrinsics with shape :math:`(B,3,3)`.
  212. normalize_points: whether to normalize the pointcloud. This must be set to ``True`` when the depth
  213. is represented as the Euclidean ray length from the camera position.
  214. Return:
  215. the warped tensor in the source frame with shape :math:`(B,3,H,W)`.
  216. """
  217. KORNIA_CHECK_SHAPE(image_src, ["B", "D", "H", "W"])
  218. KORNIA_CHECK_SHAPE(depth_dst, ["B", "1", "H", "W"])
  219. KORNIA_CHECK_SHAPE(src_trans_dst, ["B", "4", "4"])
  220. KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"])
  221. # unproject source points to camera frame
  222. points_3d_dst: Tensor = depth_to_3d(depth_dst, camera_matrix, normalize_points) # Bx3xHxW
  223. # transform points from source to destination
  224. points_3d_dst = points_3d_dst.permute(0, 2, 3, 1) # BxHxWx3
  225. # apply transformation to the 3d points
  226. points_3d_src = transform_points(src_trans_dst[:, None], points_3d_dst) # BxHxWx3
  227. # project back to pixels
  228. camera_matrix_tmp: Tensor = camera_matrix[:, None, None] # Bx1x1xHxW
  229. points_2d_src: Tensor = project_points(points_3d_src, camera_matrix_tmp) # BxHxWx2
  230. # normalize points between [-1 / 1]
  231. height, width = depth_dst.shape[-2:]
  232. points_2d_src_norm: Tensor = normalize_pixel_coordinates(points_2d_src, height, width) # BxHxWx2
  233. return kornia_ops.map_coordinates(image_src, points_2d_src_norm, align_corners=True)
  234. class DepthWarper(Module):
  235. r"""Warp a patch by depth.
  236. .. math::
  237. P_{src}^{\{dst\}} = K_{dst} * T_{src}^{\{dst\}}
  238. I_{src} = \\omega(I_{dst}, P_{src}^{\{dst\}}, D_{src})
  239. Args:
  240. pinholes_dst: the pinhole models for the destination frame.
  241. height: the height of the image to warp.
  242. width: the width of the image to warp.
  243. mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``.
  244. padding_mode: padding mode for outside grid values ``'zeros'`` | ``'border'`` | ``'reflection'``.
  245. align_corners: interpolation flag.
  246. """
  247. # All per-instance, not global (thread safe, multiple warps)
  248. def __init__(
  249. self,
  250. pinhole_dst: PinholeCamera,
  251. height: int,
  252. width: int,
  253. mode: str = "bilinear",
  254. padding_mode: str = "zeros",
  255. align_corners: bool = True,
  256. ) -> None:
  257. super().__init__()
  258. self.width: int = width
  259. self.height: int = height
  260. self.mode: str = mode
  261. self.padding_mode: str = padding_mode
  262. self.eps = 1e-6
  263. self.align_corners: bool = align_corners
  264. # state members
  265. # _pinhole_dst is Type[PinholeCamera], enforce in constructor
  266. if not isinstance(pinhole_dst, PinholeCamera):
  267. raise TypeError(f"Expected pinhole_dst as PinholeCamera, got {type(pinhole_dst)}")
  268. self._pinhole_dst: PinholeCamera = pinhole_dst
  269. self._pinhole_src: None | PinholeCamera = None
  270. self._dst_proj_src: None | Tensor = None
  271. # Meshgrid only depends on (height, width), can be staticmethod cached
  272. self.grid: Tensor = self._create_meshgrid(height, width)
  273. @staticmethod
  274. def _create_meshgrid(height: int, width: int) -> Tensor:
  275. grid: Tensor = create_meshgrid(height, width, normalized_coordinates=False) # 1xHxWx2
  276. return convert_points_to_homogeneous(grid) # append ones to last dim
  277. def compute_projection_matrix(self, pinhole_src: PinholeCamera) -> DepthWarper:
  278. """Compute the projection matrix from the source to destination frame."""
  279. # Inline type checks for faster fail-fast
  280. if type(self._pinhole_dst) is not PinholeCamera:
  281. raise TypeError(
  282. f"Member self._pinhole_dst expected to be of class PinholeCamera. Got {type(self._pinhole_dst)}"
  283. )
  284. if type(pinhole_src) is not PinholeCamera:
  285. raise TypeError(f"Argument pinhole_src expected to be of class PinholeCamera. Got {type(pinhole_src)}")
  286. # Compute transformation matrix: dst_extrinsics @ inv(src_extrinsics)
  287. batch_shape = pinhole_src.extrinsics.shape[:-2]
  288. device = pinhole_src.extrinsics.device
  289. dtype = pinhole_src.extrinsics.dtype
  290. # Create 4x4 identity matrices efficiently
  291. inv_extr = torch.eye(4, device=device, dtype=dtype).expand(*batch_shape, 4, 4).contiguous()
  292. dst_trans_src = torch.eye(4, device=device, dtype=dtype).expand(*batch_shape, 4, 4).contiguous()
  293. # Inline inverse transformation
  294. src_rmat = pinhole_src.extrinsics[..., :3, :3]
  295. src_tvec = pinhole_src.extrinsics[..., :3, 3:]
  296. inv_rmat = torch.transpose(src_rmat, -1, -2)
  297. inv_tvec = torch.matmul(-inv_rmat, src_tvec)
  298. # Set rotation and translation parts
  299. inv_extr[..., :3, :3] = inv_rmat
  300. inv_extr[..., :3, 3:] = inv_tvec
  301. # Compose with dst extrinsics
  302. dst_rmat = self._pinhole_dst.extrinsics[..., :3, :3]
  303. dst_tvec = self._pinhole_dst.extrinsics[..., :3, 3:]
  304. composed_rmat = torch.matmul(dst_rmat, inv_rmat)
  305. composed_tvec = torch.matmul(dst_rmat, inv_tvec) + dst_tvec
  306. dst_trans_src[..., :3, :3] = composed_rmat
  307. dst_trans_src[..., :3, 3:] = composed_tvec
  308. # intrinsics (Nx3x3) @ extrinsics (Nx4x4)
  309. dst_proj_src = torch.matmul(self._pinhole_dst.intrinsics, dst_trans_src)
  310. self._pinhole_src = pinhole_src
  311. self._dst_proj_src = dst_proj_src
  312. return self
  313. def _compute_projection(self, x: float, y: float, invd: float) -> Tensor:
  314. if self._dst_proj_src is None or self._pinhole_src is None:
  315. raise ValueError("Please, call compute_projection_matrix.")
  316. point = tensor([[[x], [y], [invd], [1.0]]], device=self._dst_proj_src.device, dtype=self._dst_proj_src.dtype)
  317. flow = torch.matmul(self._dst_proj_src, point)
  318. z = 1.0 / flow[:, 2]
  319. _x = flow[:, 0] * z
  320. _y = flow[:, 1] * z
  321. return kornia_ops.concatenate([_x, _y], 1)
  322. def compute_subpixel_step(self) -> Tensor:
  323. """Compute the inverse depth step for sub pixel accurate sampling of the depth cost volume, per camera.
  324. Szeliski, Richard, and Daniel Scharstein. "Symmetric sub-pixel stereo matching." European Conference on Computer
  325. Vision. Springer Berlin Heidelberg, 2002.
  326. """
  327. if self._dst_proj_src is None:
  328. raise RuntimeError("Expected Tensor, but got None Type from the projection matrix")
  329. delta_d = 0.01
  330. center_x = self.width / 2
  331. center_y = self.height / 2
  332. # Batch both invds in one call (for potential fused kernels in future) for efficiency
  333. invds = (1.0 - delta_d, 1.0 + delta_d)
  334. # Instead of two calls, process both at once with minimal tensor construction
  335. points = (
  336. torch.tensor(
  337. [[center_x, center_y, invds[0], 1.0], [center_x, center_y, invds[1], 1.0]],
  338. dtype=self._dst_proj_src.dtype,
  339. device=self._dst_proj_src.device,
  340. )
  341. .transpose(0, 1)
  342. .unsqueeze(0)
  343. ) # (1, 4, 2)
  344. # Repeat projection matrix for batch
  345. proj = self._dst_proj_src
  346. flow = torch.matmul(proj, points) # (N, 3/4, 2)
  347. zs = 1.0 / flow[:, 2] # (N, 2)
  348. xs = flow[:, 0] * zs
  349. ys = flow[:, 1] * zs
  350. xys = torch.stack((xs, ys), dim=-1) # (N, 2, 2)
  351. dxy = torch.norm(xys[:, 1] - xys[:, 0], p=2, dim=1) / 2.0
  352. dxdd = dxy / delta_d
  353. # half pixel sampling, min for all cameras
  354. return torch.min(0.5 / dxdd)
  355. def warp_grid(self, depth_src: Tensor) -> Tensor:
  356. """Compute a grid for warping a given the depth from the reference pinhole camera.
  357. The function `compute_projection_matrix` has to be called beforehand in order to have precomputed the relative
  358. projection matrices encoding the relative pose and the intrinsics between the reference and a non reference
  359. camera.
  360. """
  361. # TODO: add type and value checkings
  362. if self._dst_proj_src is None or self._pinhole_src is None:
  363. raise ValueError("Please, call compute_projection_matrix.")
  364. if len(depth_src.shape) != 4:
  365. raise ValueError(f"Input depth_src has to be in the shape of Bx1xHxW. Got {depth_src.shape}")
  366. # unpack depth attributes
  367. batch_size, _, _, _ = depth_src.shape
  368. device: torch.device = depth_src.device
  369. dtype: torch.dtype = depth_src.dtype
  370. # expand the base coordinate grid according to the input batch size
  371. pixel_coords: Tensor = self.grid.to(device=device, dtype=dtype).expand(batch_size, -1, -1, -1) # BxHxWx3
  372. # reproject the pixel coordinates to the camera frame
  373. cam_coords_src: Tensor = pixel2cam(
  374. depth_src, self._pinhole_src.intrinsics_inverse().to(device=device, dtype=dtype), pixel_coords
  375. ) # BxHxWx3
  376. # reproject the camera coordinates to the pixel
  377. pixel_coords_src: Tensor = cam2pixel(
  378. cam_coords_src, self._dst_proj_src.to(device=device, dtype=dtype)
  379. ) # (B*N)xHxWx2
  380. # normalize between -1 and 1 the coordinates
  381. pixel_coords_src_norm: Tensor = normalize_pixel_coordinates(pixel_coords_src, self.height, self.width)
  382. return pixel_coords_src_norm
  383. def forward(self, depth_src: Tensor, patch_dst: Tensor) -> Tensor:
  384. """Warp a tensor from destination frame to reference given the depth in the reference frame.
  385. Args:
  386. depth_src: the depth in the reference frame. The tensor must have a shape :math:`(B, 1, H, W)`.
  387. patch_dst: the patch in the destination frame. The tensor must have a shape :math:`(B, C, H, W)`.
  388. Return:
  389. the warped patch from destination frame to reference.
  390. Shape:
  391. - Output: :math:`(N, C, H, W)` where C = number of channels.
  392. Example:
  393. >>> # pinholes camera models
  394. >>> pinhole_dst = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4),
  395. ... torch.tensor([32]), torch.tensor([32]))
  396. >>> pinhole_src = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4),
  397. ... torch.tensor([32]), torch.tensor([32]))
  398. >>> # create the depth warper, compute the projection matrix
  399. >>> warper = DepthWarper(pinhole_dst, 32, 32)
  400. >>> _ = warper.compute_projection_matrix(pinhole_src)
  401. >>> # warp the destination frame to reference by depth
  402. >>> depth_src = torch.ones(1, 1, 32, 32) # Nx1xHxW
  403. >>> image_dst = torch.rand(1, 3, 32, 32) # NxCxHxW
  404. >>> image_src = warper(depth_src, image_dst) # NxCxHxW
  405. """
  406. return kornia_ops.map_coordinates(
  407. patch_dst,
  408. self.warp_grid(depth_src),
  409. mode=self.mode,
  410. padding_mode=self.padding_mode,
  411. align_corners=self.align_corners,
  412. )
  413. def depth_warp(
  414. pinhole_dst: PinholeCamera,
  415. pinhole_src: PinholeCamera,
  416. depth_src: Tensor,
  417. patch_dst: Tensor,
  418. height: int,
  419. width: int,
  420. align_corners: bool = True,
  421. ) -> Tensor:
  422. """Warp a tensor from destination frame to reference given the depth in the reference frame.
  423. See :class:`~kornia.geometry.warp.DepthWarper` for details.
  424. Example:
  425. >>> # pinholes camera models
  426. >>> pinhole_dst = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4),
  427. ... torch.tensor([32]), torch.tensor([32]))
  428. >>> pinhole_src = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4),
  429. ... torch.tensor([32]), torch.tensor([32]))
  430. >>> # warp the destination frame to reference by depth
  431. >>> depth_src = torch.ones(1, 1, 32, 32) # Nx1xHxW
  432. >>> image_dst = torch.rand(1, 3, 32, 32) # NxCxHxW
  433. >>> image_src = depth_warp(pinhole_dst, pinhole_src, depth_src, image_dst, 32, 32) # NxCxHxW
  434. """
  435. # Cache and reuse warper and projection matrix (single use/call)
  436. # Inlined for performance, use local variables and freed objects
  437. # instead of class members where possible.
  438. warper = DepthWarper(pinhole_dst, height, width, align_corners=align_corners)
  439. # projection matrix is required for each call, avoid double checking in class
  440. warper.compute_projection_matrix(pinhole_src)
  441. # __call__ implemented by Module (likely calls forward, not shown).
  442. return warper(depth_src, patch_dst)
  443. def depth_from_disparity(disparity: Tensor, baseline: float | Tensor, focal: float | Tensor) -> Tensor:
  444. """Compute depth from disparity.
  445. Args:
  446. disparity: Disparity tensor of shape :math:`(*, H, W)`.
  447. baseline: float/tensor containing the distance between the two lenses.
  448. focal: float/tensor containing the focal length.
  449. Return:
  450. Depth map of the shape :math:`(*, H, W)`.
  451. Example:
  452. >>> disparity = torch.rand(4, 1, 4, 4)
  453. >>> baseline = torch.rand(1)
  454. >>> focal = torch.rand(1)
  455. >>> depth_from_disparity(disparity, baseline, focal).shape
  456. torch.Size([4, 1, 4, 4])
  457. """
  458. KORNIA_CHECK_IS_TENSOR(disparity, f"Input disparity type is not a Tensor. Got {type(disparity)}.")
  459. KORNIA_CHECK_SHAPE(disparity, ["*", "H", "W"])
  460. KORNIA_CHECK(
  461. isinstance(baseline, (float, Tensor)),
  462. f"Input baseline should be either a float or Tensor. Got {type(baseline)}",
  463. )
  464. KORNIA_CHECK(
  465. isinstance(focal, (float, Tensor)), f"Input focal should be either a float or Tensor. Got {type(focal)}"
  466. )
  467. if isinstance(baseline, Tensor):
  468. KORNIA_CHECK_SHAPE(baseline, ["1"])
  469. if isinstance(focal, Tensor):
  470. KORNIA_CHECK_SHAPE(focal, ["1"])
  471. return baseline * focal / (disparity + 1e-8)