1 from __future__
import division
2 from __future__
import absolute_import
3 from __future__
import print_function
4 from __future__
import unicode_literals
15 _INFERENCE_MIN_SIZE = 0
18 def heatmaps_to_keypoints(maps, rois):
19 """Extracts predicted keypoint locations from heatmaps. Output has shape 20 (#rois, 4, #keypoints) with the 4 rows corresponding to (x, y, logit, prob) 31 widths = rois[:, 2] - rois[:, 0]
32 heights = rois[:, 3] - rois[:, 1]
33 widths = np.maximum(widths, 1)
34 heights = np.maximum(heights, 1)
35 widths_ceil = np.ceil(widths).astype(np.int)
36 heights_ceil = np.ceil(heights).astype(np.int)
38 num_keypoints = np.maximum(maps.shape[1], _NUM_KEYPOINTS)
41 maps = np.transpose(maps, [0, 2, 3, 1])
42 min_size = _INFERENCE_MIN_SIZE
45 (len(rois), 4, num_keypoints), dtype=np.float32)
46 for i
in range(len(rois)):
48 roi_map_width = int(np.maximum(widths_ceil[i], min_size))
49 roi_map_height = int(np.maximum(heights_ceil[i], min_size))
51 roi_map_width = widths_ceil[i]
52 roi_map_height = heights_ceil[i]
53 width_correction = widths[i] / roi_map_width
54 height_correction = heights[i] / roi_map_height
56 maps[i], (roi_map_width, roi_map_height),
57 interpolation=cv2.INTER_CUBIC)
60 roi_map = np.transpose(roi_map, [2, 0, 1])
61 roi_map_probs = scores_to_probs(roi_map.copy())
63 for k
in range(num_keypoints):
64 pos = roi_map[k, :, :].argmax()
66 y_int = (pos - x_int) // w
67 assert (roi_map_probs[k, y_int, x_int] ==
68 roi_map_probs[k, :, :].max())
69 x = (x_int + 0.5) * width_correction
70 y = (y_int + 0.5) * height_correction
71 xy_preds[i, 0, k] = x + offset_x[i]
72 xy_preds[i, 1, k] = y + offset_y[i]
73 xy_preds[i, 2, k] = roi_map[k, y_int, x_int]
74 xy_preds[i, 3, k] = roi_map_probs[k, y_int, x_int]
79 def scores_to_probs(scores):
80 """Transforms CxHxW of scores to probabilities spatially.""" 81 channels = scores.shape[0]
82 for c
in range(channels):
83 temp = scores[c, :, :]
84 max_score = temp.max()
85 temp = np.exp(temp - max_score) / np.sum(np.exp(temp - max_score))
86 scores[c, :, :] = temp
90 def approx_heatmap_keypoint(heatmaps_in, bboxes_in):
92 Mask R-CNN uses bicubic upscaling before taking the maximum of the heat map 93 for keypoints. We are using bilinear upscaling, which means we can approximate 94 the maximum coordinate with the low dimension maximum coordinates. We would like 95 to avoid bicubic upscaling, because it is computationally expensive. Brown and 96 Lowe (Invariant Features from Interest Point Groups, 2002) uses a method for 97 fitting a 3D quadratic function to the local sample points to determine the 98 interpolated location of the maximum of scale space, and his experiments showed 99 that this provides a substantial improvement to matching and stability for 100 keypoint extraction. This approach uses the Taylor expansion (up to the 101 quadratic terms) of the scale-space function. It is equivalent with the Newton 102 method. This efficient method were used in many keypoint estimation algorithms 103 like SIFT, SURF etc... 105 The implementation of Newton methods with numerical analysis is straight forward 106 and super simple, though we need a linear solver. 109 assert len(bboxes_in.shape) == 2
110 N = bboxes_in.shape[0]
111 assert bboxes_in.shape[1] == 4
112 assert len(heatmaps_in.shape) == 4
113 assert heatmaps_in.shape[0] == N
114 keypoint_count = heatmaps_in.shape[1]
115 heatmap_size = heatmaps_in.shape[2]
116 assert heatmap_size >= 2
117 assert heatmaps_in.shape[3] == heatmap_size
119 keypoints_out = np.zeros((N, keypoint_count, 4))
122 x0, y0, x1, y1 = bboxes_in[k, :]
123 xLen = np.maximum(x1 - x0, 1)
124 yLen = np.maximum(y1 - y0, 1)
125 softmax_map = scores_to_probs(heatmaps_in[k, :, :, :].copy())
127 for j
in range(keypoint_count):
128 f = heatmaps_in[k][j]
133 for y
in range(heatmap_size):
134 for x
in range(heatmap_size):
136 prob = softmax_map[j, y, x]
137 if maxX < 0
or maxScore < score:
146 fmax = [[0] * 3
for r
in range(3)]
151 hm_x = hm_x - 2 * (hm_x >= heatmap_size) + 2 * (hm_x < 0)
152 hm_y = hm_y - 2 * (hm_y >= heatmap_size) + 2 * (hm_y < 0)
153 assert((hm_x < heatmap_size)
and (hm_x >= 0))
154 assert((hm_y < heatmap_size)
and (hm_y >= 0))
155 fmax[y][x] = f[hm_y][hm_x]
159 b = [-(fmax[1][2] - fmax[1][0]) / 2, -
160 (fmax[2][1] - fmax[0][1]) / 2]
161 A = [[fmax[1][0] - 2 * fmax[1][1] + fmax[1][2],
162 (fmax[2][2] - fmax[2][0] - fmax[0][2] + fmax[0][0]) / 4],
163 [(fmax[2][2] - fmax[2][0] - fmax[0][2] + fmax[0][0]) / 4,
164 fmax[0][1] - 2 * fmax[1][1] + fmax[2][1]]]
168 div = A[1][1] * A[0][0] - A[0][1] * A[1][0]
169 if abs(div) < 0.0001:
172 deltaScore = maxScore
174 deltaY = (b[1] * A[0][0] - b[0] * A[1][0]) / div
175 deltaX = (b[0] * A[1][1] - b[1] * A[0][1]) / div
177 if abs(deltaX) > 1.5
or abs(deltaY) > 1.5:
178 scale = 1.5 / max(abs(deltaX), abs(deltaY))
184 fmax[1][1] - (b[0] * deltaX + b[1] * deltaY) +
185 0.5 * (deltaX * deltaX * A[0][0] +
186 deltaX * deltaY * A[1][0] +
187 deltaY * deltaX * A[0][1] +
188 deltaY * deltaY * A[1][1]))
190 assert abs(deltaX) <= 1.5
191 assert abs(deltaY) <= 1.5
194 keypoints_out[k, j, :] = (
195 x0 + (maxX + deltaX + .5) * xLen / heatmap_size,
196 y0 + (maxY + deltaY + .5) * yLen / heatmap_size,
201 keypoints_out = np.transpose(keypoints_out, [0, 2, 1])