From fd847c504e229187774a15c0c0826a0d17072800 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Wed, 11 Jan 2017 14:47:19 +0000 Subject: [PATCH 01/75] Allow pixel masks to be input on a per image basis --- opensfm/commands/detect_features.py | 11 ++++++++- opensfm/dataset.py | 38 +++++++++++++++++++++++++++++ opensfm/features.py | 19 ++++++++++++--- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/opensfm/commands/detect_features.py b/opensfm/commands/detect_features.py index cbb6a10e6..223c23332 100644 --- a/opensfm/commands/detect_features.py +++ b/opensfm/commands/detect_features.py @@ -40,10 +40,19 @@ def detect(args): image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) + + mask_path = None + mask_name = image.split('.')[0] + 'mask' + if mask_name in data.masks(): + mask_path = data.mask_files[mask_name] + logger.info('Found mask {} to apply'.format( + mask_name + )) + if not data.feature_index_exists(image): preemptive_max = data.config.get('preemptive_max', 200) p_unsorted, f_unsorted, c_unsorted = features.extract_features( - data.image_as_array(image), data.config) + data.image_as_array(image), data.config, mask_path) if len(p_unsorted) == 0: return diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 61077252d..5fa3bd970 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -41,6 +41,15 @@ def __init__(self, data_path): else: self.set_image_path(os.path.join(self.data_path, 'images')) + # Load list of masks if they exist. + mask_list_file = os.path.join(self.data_path, 'mask_list.txt') + if os.path.isfile(mask_list_file): + with open(mask_list_file) as fin: + lines = fin.read().splitlines() + self.set_mask_list(lines) + else: + self.set_mask_path(os.path.join(self.data_path, 'masks')) + def _load_config(self): config_file = os.path.join(self.data_path, 'config.yaml') self.config = config.load_config(config_file) @@ -80,6 +89,10 @@ def save_undistorted_image(self, image, array): io.mkdir_p(self._undistorted_image_path()) cv2.imwrite(self._undistorted_image_file(image), array[:, :, ::-1]) + def masks(self): + """Return list of file names of all masks in this dataset""" + return self.mask_list + def _depthmap_path(self): return os.path.join(self.data_path, 'depthmaps') @@ -134,6 +147,31 @@ def set_image_list(self, image_list): self.image_list.append(name) self.image_files[name] = path + @staticmethod + def __is_mask_file(filename): + return filename.split('.')[-1].lower() in {'xml'} + + def set_mask_path(self, path): + """Set mask path and find the all mask in there""" + self.mask_list = [] + self.mask_files = {} + if os.path.exists(path): + for name in os.listdir(path): + if self.__is_mask_file(name): + short_name = name.split('.')[0] + self.mask_list.append(short_name) + self.mask_files[short_name] = os.path.join(path, name) + + def set_mask_list(self, mask_list): + self.mask_list = [] + self.mask_files = {} + for line in mask_list: + path = os.path.join(self.data_path, line) + name = os.path.basename(path) + short_name = name.split('.')[0] + self.mask_list.append(short_name) + self.mask_files[short_name] = path + def __exif_path(self): """Return path of extracted exif directory""" return os.path.join(self.data_path, 'exif') diff --git a/opensfm/features.py b/opensfm/features.py index 54cf1dde5..ddec8c8aa 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -68,7 +68,7 @@ def denormalized_image_coordinates(norm_coords, width, height): p[:, 1] = norm_coords[:, 1] * size - 0.5 + height / 2.0 return p -def mask_and_normalize_features(points, desc, colors, width, height, config): +def mask_and_normalize_features(points, desc, colors, width, height, config, mask_path=None): masks = np.array(config.get('masks',[])) for mask in masks: top = mask['top'] * height @@ -82,6 +82,19 @@ def mask_and_normalize_features(points, desc, colors, width, height, config): points = points[ids] desc = desc[ids] colors = colors[ids] + + # We get the relevant image mask + if mask_path is not None: + test = cv2.FileStorage(mask_path, cv2.FILE_STORAGE_READ) + node = test.getFirstTopLevelNode() + mask = node.mat() + if mask.shape != (height, width): + raise TypeError("Given mask does not match image dimensions") + ids = np.array([mask[int(point[1]), int(point[0])] != 0 for point in points]) + points = points[ids] + desc = desc[ids] + colors = colors[ids] + points[:, :2] = normalized_image_coordinates(points[:, :2], width, height) return points, desc, colors @@ -222,7 +235,7 @@ def extract_features_hahog(image, config): logger.debug('Found {0} points in {1}s'.format( len(points), time.time()-t )) return points, desc -def extract_features(color_image, config): +def extract_features(color_image, config, mask_path=None): assert len(color_image.shape) == 3 color_image = resized_image(color_image, config) image = cv2.cvtColor(color_image, cv2.COLOR_RGB2GRAY) @@ -243,7 +256,7 @@ def extract_features(color_image, config): ys = points[:,1].round().astype(int) colors = color_image[ys, xs] - return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], config) + return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], config, mask_path) def build_flann_index(features, config): From 4108cd78c588894e1f7cba925d8d53650eabf04a Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Tue, 24 Jan 2017 12:19:54 +0000 Subject: [PATCH 02/75] Change to storing masks as images with data in first channel --- opensfm/dataset.py | 2 +- opensfm/features.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 5fa3bd970..043d3e5ce 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -149,7 +149,7 @@ def set_image_list(self, image_list): @staticmethod def __is_mask_file(filename): - return filename.split('.')[-1].lower() in {'xml'} + return DataSet.__is_image_file(filename) def set_mask_path(self, path): """Set mask path and find the all mask in there""" diff --git a/opensfm/features.py b/opensfm/features.py index ddec8c8aa..174baa8fa 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -85,12 +85,13 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, mas # We get the relevant image mask if mask_path is not None: - test = cv2.FileStorage(mask_path, cv2.FILE_STORAGE_READ) - node = test.getFirstTopLevelNode() - mask = node.mat() - if mask.shape != (height, width): + maskname = mask_path[:-3] + + mask = cv2.imread(mask_path) + mask_height, mask_width, _ = mask.shape + if (mask_height, mask_width) != (height, width): raise TypeError("Given mask does not match image dimensions") - ids = np.array([mask[int(point[1]), int(point[0])] != 0 for point in points]) + ids = np.array([mask[int(point[1]), int(point[0]), 0] == 0 for point in points]) points = points[ids] desc = desc[ids] colors = colors[ids] From 09fce3956b7c67ccbfe78a22d36617be485413dc Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Tue, 24 Jan 2017 14:15:22 +0000 Subject: [PATCH 03/75] Tidying code --- opensfm/commands/detect_features.py | 5 +++-- opensfm/dataset.py | 14 ++++++-------- opensfm/features.py | 2 -- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/opensfm/commands/detect_features.py b/opensfm/commands/detect_features.py index 223c23332..c901ebdac 100644 --- a/opensfm/commands/detect_features.py +++ b/opensfm/commands/detect_features.py @@ -41,13 +41,14 @@ def detect(args): logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) - mask_path = None - mask_name = image.split('.')[0] + 'mask' + mask_name = image if mask_name in data.masks(): mask_path = data.mask_files[mask_name] logger.info('Found mask {} to apply'.format( mask_name )) + else: + mask_path = None if not data.feature_index_exists(image): preemptive_max = data.config.get('preemptive_max', 200) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 043d3e5ce..53a50b1a7 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -129,7 +129,7 @@ def __is_image_file(filename): return filename.split('.')[-1].lower() in {'jpg', 'jpeg', 'png', 'tif', 'tiff', 'pgm', 'pnm', 'gif'} def set_image_path(self, path): - """Set image path and find the all images in there""" + """Set image path and find all images in there""" self.image_list = [] self.image_files = {} if os.path.exists(path): @@ -152,15 +152,14 @@ def __is_mask_file(filename): return DataSet.__is_image_file(filename) def set_mask_path(self, path): - """Set mask path and find the all mask in there""" + """Set mask path and find all masks in there""" self.mask_list = [] self.mask_files = {} if os.path.exists(path): for name in os.listdir(path): if self.__is_mask_file(name): - short_name = name.split('.')[0] - self.mask_list.append(short_name) - self.mask_files[short_name] = os.path.join(path, name) + self.mask_list.append(name) + self.mask_files[name] = os.path.join(path, name) def set_mask_list(self, mask_list): self.mask_list = [] @@ -168,9 +167,8 @@ def set_mask_list(self, mask_list): for line in mask_list: path = os.path.join(self.data_path, line) name = os.path.basename(path) - short_name = name.split('.')[0] - self.mask_list.append(short_name) - self.mask_files[short_name] = path + self.mask_list.append(name) + self.mask_files[name] = path def __exif_path(self): """Return path of extracted exif directory""" diff --git a/opensfm/features.py b/opensfm/features.py index 174baa8fa..b0354f29b 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -85,8 +85,6 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, mas # We get the relevant image mask if mask_path is not None: - maskname = mask_path[:-3] - mask = cv2.imread(mask_path) mask_height, mask_width, _ = mask.shape if (mask_height, mask_width) != (height, width): From b5b925961cfbb916ab88336c35c381bd3a936e34 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Tue, 24 Jan 2017 14:59:05 +0000 Subject: [PATCH 04/75] Add feature to treat images as video series and only match nearby frames --- opensfm/commands/match_features.py | 15 +++++++++++---- opensfm/config.py | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/opensfm/commands/match_features.py b/opensfm/commands/match_features.py index 76497d5a2..17ab9bfbb 100644 --- a/opensfm/commands/match_features.py +++ b/opensfm/commands/match_features.py @@ -108,6 +108,7 @@ def match_candidates_from_metadata(images, exifs, data): max_distance = data.config['matching_gps_distance'] max_neighbors = data.config['matching_gps_neighbors'] max_time_neighbors = data.config['matching_time_neighbors'] + max_order_neighbors = data.config['matching_order_neighbors'] if not all(map(has_gps_info, exifs.values())) and max_neighbors != 0: logger.warn("Not all images have GPS info. " @@ -115,24 +116,30 @@ def match_candidates_from_metadata(images, exifs, data): max_neighbors = 0 pairs = set() - for im1 in images: + images.sort() + for index1, im1 in enumerate(images): distances = [] timediffs = [] - for im2 in images: + indexdiffs = [] + for index2, im2 in enumerate(images): if im1 != im2: dx = distance_from_exif(exifs[im1], exifs[im2]) dt = timediff_from_exif(exifs[im1], exifs[im2]) + di = abs(index1 - index2) if dx <= max_distance: distances.append((dx, im2)) timediffs.append((dt, im2)) + indexdiffs.append((di, im2)) distances.sort() timediffs.sort() + indexdiffs.sort() - if max_neighbors or max_time_neighbors: + if max_neighbors or max_time_neighbors or max_order_neighbors: distances = distances[:max_neighbors] timediffs = timediffs[:max_time_neighbors] + indexdiffs = indexdiffs[:max_order_neighbors] - for d, im2 in distances + timediffs: + for d, im2 in distances + timediffs + indexdiffs: if im1 < im2: pairs.add((im1, im2)) else: diff --git a/opensfm/config.py b/opensfm/config.py index 2101451a9..b2d94f1e0 100644 --- a/opensfm/config.py +++ b/opensfm/config.py @@ -51,6 +51,7 @@ matching_gps_distance: 150 # Maximum gps distance between two images for matching matching_gps_neighbors: 0 # Number of images to match selected by GPS distance. Set to 0 to use no limit matching_time_neighbors: 0 # Number of images to match selected by time taken. Set to 0 to use no limit +matching_order_neighbors: 0 # Selects based on image names preemptive_max: 200 # Number of features to use for preemptive matching preemptive_threshold: 0 # If number of matches passes the threshold -> full feature matching From 2203d04f1b54db10fb8f1fb36213c5ca75759f03 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Tue, 24 Jan 2017 15:03:26 +0000 Subject: [PATCH 05/75] Tidy up comment --- opensfm/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/config.py b/opensfm/config.py index b2d94f1e0..5c4d3f60a 100644 --- a/opensfm/config.py +++ b/opensfm/config.py @@ -51,7 +51,7 @@ matching_gps_distance: 150 # Maximum gps distance between two images for matching matching_gps_neighbors: 0 # Number of images to match selected by GPS distance. Set to 0 to use no limit matching_time_neighbors: 0 # Number of images to match selected by time taken. Set to 0 to use no limit -matching_order_neighbors: 0 # Selects based on image names +matching_order_neighbors: 0 # Number of images to match selected by image name. Set to 0 to use no limit preemptive_max: 200 # Number of features to use for preemptive matching preemptive_threshold: 0 # If number of matches passes the threshold -> full feature matching From cca388682bea2d7e755abb9cbd73e96ca3b3bfb7 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 1 Feb 2017 23:26:16 +0300 Subject: [PATCH 06/75] Add sensor width for DJI Phantom 4. --- opensfm/data/sensor_data.json | 1 + 1 file changed, 1 insertion(+) diff --git a/opensfm/data/sensor_data.json b/opensfm/data/sensor_data.json index 68a5ab14f..8b2c9c378 100644 --- a/opensfm/data/sensor_data.json +++ b/opensfm/data/sensor_data.json @@ -703,6 +703,7 @@ "DJI FC300S": 6.16, "DJI FC300X": 6.2, "DJI FC350": 6.17, + "DJI FC330": 6.25, "Epson L-500V": 5.75, "Epson PhotoPC 3000 Zoom": 7.11, "Epson PhotoPC 3100 Zoom": 7.11, From bb91c69a9cad16281f9ee5f93ca5ea6c54570364 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Tue, 24 Jan 2017 16:18:51 +0100 Subject: [PATCH 07/75] Add option to plot matches between image subsets --- bin/plot_matches | 66 +++++++++++++++++++++++++----------------------- bin/plot_tracks | 55 ++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 56 deletions(-) diff --git a/bin/plot_matches b/bin/plot_matches index eebeb161e..776bf39f6 100755 --- a/bin/plot_matches +++ b/bin/plot_matches @@ -1,6 +1,7 @@ #!/usr/bin/env python import argparse +from itertools import combinations import matplotlib.pyplot as pl import networkx as nx @@ -33,7 +34,9 @@ if __name__ == "__main__": parser.add_argument('dataset', help='path to the dataset to be processed') parser.add_argument('--image', - help='name of the image to show') + help='show tracks for a specific') + parser.add_argument('--images', + help='show tracks between a subset of images (separated by commas)') parser.add_argument('--graph', help='display image graph', action='store_true') @@ -65,39 +68,40 @@ if __name__ == "__main__": else: # Plot matches between images if args.image: - toplot = [args.image] + pairs = [(args.image, o) for o in images if o != args.image] + elif args.images: + subset = args.images.split(',') + pairs = combinations(subset, 2) else: - toplot = images + pairs = combinations(images, 2) i = 0 - for im1 in toplot: - for im2 in images: - if im1 != im2: - matches = data.find_matches(im1, im2) - if len(matches) == 0: - continue - print 'plotting matches between', im1, im2 - - p1, f1, c1 = data.load_features(im1) - p2, f2, c2 = data.load_features(im2) - p1 = p1[matches[:, 0]] - p2 = p2[matches[:, 1]] - - pl.figure(figsize=(20, 10)) - pl.title('Images: ' + im1 + ' - ' + im2 + ', matches: ' + - str(matches.shape[0])) - plot_matches(data.image_as_array(im1), - data.image_as_array(im2), p1, p2) - i += 1 - if args.save_figs: - p = args.dataset + '/plot_tracks' - io.mkdir_p(p) - pl.savefig(p + '/' + im1 + '_' + im2 + '.jpg', dpi=100) - pl.close() - else: - if i >= 10: - i = 0 - pl.show() + for im1, im2 in pairs: + matches = data.find_matches(im1, im2) + if len(matches) == 0: + continue + print 'plotting matches between', im1, im2 + + p1, f1, c1 = data.load_features(im1) + p2, f2, c2 = data.load_features(im2) + p1 = p1[matches[:, 0]] + p2 = p2[matches[:, 1]] + + pl.figure(figsize=(20, 10)) + pl.title('Images: ' + im1 + ' - ' + im2 + ', matches: ' + + str(matches.shape[0])) + plot_matches(data.image_as_array(im1), + data.image_as_array(im2), p1, p2) + i += 1 + if args.save_figs: + p = args.dataset + '/plot_tracks' + io.mkdir_p(p) + pl.savefig(p + '/' + im1 + '_' + im2 + '.jpg', dpi=100) + pl.close() + else: + if i >= 10: + i = 0 + pl.show() if not args.save_figs and i > 0: pl.show() diff --git a/bin/plot_tracks b/bin/plot_tracks index 932b6e00c..cc76017ec 100755 --- a/bin/plot_tracks +++ b/bin/plot_tracks @@ -1,6 +1,8 @@ #!/usr/bin/env python import argparse +from itertools import combinations + import matplotlib.pyplot as pl import networkx as nx import numpy as np @@ -30,11 +32,13 @@ def plot_matches(im1, im2, p1, p2): if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Compute reconstruction') + parser = argparse.ArgumentParser(description='Plot tracks') parser.add_argument('dataset', help='path to the dataset to be processed') parser.add_argument('--image', - help='name of the image to show') + help='show tracks for a specific') + parser.add_argument('--images', + help='show tracks between a subset of images (separated by commas)') parser.add_argument('--graph', help='display image graph', action='store_true') @@ -60,32 +64,33 @@ if __name__ == "__main__": else: # Plot matches between images if args.image: - toplot = [args.image] + pairs = [(args.image, o) for o in images if o != args.image] + elif args.images: + subset = args.images.split(',') + pairs = combinations(subset, 2) else: - toplot = images + pairs = combinations(images, 2) i = 0 - for im1 in toplot: - for im2 in images: - if im1 != im2: - t, p1, p2 = matching.common_tracks(graph, im1, im2) - if len(t) >= 10: - pl.figure(figsize=(20, 10)) - pl.title('Images: ' + im1 + ' - ' + im2 + - ', matches: ' + str(len(t))) - plot_matches(data.image_as_array(im1), - data.image_as_array(im2), p1, p2) - i += 1 - if args.save_figs: - p = args.dataset + '/plot_tracks' - io.mkdir_p(p) - pl.savefig(p + '/' + im1 + '_' + im2 + '.jpg', - dpi=100) - pl.close() - else: - if i >= 10: - i = 0 - pl.show() + for im1, im2 in pairs: + t, p1, p2 = matching.common_tracks(graph, im1, im2) + if len(t) >= 10: + pl.figure(figsize=(20, 10)) + pl.title('Images: ' + im1 + ' - ' + im2 + + ', matches: ' + str(len(t))) + plot_matches(data.image_as_array(im1), + data.image_as_array(im2), p1, p2) + i += 1 + if args.save_figs: + p = args.dataset + '/plot_tracks' + io.mkdir_p(p) + pl.savefig(p + '/' + im1 + '_' + im2 + '.jpg', + dpi=100) + pl.close() + else: + if i >= 10: + i = 0 + pl.show() if not args.save_figs and i > 0: pl.show() From 0abbe043bd02804e784928f6d7c25fd413102dfe Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 26 Jan 2017 00:38:21 +0100 Subject: [PATCH 08/75] Convert id to str when bundling with GPS --- opensfm/reconstruction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensfm/reconstruction.py b/opensfm/reconstruction.py index 37a3036cb..4563368c9 100644 --- a/opensfm/reconstruction.py +++ b/opensfm/reconstruction.py @@ -60,14 +60,14 @@ def bundle(graph, reconstruction, gcp, config, fix_cameras=False): if config['bundle_use_gps']: for shot in reconstruction.shots.values(): g = shot.metadata.gps_position - ba.add_position_prior(shot.id, g[0], g[1], g[2], + ba.add_position_prior(str(shot.id), g[0], g[1], g[2], shot.metadata.gps_dop) if config['bundle_use_gcp'] and gcp: for observation in gcp: if observation.shot_id in reconstruction.shots: ba.add_ground_control_point_observation( - observation.shot_id, + str(observation.shot_id), observation.coordinates[0], observation.coordinates[1], observation.coordinates[2], From c4cbd493bf604ec1e5500482706ffd52414fb1ec Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 26 Jan 2017 00:38:54 +0100 Subject: [PATCH 09/75] Use bilinear interpolation to remap panoramas --- opensfm/commands/undistort.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/opensfm/commands/undistort.py b/opensfm/commands/undistort.py index fb350a9fc..0489cc2be 100644 --- a/opensfm/commands/undistort.py +++ b/opensfm/commands/undistort.py @@ -42,8 +42,10 @@ def undistort_images(self, graph, reconstruction, data): data.save_undistorted_image(shot.id, undistorted) elif shot.camera.projection_type in ['equirectangular', 'spherical']: original = data.image_as_array(shot.id) - image = cv2.resize(original, (2048, 1024), interpolation=cv2.INTER_AREA) - shots = perspective_views_of_a_panorama(shot) + width = int(data.config['depthmap_resolution']) + height = width / 2 + image = cv2.resize(original, (width, height), interpolation=cv2.INTER_AREA) + shots = perspective_views_of_a_panorama(shot, width) for subshot in shots: urec.add_camera(subshot.camera) urec.add_shot(subshot) @@ -65,12 +67,12 @@ def undistort_image(image, shot): return cv2.undistort(image, K, distortion) -def perspective_views_of_a_panorama(spherical_shot): +def perspective_views_of_a_panorama(spherical_shot, width): """Create 6 perspective views of a panorama.""" camera = types.PerspectiveCamera() camera.id = 'perspective_panorama_camera' - camera.width = 640 - camera.height = 640 + camera.width = width + camera.height = width camera.focal = 0.5 camera.focal_prior = camera.focal camera.k1 = camera.k1_prior = camera.k2 = camera.k2_prior = 0.0 @@ -125,13 +127,13 @@ def render_perspective_view_of_a_panorama(image, panoshot, perspectiveshot): src_pixels = np.column_stack([src_x.ravel(), src_y.ravel()]) src_pixels_denormalized = features.denormalized_image_coordinates( - src_pixels, - image.shape[1], - image.shape[0]) + src_pixels, image.shape[1], image.shape[0]) # Sample color - colors = image[src_pixels_denormalized[:, 1].astype(int), - src_pixels_denormalized[:, 0].astype(int)] + colors = cv2.remap(image, + src_pixels_denormalized[:, 0].astype(np.float32), + src_pixels_denormalized[:, 1].astype(np.float32), + cv2.INTER_LINEAR) colors.shape = dst_shape + (-1,) return colors From 5fc86b9890cca83f630dafc656c3278b7ae8542e Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sat, 4 Feb 2017 19:07:06 +0100 Subject: [PATCH 10/75] Prevent opencv3 imread from using EXIF orientation Fixes #135 --- opensfm/dataset.py | 6 ++---- opensfm/io.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 61077252d..1db80ca22 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -61,8 +61,7 @@ def load_image(self, image): def image_as_array(self, image): """Return image pixels as 3-dimensional numpy array (R G B order)""" - IMREAD_COLOR = cv2.IMREAD_COLOR if context.OPENCV3 else cv2.CV_LOAD_IMAGE_COLOR - return cv2.imread(self.__image_file(image), IMREAD_COLOR)[:,:,::-1] # Turn BGR to RGB + return io.imread(self.__image_file(image)) def _undistorted_image_path(self): return os.path.join(self.data_path, 'undistorted') @@ -73,8 +72,7 @@ def _undistorted_image_file(self, image): def undistorted_image_as_array(self, image): """Undistorted image pixels as 3-dimensional numpy array (R G B order)""" - IMREAD_COLOR = cv2.IMREAD_COLOR if context.OPENCV3 else cv2.CV_LOAD_IMAGE_COLOR - return cv2.imread(self._undistorted_image_file(image), IMREAD_COLOR)[:,:,::-1] # Turn BGR to RGB + return io.imread(self._undistorted_image_file(image)) def save_undistorted_image(self, image, array): io.mkdir_p(self._undistorted_image_path()) diff --git a/opensfm/io.py b/opensfm/io.py index 821dbfc24..b7379edb3 100644 --- a/opensfm/io.py +++ b/opensfm/io.py @@ -9,6 +9,7 @@ from opensfm import features from opensfm import geo from opensfm import types +from opensfm import context def camera_from_json(key, obj): @@ -353,6 +354,16 @@ def json_loads(text, codec='utf-8'): return json.loads(text.decode(codec)) +def imread(filename): + """Load image as an RGB array ignoring EXIF orientation.""" + if context.OPENCV3: + flags = cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION + else: + flags = cv2.CV_LOAD_IMAGE_COLOR + bgr = cv2.imread(filename, flags) + return bgr[:, :, ::-1] # Turn BGR to RGB + + # Bundler def export_bundler(image_list, reconstructions, track_graph, bundle_file_path, @@ -473,7 +484,7 @@ def import_bundler(data_path, bundle_file, list_file, track_file, focal, k1, k2 = map(float, lines[offset].rstrip('\n').split(' ')) if focal > 0: - im = cv2.imread(os.path.join(data_path, image_list[i])) + im = imread(os.path.join(data_path, image_list[i])) height, width = im.shape[0:2] camera = types.PerspectiveCamera() camera.id = 'camera_' + str(i) From 06fe9695254cf09c5f2f8447f626a149ddaa6933 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 5 Feb 2017 01:15:41 +0100 Subject: [PATCH 11/75] Add fisheye camera model --- opensfm/io.py | 4 +-- opensfm/types.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/opensfm/io.py b/opensfm/io.py index b7379edb3..ba58f7f5f 100644 --- a/opensfm/io.py +++ b/opensfm/io.py @@ -17,7 +17,7 @@ def camera_from_json(key, obj): Read camera from a json object """ pt = obj.get('projection_type', 'perspective') - if pt == 'perspective': + if pt in ['perspective', 'fisheye']: camera = types.PerspectiveCamera() camera.id = key camera.width = obj.get('width', 0) @@ -142,7 +142,7 @@ def camera_to_json(camera): """ Write camera to a json object """ - if camera.projection_type == 'perspective': + if camera.projection_type in ['perspective', 'fisheye']: return { 'projection_type': camera.projection_type, 'width': camera.width, diff --git a/opensfm/types.py b/opensfm/types.py index c25f73c6a..47adc00ab 100644 --- a/opensfm/types.py +++ b/opensfm/types.py @@ -238,6 +238,90 @@ def get_K_in_pixel_coordinates(self, width=None, height=None): [0, 0, 1.0]]) +class FisheyeCamera(Camera): + """Define a fisheye camera. + + Attributes: + widht (int): image width. + height (int): image height. + focal (real): estimated focal lenght. + k1 (real): estimated first distortion parameter. + k2 (real): estimated second distortion parameter. + focal_prior (real): prior focal lenght. + k1_prior (real): prior first distortion parameter. + k2_prior (real): prior second distortion parameter. + """ + + def __init__(self): + """Defaut constructor.""" + self.id = None + self.projection_type = 'fisheye' + self.width = None + self.height = None + self.focal = None + self.k1 = None + self.k2 = None + self.focal_prior = None + self.k1_prior = None + self.k2_prior = None + + def project(self, point): + """Project a 3D point in camera coordinates to the image plane.""" + x, y, z = point + l = np.sqrt(x**2 + y**2) + theta = np.arctan2(l, z) + theta_d = theta * (1.0 + theta * (self.k1 + theta * self.k2)) + s = self.focal * theta_d / l + return np.array([s * x, s * y]) + + def pixel_bearing(self, pixel): + """Unit vector pointing to the pixel viewing direction.""" + point = np.asarray(pixel).reshape((1, 1, 2)) + distortion = np.array([self.k1, self.k2, 0., 0.]) + x, y = cv2.fisheye.undistortPoints(point, self.get_K(), distortion).flat + l = np.sqrt(x * x + y * y + 1.0) + return np.array([x / l, y / l, 1.0 / l]) + + def pixel_bearings(self, pixels): + """Unit vector pointing to the pixel viewing directions.""" + points = pixels.reshape((-1, 1, 2)).astype(np.float64) + distortion = np.array([self.k1, self.k2, 0., 0.]) + up = cv2.fisheye.undistortPoints(points, self.get_K(), distortion) + up = up.reshape((-1, 2)) + x = up[:, 0] + y = up[:, 1] + l = np.sqrt(x * x + y * y + 1.0) + return np.column_stack((x / l, y / l, 1.0 / l)) + + def back_project(self, pixel, depth): + """Project a pixel to a fronto-parallel plane at a given depth.""" + bearing = self.pixel_bearing(pixel) + scale = depth / bearing[2] + return scale * bearing + + def get_K(self): + """The calibration matrix.""" + return np.array([[self.focal, 0., 0.], + [0., self.focal, 0.], + [0., 0., 1.]]) + + def get_K_in_pixel_coordinates(self, width=None, height=None): + """The calibration matrix that maps to pixel coordinates. + + Coordinates (0,0) correspond to the center of the top-left pixel, + and (width - 1, height - 1) to the center of bottom-right pixel. + + You can optionally pass the width and height of the image, in case + you are using a resized versior of the original image. + """ + w = width or self.width + h = height or self.height + f = self.focal * max(w, h) + return np.array([[f, 0, 0.5 * (w - 1)], + [0, f, 0.5 * (h - 1)], + [0, 0, 1.0]]) + + class SphericalCamera(Camera): """A spherical camera generating equirectangular projections. From 7fad29b1936a43d7a1fd624ac6efd5bd2860c37f Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 5 Feb 2017 11:11:02 +0100 Subject: [PATCH 12/75] Handle fisheye io independently of perspective --- opensfm/io.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/opensfm/io.py b/opensfm/io.py index ba58f7f5f..15b288a10 100644 --- a/opensfm/io.py +++ b/opensfm/io.py @@ -17,7 +17,7 @@ def camera_from_json(key, obj): Read camera from a json object """ pt = obj.get('projection_type', 'perspective') - if pt in ['perspective', 'fisheye']: + if pt == 'perspective': camera = types.PerspectiveCamera() camera.id = key camera.width = obj.get('width', 0) @@ -29,6 +29,18 @@ def camera_from_json(key, obj): camera.k1_prior = obj.get('k1_prior', camera.k1) camera.k2_prior = obj.get('k2_prior', camera.k2) return camera + elif pt == 'fisheye': + camera = types.FisheyeCamera() + camera.id = key + camera.width = obj.get('width', 0) + camera.height = obj.get('height', 0) + camera.focal = obj['focal'] + camera.k1 = obj.get('k1', 0.0) + camera.k2 = obj.get('k2', 0.0) + camera.focal_prior = obj.get('focal_prior', camera.focal) + camera.k1_prior = obj.get('k1_prior', camera.k1) + camera.k2_prior = obj.get('k2_prior', camera.k2) + return camera elif pt in ['equirectangular', 'spherical']: camera = types.SphericalCamera() camera.id = key @@ -142,7 +154,19 @@ def camera_to_json(camera): """ Write camera to a json object """ - if camera.projection_type in ['perspective', 'fisheye']: + if camera.projection_type == 'perspective': + return { + 'projection_type': camera.projection_type, + 'width': camera.width, + 'height': camera.height, + 'focal': camera.focal, + 'k1': camera.k1, + 'k2': camera.k2, + 'focal_prior': camera.focal_prior, + 'k1_prior': camera.k1_prior, + 'k2_prior': camera.k2_prior + } + elif camera.projection_type == 'fisheye': return { 'projection_type': camera.projection_type, 'width': camera.width, From 25ad5e64e8a72aa7af98baafc11aaf1bbc943bb5 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 5 Feb 2017 11:11:30 +0100 Subject: [PATCH 13/75] Bundle adjust fisheye cameras --- opensfm/reconstruction.py | 15 +++- opensfm/src/bundle.h | 157 ++++++++++++++++++++++++++++++++++++++ opensfm/src/csfm.cc | 11 +++ 3 files changed, 182 insertions(+), 1 deletion(-) diff --git a/opensfm/reconstruction.py b/opensfm/reconstruction.py index 4563368c9..1dc884730 100644 --- a/opensfm/reconstruction.py +++ b/opensfm/reconstruction.py @@ -32,7 +32,11 @@ def bundle(graph, reconstruction, gcp, config, fix_cameras=False): str(camera.id), camera.focal, camera.k1, camera.k2, camera.focal_prior, camera.k1_prior, camera.k2_prior, fix_cameras) - + elif camera.projection_type == 'fisheye': + ba.add_fisheye_camera( + str(camera.id), camera.focal, camera.k1, camera.k2, + camera.focal_prior, camera.k1_prior, camera.k2_prior, + fix_cameras) elif camera.projection_type in ['equirectangular', 'spherical']: ba.add_equirectangular_camera(str(camera.id)) @@ -96,6 +100,11 @@ def bundle(graph, reconstruction, gcp, config, fix_cameras=False): camera.focal = c.focal camera.k1 = c.k1 camera.k2 = c.k2 + elif camera.projection_type == 'fisheye': + c = ba.get_fisheye_camera(str(camera.id)) + camera.focal = c.focal + camera.k1 = c.k1 + camera.k2 = c.k2 for shot in reconstruction.shots.values(): s = ba.get_shot(str(shot.id)) @@ -123,6 +132,10 @@ def bundle_single_view(graph, reconstruction, shot_id, config): ba.add_perspective_camera( str(camera.id), camera.focal, camera.k1, camera.k2, camera.focal_prior, camera.k1_prior, camera.k2_prior, True) + elif camera.projection_type == 'fisheye': + ba.add_fisheye_camera( + str(camera.id), camera.focal, camera.k1, camera.k2, + camera.focal_prior, camera.k1_prior, camera.k2_prior, True) elif camera.projection_type in ['equirectangular', 'spherical']: ba.add_equirectangular_camera(str(camera.id)) diff --git a/opensfm/src/bundle.h b/opensfm/src/bundle.h index 83d83b5b1..b04663f65 100644 --- a/opensfm/src/bundle.h +++ b/opensfm/src/bundle.h @@ -18,6 +18,7 @@ extern "C" { enum BACameraType { BA_PERSPECTIVE_CAMERA, + BA_FISHEYE_CAMERA, BA_EQUIRECTANGULAR_CAMERA }; @@ -50,6 +51,21 @@ struct BAPerspectiveCamera : public BACamera{ void SetK2(double v) { parameters[BA_CAMERA_K2] = v; } }; +struct BAFisheyeCamera : public BACamera{ + double parameters[BA_CAMERA_NUM_PARAMS]; + double focal_prior; + double k1_prior; + double k2_prior; + + BACameraType type() { return BA_PERSPECTIVE_CAMERA; } + double GetFocal() { return parameters[BA_CAMERA_FOCAL]; } + double GetK1() { return parameters[BA_CAMERA_K1]; } + double GetK2() { return parameters[BA_CAMERA_K2]; } + void SetFocal(double v) { parameters[BA_CAMERA_FOCAL] = v; } + void SetK1(double v) { parameters[BA_CAMERA_K1] = v; } + void SetK2(double v) { parameters[BA_CAMERA_K2] = v; } +}; + struct BAEquirectangularCamera : public BACamera { BACameraType type() { return BA_EQUIRECTANGULAR_CAMERA; } }; @@ -231,6 +247,62 @@ struct PerspectiveReprojectionError { double scale_; }; +template +void FisheyeProject(const T* const camera, + const T point[3], + T projection[2]) { + const T& focal = camera[BA_CAMERA_FOCAL]; + const T& k1 = camera[BA_CAMERA_K1]; + const T& k2 = camera[BA_CAMERA_K2]; + const T &x = point[0]; + const T &y = point[1]; + const T &z = point[2]; + + T l = sqrt(x * x + y * y); + T theta = atan2(l, z); + T theta_d = theta * (T(1.0) + theta * (k1 + theta * k2)); + T s = focal * theta_d / l; + + projection[0] = s * x; + projection[1] = s * y; +} + + +struct FisheyeReprojectionError { + FisheyeReprojectionError(double observed_x, double observed_y, double std_deviation) + : observed_x_(observed_x) + , observed_y_(observed_y) + , scale_(1.0 / std_deviation) + {} + + template + bool operator()(const T* const camera, + const T* const shot, + const T* const point, + T* residuals) const { + T camera_point[3]; + WorldToCameraCoordinates(shot, point, camera_point); + + if (camera_point[2] <= T(0.0)) { + residuals[0] = residuals[1] = T(99.0); + return true; + } + + T predicted[2]; + FisheyeProject(camera, camera_point, predicted); + + // The error is the difference between the predicted and observed position. + residuals[0] = T(scale_) * (predicted[0] - T(observed_x_)); + residuals[1] = T(scale_) * (predicted[1] - T(observed_y_)); + + return true; + } + + double observed_x_; + double observed_y_; + double scale_; +}; + struct EquirectangularReprojectionError { EquirectangularReprojectionError(double observed_x, double observed_y, double std_deviation) : scale_(1.0 / std_deviation) @@ -515,6 +587,10 @@ class BundleAdjuster { return *(BAPerspectiveCamera *)cameras_[id].get(); } + BAFisheyeCamera GetFisheyeCamera(const std::string &id) { + return *(BAFisheyeCamera *)cameras_[id].get(); + } + BAEquirectangularCamera GetEquirectangularCamera(const std::string &id) { return *(BAEquirectangularCamera *)cameras_[id].get(); } @@ -548,6 +624,27 @@ class BundleAdjuster { c.k2_prior = k2_prior; } + void AddFisheyeCamera( + const std::string &id, + double focal, + double k1, + double k2, + double focal_prior, + double k1_prior, + double k2_prior, + bool constant) { + cameras_[id] = std::unique_ptr(new BAFisheyeCamera()); + BAFisheyeCamera &c = static_cast(*cameras_[id]); + c.id = id; + c.parameters[BA_CAMERA_FOCAL] = focal; + c.parameters[BA_CAMERA_K1] = k1; + c.parameters[BA_CAMERA_K2] = k2; + c.constant = constant; + c.focal_prior = focal_prior; + c.k1_prior = k1_prior; + c.k2_prior = k2_prior; + } + void AddEquirectangularCamera( const std::string &id) { cameras_[id] = std::unique_ptr(new BAEquirectangularCamera()); @@ -761,6 +858,13 @@ class BundleAdjuster { problem.SetParameterBlockConstant(c.parameters); break; } + case BA_FISHEYE_CAMERA: + { + BAFisheyeCamera &c = static_cast(*i.second); + problem.AddParameterBlock(c.parameters, BA_CAMERA_NUM_PARAMS); + problem.SetParameterBlockConstant(c.parameters); + break; + } case BA_EQUIRECTANGULAR_CAMERA: // No parameters for now break; @@ -799,6 +903,22 @@ class BundleAdjuster { observation.point->coordinates); break; } + case BA_FISHEYE_CAMERA: + { + BAFisheyeCamera &c = static_cast(*observation.camera); + ceres::CostFunction* cost_function = + new ceres::AutoDiffCostFunction( + new FisheyeReprojectionError(observation.coordinates[0], + observation.coordinates[1], + reprojection_error_sd_)); + + problem.AddResidualBlock(cost_function, + loss, + c.parameters, + observation.shot->parameters, + observation.point->coordinates); + break; + } case BA_EQUIRECTANGULAR_CAMERA: { BAEquirectangularCamera &c = static_cast(*observation.camera); @@ -878,6 +998,11 @@ class BundleAdjuster { observation.shot->parameters); break; } + case BA_FISHEYE_CAMERA: + { + std::cerr << "NotImplemented: GCP for fisheye cameras\n"; + break; + } case BA_EQUIRECTANGULAR_CAMERA: { ceres::CostFunction* cost_function = @@ -912,6 +1037,21 @@ class BundleAdjuster { c.parameters); break; } + case BA_FISHEYE_CAMERA: + { + BAFisheyeCamera &c = static_cast(*i.second); + + ceres::CostFunction* cost_function = + new ceres::AutoDiffCostFunction( + new InternalParametersPriorError(c.focal_prior, focal_prior_sd_, + c.k1_prior, k1_sd_, + c.k2_prior, k2_sd_)); + + problem.AddResidualBlock(cost_function, + NULL, + c.parameters); + break; + } case BA_EQUIRECTANGULAR_CAMERA: break; } @@ -1005,6 +1145,23 @@ class BundleAdjuster { std::max(observations_[i].point->reprojection_error, error); break; } + case BA_FISHEYE_CAMERA: + { + BAFisheyeCamera &c = static_cast(*observations_[i].camera); + + FisheyeReprojectionError pre(observations_[i].coordinates[0], + observations_[i].coordinates[1], + 1.0); + double residuals[2]; + pre(c.parameters, + observations_[i].shot->parameters, + observations_[i].point->coordinates, + residuals); + double error = sqrt(residuals[0] * residuals[0] + residuals[1] * residuals[1]); + observations_[i].point->reprojection_error = + std::max(observations_[i].point->reprojection_error, error); + break; + } case BA_EQUIRECTANGULAR_CAMERA: { BAEquirectangularCamera &c = static_cast(*observations_[i].camera); diff --git a/opensfm/src/csfm.cc b/opensfm/src/csfm.cc index d3110852d..c528d300b 100644 --- a/opensfm/src/csfm.cc +++ b/opensfm/src/csfm.cc @@ -83,10 +83,12 @@ BOOST_PYTHON_MODULE(csfm) { class_("BundleAdjuster") .def("run", &BundleAdjuster::Run) .def("get_perspective_camera", &BundleAdjuster::GetPerspectiveCamera) + .def("get_fisheye_camera", &BundleAdjuster::GetFisheyeCamera) .def("get_equirectangular_camera", &BundleAdjuster::GetEquirectangularCamera) .def("get_shot", &BundleAdjuster::GetShot) .def("get_point", &BundleAdjuster::GetPoint) .def("add_perspective_camera", &BundleAdjuster::AddPerspectiveCamera) + .def("add_fisheye_camera", &BundleAdjuster::AddFisheyeCamera) .def("add_equirectangular_camera", &BundleAdjuster::AddEquirectangularCamera) .def("add_shot", &BundleAdjuster::AddShot) .def("add_point", &BundleAdjuster::AddPoint) @@ -119,6 +121,15 @@ BOOST_PYTHON_MODULE(csfm) { .def_readwrite("id", &BAPerspectiveCamera::id) ; + class_("BAFisheyeCamera") + .add_property("focal", &BAFisheyeCamera::GetFocal, &BAFisheyeCamera::SetFocal) + .add_property("k1", &BAFisheyeCamera::GetK1, &BAFisheyeCamera::SetK1) + .add_property("k2", &BAFisheyeCamera::GetK2, &BAFisheyeCamera::SetK2) + .def_readwrite("constant", &BAFisheyeCamera::constant) + .def_readwrite("focal_prior", &BAFisheyeCamera::focal_prior) + .def_readwrite("id", &BAFisheyeCamera::id) + ; + class_("BAShot") .add_property("rx", &BAShot::GetRX, &BAShot::SetRX) .add_property("ry", &BAShot::GetRY, &BAShot::SetRY) From 52abc4aded248b2d195a2e9a1a2597699814ec6d Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Wed, 8 Feb 2017 14:26:44 +0000 Subject: [PATCH 14/75] Load masks from within the Dataset class --- opensfm/commands/detect_features.py | 14 ++++---------- opensfm/dataset.py | 10 ++++++++++ opensfm/features.py | 9 ++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/opensfm/commands/detect_features.py b/opensfm/commands/detect_features.py index c901ebdac..80ccca050 100644 --- a/opensfm/commands/detect_features.py +++ b/opensfm/commands/detect_features.py @@ -41,19 +41,13 @@ def detect(args): logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) - mask_name = image - if mask_name in data.masks(): - mask_path = data.mask_files[mask_name] - logger.info('Found mask {} to apply'.format( - mask_name - )) - else: - mask_path = None - if not data.feature_index_exists(image): + mask = data.mask_as_array(image) + if mask is not None: + logger.info('Found mask to apply for image {}'.format(image)) preemptive_max = data.config.get('preemptive_max', 200) p_unsorted, f_unsorted, c_unsorted = features.extract_features( - data.image_as_array(image), data.config, mask_path) + data.image_as_array(image), data.config, mask) if len(p_unsorted) == 0: return diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 53a50b1a7..234386edc 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -93,6 +93,16 @@ def masks(self): """Return list of file names of all masks in this dataset""" return self.mask_list + def mask_as_array(self, image): + """Given an image, returns the associated mask as an array if it exists, otherwise returns None""" + mask_name = image + if mask_name in self.masks(): + mask_path = self.mask_files[mask_name] + mask = cv2.imread(mask_path) + else: + mask = None + return mask + def _depthmap_path(self): return os.path.join(self.data_path, 'depthmaps') diff --git a/opensfm/features.py b/opensfm/features.py index b0354f29b..f30d2b8b6 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -68,7 +68,7 @@ def denormalized_image_coordinates(norm_coords, width, height): p[:, 1] = norm_coords[:, 1] * size - 0.5 + height / 2.0 return p -def mask_and_normalize_features(points, desc, colors, width, height, config, mask_path=None): +def mask_and_normalize_features(points, desc, colors, width, height, config, mask=None): masks = np.array(config.get('masks',[])) for mask in masks: top = mask['top'] * height @@ -84,8 +84,7 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, mas colors = colors[ids] # We get the relevant image mask - if mask_path is not None: - mask = cv2.imread(mask_path) + if mask is not None: mask_height, mask_width, _ = mask.shape if (mask_height, mask_width) != (height, width): raise TypeError("Given mask does not match image dimensions") @@ -234,7 +233,7 @@ def extract_features_hahog(image, config): logger.debug('Found {0} points in {1}s'.format( len(points), time.time()-t )) return points, desc -def extract_features(color_image, config, mask_path=None): +def extract_features(color_image, config, mask=None): assert len(color_image.shape) == 3 color_image = resized_image(color_image, config) image = cv2.cvtColor(color_image, cv2.COLOR_RGB2GRAY) @@ -255,7 +254,7 @@ def extract_features(color_image, config, mask_path=None): ys = points[:,1].round().astype(int) colors = color_image[ys, xs] - return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], config, mask_path) + return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], config, mask) def build_flann_index(features, config): From c1ed100166c7f26e73e7b457b1493e3a0249ead3 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Wed, 8 Feb 2017 14:28:22 +0000 Subject: [PATCH 15/75] Switch to using 0 as the mask out value for per image masks --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index f30d2b8b6..562c09ac5 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -88,7 +88,7 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, mas mask_height, mask_width, _ = mask.shape if (mask_height, mask_width) != (height, width): raise TypeError("Given mask does not match image dimensions") - ids = np.array([mask[int(point[1]), int(point[0]), 0] == 0 for point in points]) + ids = np.array([mask[int(point[1]), int(point[0]), 0] != 0 for point in points]) points = points[ids] desc = desc[ids] colors = colors[ids] From 08eab3684d9475a26b6ab68c7adfdbc72ce81dad Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 18:34:42 +0100 Subject: [PATCH 16/75] Handle fisheye cameras on the viewer --- viewer/reconstruction.html | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/viewer/reconstruction.html b/viewer/reconstruction.html index 07e255088..53a208c37 100644 --- a/viewer/reconstruction.html +++ b/viewer/reconstruction.html @@ -169,20 +169,38 @@ - @@ -714,15 +732,14 @@ } function imageVertexShader(cam) { - if (cam.projection_type == 'equirectangular' || cam.projection_type == 'spherical') - return $('#vertexshader_equirectangular').text(); - else - return $('#vertexshader').text(); + return $('#vertexshader').text(); } function imageFragmentShader(cam) { if (cam.projection_type == 'equirectangular' || cam.projection_type == 'spherical') return $('#fragmentshader_equirectangular').text(); + else if (cam.projection_type == 'fisheye') + return $('#fragmentshader_fisheye').text(); else return $('#fragmentshader').text(); } From 310f75da3a61d3bf36a39fd746c2526c51018d45 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 18:35:38 +0100 Subject: [PATCH 17/75] Fix writing partial reconstructions --- opensfm/reconstruction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/reconstruction.py b/opensfm/reconstruction.py index 1dc884730..830c520e6 100644 --- a/opensfm/reconstruction.py +++ b/opensfm/reconstruction.py @@ -701,7 +701,7 @@ def grow_reconstruction(data, graph, reconstruction, images, gcp): if data.config.get('save_partial_reconstructions', False): paint_reconstruction(data, graph, reconstruction) data.save_reconstruction( - reconstruction, 'reconstruction.{}.json'.format( + [reconstruction], 'reconstruction.{}.json'.format( datetime.datetime.now().isoformat().replace(':', '_'))) common_tracks = reconstructed_points_for_images(graph, reconstruction, From 3cc44ecbc09afb9da2b0a24f971d10d3102bafb9 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 18:35:38 +0100 Subject: [PATCH 18/75] Fix writing partial reconstructions --- opensfm/reconstruction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/reconstruction.py b/opensfm/reconstruction.py index 4563368c9..d2a54ba9f 100644 --- a/opensfm/reconstruction.py +++ b/opensfm/reconstruction.py @@ -688,7 +688,7 @@ def grow_reconstruction(data, graph, reconstruction, images, gcp): if data.config.get('save_partial_reconstructions', False): paint_reconstruction(data, graph, reconstruction) data.save_reconstruction( - reconstruction, 'reconstruction.{}.json'.format( + [reconstruction], 'reconstruction.{}.json'.format( datetime.datetime.now().isoformat().replace(':', '_'))) common_tracks = reconstructed_points_for_images(graph, reconstruction, From 62a4db07d975f75be52509a5c5dd9c9803517de0 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 18:41:10 +0100 Subject: [PATCH 19/75] Undistort fisheye images --- opensfm/commands/undistort.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/opensfm/commands/undistort.py b/opensfm/commands/undistort.py index 0489cc2be..d67536c52 100644 --- a/opensfm/commands/undistort.py +++ b/opensfm/commands/undistort.py @@ -40,6 +40,13 @@ def undistort_images(self, graph, reconstruction, data): image = data.image_as_array(shot.id) undistorted = undistort_image(image, shot) data.save_undistorted_image(shot.id, undistorted) + elif shot.camera.projection_type == 'fisheye': + urec.add_camera(shot.camera) + urec.add_shot(shot) + + image = data.image_as_array(shot.id) + undistorted = undistort_fisheye_image(image, shot) + data.save_undistorted_image(shot.id, undistorted) elif shot.camera.projection_type in ['equirectangular', 'spherical']: original = data.image_as_array(shot.id) width = int(data.config['depthmap_resolution']) @@ -67,6 +74,15 @@ def undistort_image(image, shot): return cv2.undistort(image, K, distortion) +def undistort_fisheye_image(image, shot): + """Remove radial distortion from a perspective image.""" + camera = shot.camera + height, width = image.shape[:2] + K = camera.get_K_in_pixel_coordinates(width, height) + distortion = np.array([camera.k1, camera.k2, 0, 0]) + return cv2.fisheye.undistortImage(image, K, distortion, K) + + def perspective_views_of_a_panorama(spherical_shot, width): """Create 6 perspective views of a panorama.""" camera = types.PerspectiveCamera() From 88ceac30dacc04de7de7e83b974023dfcf0898ae Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 18:49:12 +0100 Subject: [PATCH 20/75] Add failing fisheye projection test --- opensfm/test/test_types.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/opensfm/test/test_types.py b/opensfm/test/test_types.py index a074b6e89..c49a61539 100644 --- a/opensfm/test/test_types.py +++ b/opensfm/test/test_types.py @@ -101,6 +101,20 @@ def test_perspective_camera_projection(): assert np.allclose(pixel, projected) +def test_fisheye_camera_projection(): + """Test fisheye projection--backprojection loop.""" + camera = types.FisheyeCamera() + camera.width = 800 + camera.height = 600 + camera.focal = 0.6 + camera.k1 = -0.1 + camera.k2 = 0.01 + pixel = [0.1, 0.2] + bearing = camera.pixel_bearing(pixel) + projected = camera.project(bearing) + assert np.allclose(pixel, projected) + + def test_spherical_camera_projection(): """Test spherical projection--backprojection loop.""" camera = types.SphericalCamera() From b70e34c910e39c1a930089e133d9364a40efa279 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 9 Feb 2017 21:45:21 +0100 Subject: [PATCH 21/75] Fix fisheye projection --- opensfm/src/bundle.h | 3 ++- opensfm/types.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/opensfm/src/bundle.h b/opensfm/src/bundle.h index b04663f65..57246b0cd 100644 --- a/opensfm/src/bundle.h +++ b/opensfm/src/bundle.h @@ -260,7 +260,8 @@ void FisheyeProject(const T* const camera, T l = sqrt(x * x + y * y); T theta = atan2(l, z); - T theta_d = theta * (T(1.0) + theta * (k1 + theta * k2)); + T theta2 = theta * theta; + T theta_d = theta * (T(1.0) + theta2 * (k1 + theta2 * k2)); T s = focal * theta_d / l; projection[0] = s * x; diff --git a/opensfm/types.py b/opensfm/types.py index 47adc00ab..48aa247f2 100644 --- a/opensfm/types.py +++ b/opensfm/types.py @@ -270,7 +270,7 @@ def project(self, point): x, y, z = point l = np.sqrt(x**2 + y**2) theta = np.arctan2(l, z) - theta_d = theta * (1.0 + theta * (self.k1 + theta * self.k2)) + theta_d = theta * (1.0 + theta**2 * (self.k1 + theta**2 * self.k2)) s = self.focal * theta_d / l return np.array([s * x, s * y]) From 13058fd20354884b71becb2deea61913b4d52ff3 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Fri, 10 Feb 2017 10:18:44 +0100 Subject: [PATCH 22/75] Fix viewer's fisheye projection --- viewer/reconstruction.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/viewer/reconstruction.html b/viewer/reconstruction.html index 53a208c37..f59d7628f 100644 --- a/viewer/reconstruction.html +++ b/viewer/reconstruction.html @@ -192,7 +192,8 @@ float l = sqrt(x * x + y * y); float theta = atan(l, z); - float theta_d = theta * (1.0 + theta * (k1 + theta * k2)); + float theta2 = theta * theta; + float theta_d = theta * (1.0 + theta2 * (k1 + theta2 * k2)); float s = focal * theta_d / l; float u = scale_x * s * x + 0.5; From 213adde9fa63ff35ea73814f62b7abf7956a0acb Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Fri, 10 Feb 2017 13:33:04 +0100 Subject: [PATCH 23/75] =?UTF-8?q?Fisheye=20cameras=20are=20of=20fisheye=20?= =?UTF-8?q?type=20=F0=9F=99=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- opensfm/src/bundle.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/src/bundle.h b/opensfm/src/bundle.h index 57246b0cd..fc78ebd4e 100644 --- a/opensfm/src/bundle.h +++ b/opensfm/src/bundle.h @@ -57,7 +57,7 @@ struct BAFisheyeCamera : public BACamera{ double k1_prior; double k2_prior; - BACameraType type() { return BA_PERSPECTIVE_CAMERA; } + BACameraType type() { return BA_FISHEYE_CAMERA; } double GetFocal() { return parameters[BA_CAMERA_FOCAL]; } double GetK1() { return parameters[BA_CAMERA_K1]; } double GetK2() { return parameters[BA_CAMERA_K2]; } From 73339745fa1846ceb71163a6278e642da8627b65 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Tue, 14 Feb 2017 12:10:10 +0100 Subject: [PATCH 24/75] Convert fisheye to perspective cameras when undistorting --- opensfm/commands/undistort.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/opensfm/commands/undistort.py b/opensfm/commands/undistort.py index d67536c52..f46a7ee10 100644 --- a/opensfm/commands/undistort.py +++ b/opensfm/commands/undistort.py @@ -34,19 +34,20 @@ def undistort_images(self, graph, reconstruction, data): for shot in reconstruction.shots.values(): if shot.camera.projection_type == 'perspective': - urec.add_camera(shot.camera) - urec.add_shot(shot) - image = data.image_as_array(shot.id) - undistorted = undistort_image(image, shot) + undistorted = undistort_image(image, shot.camera) data.save_undistorted_image(shot.id, undistorted) - elif shot.camera.projection_type == 'fisheye': + urec.add_camera(shot.camera) urec.add_shot(shot) - + elif shot.camera.projection_type == 'fisheye': image = data.image_as_array(shot.id) - undistorted = undistort_fisheye_image(image, shot) + undistorted = undistort_fisheye_image(image, shot.camera) data.save_undistorted_image(shot.id, undistorted) + + shot.camera = perspective_camera_from_fisheye(shot.camera) + urec.add_camera(shot.camera) + urec.add_shot(shot) elif shot.camera.projection_type in ['equirectangular', 'spherical']: original = data.image_as_array(shot.id) width = int(data.config['depthmap_resolution']) @@ -65,24 +66,34 @@ def undistort_images(self, graph, reconstruction, data): data.save_undistorted_reconstruction([urec]) -def undistort_image(image, shot): +def undistort_image(image, camera): """Remove radial distortion from a perspective image.""" - camera = shot.camera height, width = image.shape[:2] K = camera.get_K_in_pixel_coordinates(width, height) distortion = np.array([camera.k1, camera.k2, 0, 0]) return cv2.undistort(image, K, distortion) -def undistort_fisheye_image(image, shot): +def undistort_fisheye_image(image, camera): """Remove radial distortion from a perspective image.""" - camera = shot.camera height, width = image.shape[:2] K = camera.get_K_in_pixel_coordinates(width, height) distortion = np.array([camera.k1, camera.k2, 0, 0]) return cv2.fisheye.undistortImage(image, K, distortion, K) +def perspective_camera_from_fisheye(fisheye): + """Create a perspective camera from a fisheye.""" + camera = types.PerspectiveCamera() + camera.id = fisheye.id + camera.width = fisheye.width + camera.height = fisheye.height + camera.focal = fisheye.focal + camera.focal_prior = fisheye.focal_prior + camera.k1 = camera.k1_prior = camera.k2 = camera.k2_prior = 0.0 + return camera + + def perspective_views_of_a_panorama(spherical_shot, width): """Create 6 perspective views of a panorama.""" camera = types.PerspectiveCamera() @@ -155,7 +166,7 @@ def render_perspective_view_of_a_panorama(image, panoshot, perspectiveshot): def add_subshot_tracks(graph, panoshot, perspectiveshot): - """Add edges betwene subshots and visible tracks.""" + """Add edges between subshots and visible tracks.""" graph.add_node(perspectiveshot.id, bipartite=0) for track in graph[panoshot.id]: edge = graph[panoshot.id][track] From 225d570748fb97aa827f60bd617b1e740dad2fee Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Tue, 14 Feb 2017 15:01:39 +0000 Subject: [PATCH 25/75] Fix name clash for masks in mask_and_normalize_features --- opensfm/features.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/opensfm/features.py b/opensfm/features.py index 562c09ac5..50ad62105 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -68,7 +68,7 @@ def denormalized_image_coordinates(norm_coords, width, height): p[:, 1] = norm_coords[:, 1] * size - 0.5 + height / 2.0 return p -def mask_and_normalize_features(points, desc, colors, width, height, config, mask=None): +def mask_and_normalize_features(points, desc, colors, width, height, config, image_mask=None): masks = np.array(config.get('masks',[])) for mask in masks: top = mask['top'] * height @@ -83,12 +83,12 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, mas desc = desc[ids] colors = colors[ids] - # We get the relevant image mask - if mask is not None: - mask_height, mask_width, _ = mask.shape - if (mask_height, mask_width) != (height, width): + # We now compare with the image mask for this specific image if it exists + if image_mask is not None: + image_mask_height, image_mask_width, _ = image_mask.shape + if (image_mask_height, image_mask_width) != (height, width): raise TypeError("Given mask does not match image dimensions") - ids = np.array([mask[int(point[1]), int(point[0]), 0] != 0 for point in points]) + ids = np.array([image_mask[int(point[1]), int(point[0]), 0] != 0 for point in points]) points = points[ids] desc = desc[ids] colors = colors[ids] From 7181f4b018d55fcc336fbb4416190abcd7733ad3 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 15 Feb 2017 00:03:19 +0100 Subject: [PATCH 26/75] Compute meshes for fisheyes --- opensfm/mesh.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/opensfm/mesh.py b/opensfm/mesh.py index 260bcfd0d..3afc5db0d 100644 --- a/opensfm/mesh.py +++ b/opensfm/mesh.py @@ -18,6 +18,8 @@ def triangle_mesh(shot_id, r, graph, data): if shot.camera.projection_type == 'perspective': return triangle_mesh_perspective(shot_id, r, graph) + elif shot.camera.projection_type == 'fisheye': + return triangle_mesh_fisheye(shot_id, r, graph) elif shot.camera.projection_type in ['equirectangular', 'spherical']: return triangle_mesh_equirectangular(shot_id, r, graph) else: @@ -76,6 +78,53 @@ def back_project_no_distortion(shot, pixel, depth): return shot.pose.transform_inverse(p) +def triangle_mesh_fisheye(shot_id, r, graph): + shot = r.shots[shot_id] + + bearings = [] + vertices = [] + + # Add boundary vertices + num_circle_points = 20 + for i in range(num_circle_points): + a = 2 * np.pi * float(i) / num_circle_points + point = 30 * np.array([np.cos(a), np.sin(a), 0]) + bearing = point / np.linalg.norm(point) + point = shot.pose.transform_inverse(point) + vertices.append(point.tolist()) + bearings.append(bearing) + + # Add a single vertex in front of the camera + point = 30 * np.array([0, 0, 1]) + bearing = 0.3 * point / np.linalg.norm(point) + point = shot.pose.transform_inverse(point) + vertices.append(point.tolist()) + bearings.append(bearing) + + # Add reconstructed points + for track_id, edge in graph[shot_id].items(): + if track_id in r.points: + point = r.points[track_id].coordinates + vertices.append(point) + direction = shot.pose.transform(point) + pixel = direction / np.linalg.norm(direction) + bearings.append(pixel.tolist()) + + # Triangulate + tri = scipy.spatial.ConvexHull(bearings) + faces = tri.simplices.tolist() + + # Remove faces having only boundary vertices + def good_face(face): + return (face[0] >= num_circle_points or + face[1] >= num_circle_points or + face[2] >= num_circle_points) + + faces = filter(good_face, faces) + + return vertices, faces + + def triangle_mesh_equirectangular(shot_id, r, graph): shot = r.shots[shot_id] @@ -100,4 +149,5 @@ def triangle_mesh_equirectangular(shot_id, r, graph): tri = scipy.spatial.ConvexHull(bearings) faces = tri.simplices.tolist() + return vertices, faces From e89464955f52060e9392f1dda28f028c26ea2c23 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 15 Feb 2017 17:44:49 +0100 Subject: [PATCH 27/75] Convert masks to 2D arrays --- opensfm/dataset.py | 2 ++ opensfm/features.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index b6dcd3d33..9116ea7d9 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -97,6 +97,8 @@ def mask_as_array(self, image): if mask_name in self.masks(): mask_path = self.mask_files[mask_name] mask = cv2.imread(mask_path) + if len(mask.shape) == 3: + mask = mask.max(axis=2) else: mask = None return mask diff --git a/opensfm/features.py b/opensfm/features.py index 50ad62105..ecdc43f3f 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -85,10 +85,10 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, ima # We now compare with the image mask for this specific image if it exists if image_mask is not None: - image_mask_height, image_mask_width, _ = image_mask.shape if (image_mask_height, image_mask_width) != (height, width): raise TypeError("Given mask does not match image dimensions") ids = np.array([image_mask[int(point[1]), int(point[0]), 0] != 0 for point in points]) + image_mask_height, image_mask_width = image_mask.shape points = points[ids] desc = desc[ids] colors = colors[ids] From c2c875eedeabd198d5b59a0a63ddcc404c37fc81 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 15 Feb 2017 17:46:47 +0100 Subject: [PATCH 28/75] Scale point coordinates to fit mask size --- opensfm/features.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/opensfm/features.py b/opensfm/features.py index ecdc43f3f..158db6878 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -85,10 +85,7 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, ima # We now compare with the image mask for this specific image if it exists if image_mask is not None: - if (image_mask_height, image_mask_width) != (height, width): - raise TypeError("Given mask does not match image dimensions") - ids = np.array([image_mask[int(point[1]), int(point[0]), 0] != 0 for point in points]) - image_mask_height, image_mask_width = image_mask.shape + ids = np.array([_in_mask(point, width, height, image_mask) for point in points]) points = points[ids] desc = desc[ids] colors = colors[ids] @@ -96,6 +93,14 @@ def mask_and_normalize_features(points, desc, colors, width, height, config, ima points[:, :2] = normalized_image_coordinates(points[:, :2], width, height) return points, desc, colors + +def _in_mask(point, width, height, mask): + """Check if a point is inside a binary mask.""" + u = mask.shape[1] * (point[0] + 0.5) / width + v = mask.shape[0] * (point[1] + 0.5) / height + return mask[int(v), int(u)] != 0 + + def extract_features_sift(image, config): sift_edge_threshold = config.get('sift_edge_threshold', 10) sift_peak_threshold = float(config.get('sift_peak_threshold', 0.1)) From 2dccb849d80a28122bda961aea9d8d659ed1681e Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 15 Feb 2017 18:00:52 +0100 Subject: [PATCH 29/75] Name mask for image.jpg as image.jpg.png --- opensfm/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 9116ea7d9..ab09ab178 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -93,7 +93,7 @@ def masks(self): def mask_as_array(self, image): """Given an image, returns the associated mask as an array if it exists, otherwise returns None""" - mask_name = image + mask_name = image + '.png' if mask_name in self.masks(): mask_path = self.mask_files[mask_name] mask = cv2.imread(mask_path) From 663c93042cc788aa4cbc0afe1d8d9cd53f0a1da3 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 15 Feb 2017 18:01:47 +0100 Subject: [PATCH 30/75] Add image masks for berlin --- data/berlin/masks/01.jpg.png | Bin 0 -> 32223 bytes data/berlin/masks/02.jpg.png | Bin 0 -> 33954 bytes data/berlin/masks/03.jpg.png | Bin 0 -> 29138 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/berlin/masks/01.jpg.png create mode 100644 data/berlin/masks/02.jpg.png create mode 100644 data/berlin/masks/03.jpg.png diff --git a/data/berlin/masks/01.jpg.png b/data/berlin/masks/01.jpg.png new file mode 100644 index 0000000000000000000000000000000000000000..59cee8b11682c2a0047ee1ea823602f248933821 GIT binary patch literal 32223 zcmeHQdsIzZ+kbZLZo2J?3SmboBq>x#JC%}3u0_Q@shnK0D-u#m5<00UD&>enkxRMd z&T{6e~$5uvDO&uHRpQfoX`Bt=lRWN&emnE zpRb2@r*549KzppG+e82@7C<9YU5VuQ9yB{depExodAN~Zaj7g^2!m&Q?YQHJzitJUV4qVLwh+(Xo%jCGbO?BrYTm_w#U7TC)ADSqMRCP^= zEfQ`qsxLPp<+#IX0jW%J4AH3kathmpp?{uFP_q4U&R7b%emR$9Gg@EH#aH&Cgh&k|@-k@c)z)TEDxkBSig$ zZGpPcQS=nRrtjB{J8zAzdx4Xb{tS%S?5}%%yg2a4@J7HJ0Z$yj8v$mjCm}lT`$iN5Ys?pnPlG^vF9e1Fo zP$c|%TVKBwf8NR`_4<0NMy~$X`)_!09vR+dfHwl(2zVm^-UxUj;Ee!yBjAmIHv+QL z&94slz>qfremdZpfS(Td>3|^ck4y(l?eUccz=+gCY7=kFFi1!r>lM&!PakmkrjOZ> z{`2<-QxN$3gAoS)p#cKIH@rW1Ze#fHFZTjavn2oha|B_1B~4GBzp_5~FVBOveZ6;> zefJ7K;TIMLEuS& zfWVUi0f8q4^anmtARzFhKtT8}NFn3&vO5ud)RinZr?+>Ut((SFx=NGsJThK72BvlO zvK}pR8E-iV0&h741m1E82z;!9{=mm71O%QG2nakW5D<7$ zARzFhfFSUsKtSM0fq=l10{R0VDG(6;AyP2E>ADS|hbMj`0I}!)21r2oR(}JG9D_s9 zTsHXPC!;Dy9}P?e+>#mKM;YXkwx)!%99O0vQ`kF}-Y_A%W?yr_KJsUj1I-T+^lfdu z?v(Vr_kJJEEAK5v`MCR!FXg1rXF3Dc(a$u7 z%OYr%XI^p|FL-OocEsyeIcnnz;^TCF*`k9D+gAufBP*qcZUyZsolzW&FLVBp@d%*L zxEhsD+47$Km^RoX5O0bL+QLDxQ6${kG3)HJdkEFB{rP(22cV;MqTZqpStb?6_m8Le zye`eLf8{rhN!qeA_S7LJj&t}$4PwHp;~ez-bj zrL@MvamJ$uEhneX>_;xKQxtt0xYFWzwj)X#al-|}V!cMZ`_*|V>plnj%)H~il~MR2 zVz|XQLoDmh<%s7O|Nh6`O>>W=c-LT8t$aD%nnt^pKYm9tpq;Z6@k@5~{SFn9p_peK zHXmMfw|M#0t}}-O>S9U7P+%E_2(zo$+3>O{ecRtw?#msEN_1+MUGS~5)kLsmTe(ku z#D#N%HG+GIHk(yLpKp)n`o80m7kdz)&E|CWEesPZQzPjzCHnP{>x&|NwujM|ZTnhK zo7+D1_oEk$ZEbfA97V*~iCnc7lMa&E|40Fi8~yd8EQOW=>YDRw{j>`%p!6&au7g#5 zXwQ;+Q@z1vqZU>uTFuqV&amsqY5(axF=_cEwht18HNx zYz(Cqyh=!a4o?iX75cF6&}F4!PK2BSvz=ZCxOGSa+b&D~Tb}?-Ym>3Rx#R!VC#>RY z9hgYfr`pwe&%GkXG`OmBeZ}+6eYuX}xnK6bX$dd5bKo=;9|;#>nt7&Bal1$t{m9&a zwh*n)v2SbE|6UJP@gExwoU^YhHA;E!e}fIypy(H#f~D909f-dckoxZ%x{_uDnf?uk z%*=hq>9zDYd#Xz2TIh=z6&xNC{q3_bdFo#Stf=8CKOKx|U(OtAbE`z*)Q|i*z}bc! z6?omj=kLna=02y_j{V(AvB>{>xW01U>AM4ObGB^iuM=a&h@0+BH{4612L^w4QJjU zRyH;Bpadl3#?NzfH=HdQe+76w8&+ep?6?N%^ z!)yO(sLO9pw^Pq`M9~5<2Zc@G%X%?p=y3xfaWC(MEG$=yxyslK>|C?U-^N9p-*D1} zv6$zKGYb3g)%W|3+-lIK#y~Pa(qm@gca~fut7;Pv5@{cU&5j&BpadN7{@A1Nt3R+I zdt%d#sJoDhGh~f7!e7=)1fUf(v)+R?@42hxO!*TsK>d2m3{PwyJJ=Q&(;xN(DhLW7 zal|X;O~u*XQ&d6p$CFY86|LIR28_C5HqAySM`B_XNXeVtH()wcdG z-pC6!8gXoKWm7dG@jB3xCs?d<*qIEMm^tag9?9O1NVyxN}FG*(q|<)kUF`{s6e zGz`jaRQJ^$fz(y7SJ!E++Joj??P0;b#zLqo&b|jsSamdF!A$qvrp}x+)zOAy?n=-( zcxT)y;E46sb@0=u9f2#jfkS^T|12&xNfd-8||EV)>jMKPNTTuP4gu-(GZM zRLYf1X2h{_$>yCu)F6#NxcPL^nZ2~VBEy1&k?0x#=cxrYc6Kc8J4O}e9c4)G;Na^{ zB)Z!5;@Xq!89NuYRCfQ4%1XPSBkg*0zTB`^qI>1j7b_3mR_}0F@2yWS*QTRllcQd( z^V~P|Rbhmh4UmbkL>w3Pp0QggGmichR-*KQ zFihyQyAAI{7d{*Mrd~HqStosV_qCF4}gfsLT8yA2S<{BmGr%sfv<(+VW3 z-evR^0jq)6z^dph32T{d2Edt6`L<89kHt?@(`yYKKUDS;@t@%WNTHLKSomqV>|aD! z#g4=+gSVndP5nbl+Q+2eDafsnz~?mgKq`{%Onbn8EmxcG?j-pbI*we9r%3oBl3uEPp(rECGd8e2bcj_lu*)kJtARrWQfualtL9xQKf$qg%3#=? zd@*sh8fi(|{ljgr&Tx1D%!L#e0ON7UX>ob$3T>XTX<=Qi*&@)!moa#eYh41y;m5{^ zp#8h;?g?P8eTwd?BqgxKaDwjubKY=97ImkWx-D$u_f z`3gqY?0sVY^AK_daGQ?FWy1*OLDZ(cGiQ3Ak5|r7sampx>d$|38avZ`U^ ze>-f8o%_HoaMN4socQIz)1MfjtuiON1Cx&~%Wirbd#NHO%6lFd8NlP3?`-@4&^5|> z4IKCSRgj5YSN4bs)3(r<;6n?<8orNpI@MUTBy5i0nC8Rl%98lku^^dB$7+pt%rJZ1 z3j#J+AG+8B%*AMhv0ONCo&L_+xU`9foUWW5z3RAI=EFxa3^Qu5N{G^^3VXE8;jwOR zN0;=Q#}6M)8w&j+AF6^)omyEvL+DU)0l~&&T3Ct6t#?$-t?UzgK%`fYxr2U#%Q=^A=FpMq)maWOkD%0)i zpF1*bi*MBabsmr3H|sVfE6v7v!Di|epUMW4IO3dB4CYwe%Mf_HZD>hdp7!DJz2#9v z0BYUOUCqZiPO2=Znf%^V`9bSa$LZ1M;-4;UeYILQ!PkE}ishEx9TkfBvtw;TC6cqj zIWOk66r9vQk)Q-+@sH-cSpdwFXFEOeT6R+Co?Oa|+TM^Nu36yutYWV!NbjdMh`+>Z z#~RbM@~ns1oO|b)_m&JGaR&xn{MlH|z79zfJ4=)*`h;08B@8{Ivbzvq6r}`Zt*XqY zA<9wesn~gVdb%s27^(U>x)THp7bpEvi37(x=mE@WR7A3H+0fWQ3Dbj{&Y$W;S3;n~ zEv~Q*E;3~eB9a{}6%@9zxWIJsrBT)4_+GNjftl zT@5mS#h^iMzVAr<)4@v|s36$tiIqWaxOU}K+t6#LKgcfh=eU2nk?y(wGt zWXs&*yZNZ_8t<2cB#-o@3awd{Jd8w|z>eN*r>Mim`^d%ZM;%Efti6;eV8SGNShZ;X z_R8Q*zn?}GPF^PqKdwjTh8Uu*)!*-X`X1mojyoIIV1=s~h>o&x#j8E-#Z6H@`xRBa~@ zD#uLe@4OpZBtbV*Ix( z1?rWqCTItKS*_`qp8fkW@{M201(^J@dyv)Y(mxz+Zdbejw$r{Yc}qV3i^qfJrpFju zv$E5RR~DLM=B}TP6kSt>COQumdDVE|?4m-lTfL5wsqD|a4;C;Y;LPXH^22&VX@wK3 z-wWWqVpoVz;RdpFD$_$9@o!w(<=fnhI)Zo?uDm4Q|QR(rSu>gGT;q_Q87+)jjP^r?|%g;u5^OR{>e<3U1M zST^J3Cr``%Q8KR<8vcWHF14p$vyAVYOP;p-fUP>#Ozb+LX9q6G$b!`ECz=Jd=}7nT z*n~>Xoh06WH54{B!P`to#Kns@5e+~&-o#PABuoJ6q3HUEPL>PdD2!?{%{i0GPcQkVOw-gJYn zu=8^+HR71uz3~EXuq)2DwgHqC=LZgv9zT*9EB&R=Kx0f9h1S z3+M1V$0Pnpaf$fGiFqdy$ORZ_FVbDstYpdTGQOt!m_-JVLtE*D8B^d;3chGUQkk8i{f{+0CA zRG@_NB+YHUatX!uAx&`mjaJvm9*xR7ZXXIDnOd|~A4BY)*-!NX*kM72j2;?eM+p?O z)NgT@xZ#5;x@=~(4pNf`Q6`SKp2lWb#SYt2uG|^wM?;hIxFokHYrOl2Ve_Qr7vD7! ze#PSqda8JPuBlmcrm0;M&0{TaU|aJ}Y0{*`{KwDjNc@a!p7UeT{b219U1E=Z%azHe zGO6SWX1#0L4We&t4hcZm1&8-`&|s?bKIM&TNF6)OoK&H?Kuvg`GMJYet+!C5t4^DT zY!u2&#tlH7*NmJ72$sgI29i^!K-d-c)8f)ZTJ*T)*%gU|ImuC3F-r+smu%eWASCI8 z!dYZFL^r%TJOZz8BZESYU6g1mpSp>?Q>=cTfLT)p0!y8=+4*i#F6U{Ie$?n}qN@G< zN3GSGcBYL878()~qbiH8mW+AQp6EaBwiz}RQceDX@CRj+3fJ+n7sd@vcU?bs6eSDD kRX_6AtN@oO0VSW*q6gMRh2OY*5L+GV?(4RHv^42|0625NLjV8( literal 0 HcmV?d00001 diff --git a/data/berlin/masks/02.jpg.png b/data/berlin/masks/02.jpg.png new file mode 100644 index 0000000000000000000000000000000000000000..587354dc53f650aa43f57e54e0070970fe626f65 GIT binary patch literal 33954 zcmeHw30O^Q`}ebVyU}hgDy3aC85&6lJCUM^GKJcdL`7&8R#XzrNm6W6DpSUjd7EmJ zA%v1)SENWMnPvFyXRWo%>zwy|-}S$~@4DXq_utQTIjv{9pXa%U-+ixpt-a0$Pxpz6 z3OWi1Aw?JG@lz3E3lNgekYhrN`_Tc%;E#Nm^Q;vJ^-{yMPCtGK9j~}ls6gQ(??11j;DKggLYr<4YTYSyUk&s77!W_^CiRJ(kBxVIn zkbnY)M+qnZ6C|JjOpt&AFhK$ezyt{>U?xaF0hk~G1z>^%6tF!=A_ZWA1QdV?5>Nmp zNI(IYAOQu;1PLer6aG&?Va%CG4rUlf_$)SK<8HH_tQ>DrPoghRttYux#Og`tNrp=} zLr9cBq688pKoTX8D1k%?kVFY2N+3}J^dG7OlPB>K#P0$?)GvwUC3iUzB|s7-kSKvf z2_)_S{r{>2oXGZCnfWLwUhC}7>aW^sy}$II==q|Nhckb;>|*`$!XC3Hzh}z$<%K<> z=Y7CQzq}7<)RQJmR{Q0BK>41Op@cl}Si&4&0!N}b2oVyql3)&S0Ez?@fC&;%047L4 z0o#KlQUE4MKmnK_0R>=!1QdV?J)qDrEovbPA>YEB`UALwEGo}`{Z-Nb_?wgtllEJ3 z2-%-8_+wW0kFz*JS;I{Y6hfnP<4 z^<}p5@?|VbqnLBh}xz>hoHvm1_=+ zw`QvFSc~-O`TIe5=-cbt>{+iewkGIOtsNVXwqSFDP4(DeIZNs9hYm30$^;mGLP@lA zQ6CHrpIx8QD+Y}t*v|bYZ+C2G-57CRha!}bh_q3SiDiO0Eg8HoBwFd`HGTOEdQVoc zC#7+H1s|_fj;G~N(;pr`T-`8w5m%4?8tG@)5fbPJ9Hh5h3NmDkACA;jsRjX?2@f*d zT&~jV;2K@rKpDi>LydZ#YgFd|NxE>oew)BuR=F&Q-VHkc+bqF{3N0Nm&1BFS^y*#j z$uucE*fP;k45Gy6-|#f=szE?PcpM&{$=2Q}6z>u<|6lf1o!m|BvopcwnJ+ye z-q|IP0pT=2ycygro{Hg$Xf6069(rW}9DT0P8h=|B%3oHYnt_MEv8(KomScZ$=N-_N z!G?}W{oox0T)0Dj;~UkC5N32)T9h1fr*7p=T1I@p193lr-AwLsIq^wVz_<{1;!{7` z*v?|Lq|0$kEB}v~^v;n3?mOP}eKXZBNX1hv?)uZ!QE+a>vscBqm(2ZFo*=^M>;<}$ zr2g9mKWh1RD*?~qw@!L@2s_EzirItd{SxPX6S#QXe~`x|zV;|EarmG1 zoAU0*B>DiW0RY5j`A6Fk5BlBM1h&7}6tDk3uya>m|D12W-O7MMO#c1S+@NXC#9D(? z;(xOp@m_w{>MO%4s%afY2s<$Gz~475ZpR40_4H?_q9n0qdHrT7;-MsSa$bn_6Mb|4 zO#|tHqz#BC{Uy{|tka`=tN-Er$TQ>94B9lpl>dRL;+04}EYYAAvzWHlr!< z5{0*~=(#`0McN{FaoD5&B39=|>%HsQn=xY-yHQ)=<+gdeFL0f{y+?=U9Il<{N0CqBju$bmvo@&<5qB1Sv^gc;j@j?aoYvU$0iAN!{n;Gdl~V{97o|Zh zLcARPsr=q4x37-p4j-6df;scbo$Bfr*%@k zi#I4o*g81c)qkwtC!xFktryd}W46csZuUXn9M`?>m%mo}>?Ji+GaG0Y%w6$i5d;b| z%?^B7d2md4)_BETXNpeEG}jbOKb`5K)g*lnoOjLaP_wfTTuqwCn5=xxNLp{0G=x>{ zyLo4U#LR_mUu@oFNg?9HrM2k`TOB&mN;g(~f`A4Ju|06CqR!9i8iX>I9LXro`lBUG zbEp!IzKFZA?6nCrZMtqp(6iNV!C8f^v%fR}@mYR_m^uh?_;6_@+5wJ*vG-yd;I2072T1mp*X zJ?rD1mXm%uE?C}+8-EmJ*I)Xba@{_qRr1JF)$9m)+!V*CyK+e$i+$&TbaG4je2cKv zL&QnVJDYAFE(e4-V8;=4eEF#RJSmHT=FFaBwjab1GbqjDXR5T^ z9P6CUH1eLX_=4xGZ&q!fL#8l*6MgQW>gq)*8&&g$U%R5~#|$9Wve9GTeQ{c|*@!p| zfXeB*blu$|TnqXHx9U`oY23<)Kp8}=1$8hjvyAz&lpt?{vtj{^;cVjiHb zbvM;j<4FDvFpR6_gb>}r-Ea0zUqJgF#z}0<7=^#$*m$m9W>~wFZ2Q!xDD~C;F zHOX-v1+;7h6G12;;%C#QPe1qSjU5xgp35Ak%8|-_kc?2&^gbuadj(F8&25&^HDtK* zK1)zBNfm`no=yOhBmj!yUs*XB`OUzbeDH{qQ2ev>ZhGv!BaCm$bd!aUjv` zfnvd-iHkuWMvX-LTD&Oo^$l#UIR9gUPhUp0c1ff3OefICy`=CEciBb1*=Wr$EtA{oh>f_si2)qT&5)w|oWF5k~T+uBcL0L-XtRNG`H zNZ_}E3WGF@Wmb71U)ZP77F$`t8?F=+nWg}TWt+dApX5jMf-N1~IN7oVxt3xsW%$&G z9@%%H9D;l7;qf{jz`79HQxoTSV>OS_8ApY5Z40+K`U9=?K&xvJix>h+?A6qYBI>x= zSXsE%wij;=P4qg9#$a>V!h6i9DG0oI91%t)2u&KJ*r~34RBn#69NeX%1Fx$!9Zaxc z6GPi#8DT@fOgG5O=<(c5pI*|@MxF~nM0f95`;GXaAOaz2O#Lv~b+f*V9xC!fTzM8E zbUeCLQYy<;7P`WLik!=qFf8RU-f+Wh%L6+sr&M(5wXmK)V+TsU}p&oFj9bLTmt{ft+17|+; zwOH$>qaBNJ+S4sx|00jNkKEOf9G_(`NEEV!d~dUoeD65kB3VKk!wqpZWM*iHi?a61 zJ112NkhZB21Jz7H=z2ehF+zkR7;Yw**d)@n5!x2>yw4t&1wtZ;CFGjTO3hN*4Q4eQ zpQ*#pG56azR3(=Y-#64OWdsCS(um*8K%S+`vMSR?jyRrwfF%?R0eioyi6_+8=B3n~ zwGKEvk`>i}BltXBhXx9sJ##CEqYD7#)7(|z7LiaRo69pMW^6eo+sjHmBfh@$t|H>@ z#7P-JzmL1uzT-^Yv0e%EYXY743gNiyE@2pgnJv%mhoubV#{fr}4!B+PtQ%lFT83vX z{elD^%gw3ptLoMz2Ep`19VA%z(E_0eko{ttr!hn2Ym7n1?@$zouL&Q?%);XQxJgBo z;~Np3H@ori{k!R-UOAYvEZ0FKI_Fv=gU{~jgNTtJmucGiF>IDM2B<1bC-&`efQao) z^Y$jn=K$203c<1jpW6IzCwH{Tr|)ftq&{G%3lCUD9-TYeI$)SND@+q&fYeJ7d208h z;poIw`+W>V*rYXeI13-KuESyPdh>J`f;|ea>mU&PV>!?x9^tIZEm^(sM#yyoUI96L zE?Ku9siZ(qU&xpGI;5olxXA98$3Ui+1$>MIoWdo_KSss*%~KL2NL|FVhA+dkE&(lm zT2r}g3&)m`H)9y37H8ibQz!bu)(p*L!~M%o4u1s!7!+0K!SOp84EYR)F)tj^!Kq4y zdWS4p){}im-~>R#Id)r^xeJ}XS=>CjJjL$<1HICPZvLxXz63jYzT0(&+_hPHeZ%eh zkZefyh|?$XqIxr|ol9yjH=kViTu!mP+R}gI{THyg_0cKvh=|p~DGOn#>ll7$wZo;$ zFOqtV%6p9e2RwjW`Ih=2x}%mpQmviuU)^A*bX9hwZ@o6U0WQzY^a#-C+M>LD4!Zf$ z$|S>Ke$4B7}XDVHXv&uP>iL;R_nA*>CU*AwGQGFwL^D3%-%fpu-hpSh1 z!WvnJn(b$-9Ri8K)~grRRoJfxdiZk6+v+`ib@j^|c|PE}U1D9&!0=ct%ka1@_ISPFjTq z0~F3+v^WxMpK@E52lJ%f+NEWOF4%@~AouN1472;aqw|sWd&B$h$Il)e9`*1Nv_Z+( zq8>9>ItWr^BIQp5uH}GbqOy(pksgbu)$74^b-)MQ|06WO zf1>XwI7e^eF$3pJz#*hanM{WyUrw`!enBEYhPb?nZjp_K5D^MjCr$sJ2+p6lL0SGG z_%__Qyi5<&jCe6{&Qlxd?rHt{+pu!Td-;xSO?D7wKoK})HpT8r(9jx1lVe6cKLP6z zn(8-N_Abqb5EFFq7P*D#MV55xQ6+zLEX*A#9jf#$I?)Ie?R)2Ll4Nlvp3wPqMHG1k zm^&%d>wDss(^?rNc9nN=buDF+=&SO)sZXf z?vNWnBLTz^aTBW~F17TIlgFLx2k=u6vowaOvSKy4?aw!S!ArrWYNM`CNMhq$HbRj{ z8Iy zah+OBc_|sk^K4goqP7oKdV{D@g1NfqAfhzTwnl65)~WJ2bH`M4f;{rKsz1kIvNJ0J zHIORFKRiaz&pP1O6j{A6V{gaXA-PY3os{a+fu9%2wgF9l)>>UO-Uu6t&MCRD@`XKZRQ_sl|dc4OW!cytdYG@HpwHs zx+VCi9KCHm=mw`-A|rGw4V+{IC%Wy9qYj)mC*yUJ`Zc>0HZggVRrdfy=6Z48xUe`- z^5R{LnK=88f+|mv2)CfNjVT(?eQYGo@DZj3g?L(79_CP+733agM-Ntn7OvIFv&gQ8 zGt09Lq(qClk4LBD1u{?lPHohmPntA}BexiCFdFL?cizbh2 zgPz>%qQRdGdr{EEbPLSr7Qq-TlulaJt7E}3sQn2{+9g9Q?UpE&z=x3jrR)|=INaZZ zO>~jno9oP^C>M@alVFqc;-U>I-;7z@0nsavGEOgGpk_M5SnPl~Wa=CXxAOP!>i9Vt z7)YwgO(ePnm|DD-QJzgUrL3|-VQT<7k!Uq$0atIkxfF56zG{;diDnw4@H84uS)=lY zsa70>B1WsypO;!ue8u+{`Dct?f~h!Qc!h)!^lkL3^% zhk>S%TFjXu;{d2J4Iqr|Uj&X9_T$0lx2iXXkk6BeKEq!p*eta2cfXOv|wj z3)h9@xM~a{VH%YYGw?%@1`L+C93c6B#Sj$X@uE%CV;1*cz><36Aa~`hKRFx?BFRl5 zr*~ZV*gm*qxvm`F)VTI%?tT6H4qmM99ee|?m|F^^@9l-16JQ6%>KJMbifIUKCabSY zDiN_it;tdjlfC`RFRW7NF9cnyU5yXQ??ZiHC}RW32Y&xwzBfewI*p@PPafty{kz5G zo@6!Lgu*|OSqy$qIpCLHHJZ}jm>IxZHwd~2RThZ_-0)XcW3hlOld%M6-ag|573epj z`pL=!YOh3f$aI_zuuQ{40nR3mZvKSDpY+QYjx@+M$CbgY2QLYavfM$Bb|wwaGM=j9 zF24AaFHdSm;wWLcD&?UXlF8_`S%s<|q4-!t_r_DzKzh+BRFz_RH`6JWBZQd_C6^{u zovn*~h#F05V=FjONJG-qLDb;hH07$|%5O)ZTi`?eB#@_=n)$2`G|Y6T&~Px{93MZK zOI2?akujOvII1d8ndE?bJ#aOl+Q&u?c_tKN;d7KIiCU(U0_JGY0^aQ`18T#u`>VWp zYTKrKc>ULp}zD%Fbzt{A&Gfx735GMlEAwvn;LwRgHS7_k0%ZO zMkvgaN3l@ks6z;(cVR0f3grNUc}E`PJ0NtFGUN~^tTaGbCu=*!+!fjKxXTAT2gUq5;xmh{iht0AgJT(kepxHVs>al=S;JbWUM z3>@L}v!pEo1t?-Z;_Gf8f@|)tKLGxlP2llJjM(^^Qicbk(wgj)nh>g@S7h}KY>%0_&hHRJH?{g@?0a$JoXnttakZpk4{pz$sReuzy0>OH82 z+&&NY0jP{L%Vc*)8I)ZTgQvVAbyGe_hfX>3BGG+>9hQ z7;oG)uE7#3r2W{Rcmq8msZ)^alR_D*z!Y2RzCMvw?=tRjq4n-i~neh zb$+UN=k~N>_#W}nP#OsRwRtqW#MJmhRV4*ih1UTs&|T8}Dg1!0K++7jTw2aaq;7Lv z$wOKB9bpk8_KdYlfPuV+3qxt;mLk!cezXfG`JPi`0SYib=k!q~4YAVC9@|ijh z3O6S2k_V4rLo1ZIEfw%zjvgr}g!Tok3&YC@R@3L7rC^|o^IK)wc+~?@y6eGcITqwP z&#|P4t2O=}(bMEn`vtoJFpx08Gt067>4nag3FH%65*vsGzyz-EQ3Ib~CE%J(B(mk; z`NME?z!1-Qp`V7Z5Sf)+xa8}G3dF0S8Tyi}Sr>-Dz ziAmJ8-DRM*%IN_sf0kp9f@~wBKg%Mb#4F5Vde>V!DHJ#=ZwUkORx1e{@eDMzbEG=D z^mO;R-PU(7ZipS^=vB=VCmr207N>+fNpUs5RnkPJ6I^oJx8Q*aG{*Wzv}St;S*^kD zku!>tNQU5#N3C%l9}TlhwfDASz7mI(HcCTCu&6Wi_NLYLp!voj&F-M`g^kSWEqWw4 zpr=oNC>^wozE{*2HimA|up77UJYFv)dN>RI;8Lc)rVm4)P3nYp6$J3OoTCtiye5rQ zz^+`TMNZ*a*ME`f7=-thu|XQAFF(!v5{Yk$jK*d^gqy@1( ztyOrk*z4kJ3Rj#3pXhrCC-MMysL%qzQ!U!AVJICd9s7&#Fo(J;Jam5m<;lLUxGSogS`S5|b@Cs^MJ^r7hH zh4dy%rNirm9I^v_D{Y#siKn_2UJn);+rOlmLl#}>B4d-Xm-vY{+^DP{8!DaRC#85P z7;iQ2d(Qm571>n2*E0E&(hR*M95A?E85Yp50Z-s*4ayKK_hkf}4*q-ssyFAH{icw% zUKRHlZ#-ohwR-1`U4|L&J14yF+Wkxo0yVeF$>yJ;Nv-7>-_aC==Dy$mc+mH|An)Z) zzJVXE?y295aqER zxN%^|mZe8ZUo>DHB2Lu!3kFUXURxyd#$0)MxV4 zyH=1oe#OWl_z`w=E38T}yxf*AR2p zd3ROXcl-<-0_>^j40s6<8f)VR1BW4pwZ|W2+&N=_d!dL70D*eKeUU0uNjRc<$3S%t z3C`qTW2gpntFK52sv&>z+IS>Ug+r^AvS8F#=KL@5hNf7B%L-=%Ddr(E%J}Q2`f4>~B0dIgklXW=Om{NSqiN5Vno-tH4pEp#cIP9qaO#_VPq}(dx`3shQ65QZwqr{~Ie} zdre=Ikxb()K+e@A0fM;sTIG?)B5%>8E%US3c`nLQFIQm)L>8wm^%bsF!ZG!#4Lay- zVE4z9nzwp}pNm#LWU|iiy=7R#gSG^07f^Ub%QPQm4;MYMuH7_1=GIe2;Sj0=x!Ue} zp!4Z#*S8BF>n}Lu$noLCMMtFB1H)MVM=P?|m8`TqUxd#x6L0$8H+ER&vt>ix<+Lz8qoB87gXfa-~Fk zo)WWbqKTzic1=&{@DSKNP#Z0wqhCPISO|aGDZpNkxS|d$U)5#2gEtTs0He|4Zo&SG zhHopA*G4nwm>IaBuVWo%E->nrWNpe48AyJRNC2eD`a+au(lgvq$6EI%<=4m))OrQG zIY|jf#HAXxCb5Y=N}s6g1}xoZr?gJ+@rq^Gn+GCDA(4fc8W0fT3R3TppXrNZWmB>k zJ)6Gkb@gpvmsR;klUD_0A=AA9lr*DbE_l)GA-oC+r8c z3Os;6wS0ee$-1NiN*{XJH>l^*?VfXY{b>F+cAEF|7U^4M-QObX_}F`8)uH;Bb*qH7 z#y2#>YQonNSWD9F|Yz^Ws1UquZ#_w_)y1dJ^%j z{r(KAu7o>ATL(%xhfRy>RtZ$x+RY1i{Q5UH)|f;O(*0jUrhic&$pXK}NR;6IAA5)Y diVS`*GB({~yk8dq4+0_aIKh4Vp>e#m{|m#k_fG%- literal 0 HcmV?d00001 diff --git a/data/berlin/masks/03.jpg.png b/data/berlin/masks/03.jpg.png new file mode 100644 index 0000000000000000000000000000000000000000..4c27e47eaa8cd31d3a42a3a5a301293e1ae01af2 GIT binary patch literal 29138 zcmeI5O-K}B7{{OeC|%p7T`MY!c48BXg|dPUX|U#+uB2Id>5zDc>Sj`Cba4!VB8oi- zg+)b&5Q^{vO6cH%1$&4towC4!f`#mZAWWj@d5ziW7JD$vegop`GCS=4_xJw)@60^U zdcL`-E>u!hA|jzwee!@vFe4I37kl-Qrr`}E`YUj}{@@9bCCki@NBXa=(g%x9rW$LD zUVE2@m#_PlJ2ow1Nh(?0dTL_o;3QgtY2Gh=BhK4 zo2kTN=Z|o?<#Wd8O4A-^oD5{0q1^Hx&X4zdw>e{bqi3}kNDX0jTf{K7CnEAj; zK}En6R0K>xMZgqP1Wdt*fGMa5|C1?HO-=Ng58k$V-c@hDB=)A>{Dr#q2KU!o00Lc4vM`)CYl5~wi{GgQFl+sJ^TK#BMggV@L;?Y} zgcFcWa}ozu*ZYJXSQ6>6FV@PsigLG%Z*RaY-D~mY%k?sSZiybd3(5ci5C{+e0T2if z009sP5C8!X=xG&a2P_PMfH=Sjh=acz2Ya-!H!?0k-_WxkHQU#WcD}8u@Wxm4m`>r@ zjn-V1fxcxGfv#&6p|Gy&g-A}@;8@@9wE!3d2B8Y*hC!$Z7=%G#ftx}m-hI?BukKua zvLkNZ?(F8Dac_AR)&R2a!_#8POkUsl;;wTr3BU2>Z>G{-Wn`h}QbnM>N<}Dac0rDr zJ& Date: Wed, 15 Feb 2017 18:08:37 +0100 Subject: [PATCH 31/75] Remove bounding box masks --- config.yaml | 4 ---- opensfm/config.py | 4 ---- opensfm/features.py | 24 ++++++------------------ 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/config.yaml b/config.yaml index d732cfe20..62c135604 100644 --- a/config.yaml +++ b/config.yaml @@ -29,10 +29,6 @@ akaze_descriptor_channels: 3 # Number of feature channels (1,2,3) hahog_peak_threshold: 0.00001 hahog_edge_threshold: 10 -# Masks for regions that will be ignored for feature extraction -# List of bounding boxes specified as the ratio to image width and height -# masks: [{top: 0.96, bottom: 1.0, left: 0.0, right: 0.15}, {top: 0.95, bottom: 1.0, left: 0, right: 0.05}] - # Params for general matching lowes_ratio: 0.8 # Ratio test for matches preemptive_lowes_ratio: 0.6 # Ratio test for preemptive matches diff --git a/opensfm/config.py b/opensfm/config.py index 5c4d3f60a..d183c2cd2 100644 --- a/opensfm/config.py +++ b/opensfm/config.py @@ -33,10 +33,6 @@ hahog_peak_threshold: 0.00001 hahog_edge_threshold: 10 -# Masks for regions that will be ignored for feature extraction -# List of bounding boxes specified as the ratio to image width and height -# masks: [{top: 0.96, bottom: 1.0, left: 0.0, right: 0.15}, {top: 0.95, bottom: 1.0, left: 0, right: 0.05}] - # Params for general matching lowes_ratio: 0.8 # Ratio test for matches preemptive_lowes_ratio: 0.6 # Ratio test for preemptive matches diff --git a/opensfm/features.py b/opensfm/features.py index 158db6878..03ae5582d 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -68,24 +68,12 @@ def denormalized_image_coordinates(norm_coords, width, height): p[:, 1] = norm_coords[:, 1] * size - 0.5 + height / 2.0 return p -def mask_and_normalize_features(points, desc, colors, width, height, config, image_mask=None): - masks = np.array(config.get('masks',[])) - for mask in masks: - top = mask['top'] * height - left = mask['left'] * width - bottom = mask['bottom'] * height - right = mask['right'] * width - ids = np.invert ( (points[:,1] > top) * - (points[:,1] < bottom) * - (points[:,0] > left) * - (points[:,0] < right) ) - points = points[ids] - desc = desc[ids] - colors = colors[ids] - # We now compare with the image mask for this specific image if it exists - if image_mask is not None: - ids = np.array([_in_mask(point, width, height, image_mask) for point in points]) +def mask_and_normalize_features(points, desc, colors, width, height, mask=None): + """Remove features outside the mask and normalize image coordinates.""" + + if mask is not None: + ids = np.array([_in_mask(point, width, height, mask) for point in points]) points = points[ids] desc = desc[ids] colors = colors[ids] @@ -259,7 +247,7 @@ def extract_features(color_image, config, mask=None): ys = points[:,1].round().astype(int) colors = color_image[ys, xs] - return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], config, mask) + return mask_and_normalize_features(points, desc, colors, image.shape[1], image.shape[0], mask) def build_flann_index(features, config): From b6ca14f336a540e5704552cf7ca65c0ecde50006 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Thu, 16 Feb 2017 12:12:27 +0000 Subject: [PATCH 32/75] Minify reconstruction.meshed.py --- opensfm/commands/mesh.py | 3 ++- opensfm/dataset.py | 4 ++-- opensfm/io.py | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/opensfm/commands/mesh.py b/opensfm/commands/mesh.py index bbee53f5b..7cae57ae9 100644 --- a/opensfm/commands/mesh.py +++ b/opensfm/commands/mesh.py @@ -31,7 +31,8 @@ def run(self, args): shot.mesh.faces = faces data.save_reconstruction(reconstructions, - filename='reconstruction.meshed.json') + filename='reconstruction.meshed.json', + minify=True) end = time.time() with open(data.profile_log(), 'a') as fout: diff --git a/opensfm/dataset.py b/opensfm/dataset.py index ab09ab178..f6dbdad08 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -374,9 +374,9 @@ def load_reconstruction(self, filename=None): reconstructions = io.reconstructions_from_json(json.load(fin)) return reconstructions - def save_reconstruction(self, reconstruction, filename=None): + def save_reconstruction(self, reconstruction, filename=None, minify=False): with open(self.__reconstruction_file(filename), 'w') as fout: - io.json_dump(io.reconstructions_to_json(reconstruction), fout) + io.json_dump(io.reconstructions_to_json(reconstruction), fout, minify) def load_undistorted_reconstruction(self): return self.load_reconstruction( diff --git a/opensfm/io.py b/opensfm/io.py index b7379edb3..6e18ea012 100644 --- a/opensfm/io.py +++ b/opensfm/io.py @@ -346,8 +346,12 @@ def mkdir_p(path): raise -def json_dump(data, fout, indent=4, codec='utf-8'): - return json.dump(data, fout, indent=indent, ensure_ascii=False, encoding=codec) +def json_dump(data, fout, minify=False, codec='utf-8'): + if minify: + indent, separators = None, (',',':') + else: + indent, separators = 4, None + return json.dump(data, fout, indent=indent, ensure_ascii=False, encoding=codec, separators=separators) def json_loads(text, codec='utf-8'): From 8694fef4a3c8639cc3a56e8c4c758a1960455b3a Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 16 Feb 2017 13:32:35 +0100 Subject: [PATCH 33/75] Catch missing OpenCV EXIF rotation option --- opensfm/io.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/opensfm/io.py b/opensfm/io.py index b7379edb3..95ce91f77 100644 --- a/opensfm/io.py +++ b/opensfm/io.py @@ -1,5 +1,6 @@ import errno import json +import logging import os import cv2 @@ -12,6 +13,9 @@ from opensfm import context +logger = logging.getLogger(__name__) + + def camera_from_json(key, obj): """ Read camera from a json object @@ -357,7 +361,14 @@ def json_loads(text, codec='utf-8'): def imread(filename): """Load image as an RGB array ignoring EXIF orientation.""" if context.OPENCV3: - flags = cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION + flags = cv2.IMREAD_COLOR + try: + flags |= cv2.IMREAD_IGNORE_ORIENTATION + except AttributeError: + logger.warning( + "OpenCV version {} does not support loading images without " + "rotating them according to EXIF. Please upgrade OpenCV to " + "version 3.2 or newer.".format(cv2.__version__)) else: flags = cv2.CV_LOAD_IMAGE_COLOR bgr = cv2.imread(filename, flags) From 9dbb717f02cf2074189fdfb31163551023b7b260 Mon Sep 17 00:00:00 2001 From: Tomasz Nycz Date: Mon, 27 Feb 2017 19:42:57 +0100 Subject: [PATCH 34/75] Update sensor_data.json Added DJI Phantom 3 Standard --- opensfm/data/sensor_data.json | 1 + 1 file changed, 1 insertion(+) diff --git a/opensfm/data/sensor_data.json b/opensfm/data/sensor_data.json index 8b2c9c378..7d1ee637f 100644 --- a/opensfm/data/sensor_data.json +++ b/opensfm/data/sensor_data.json @@ -701,6 +701,7 @@ "Contax U4R": 5.33, "Contax i4R": 5.33, "DJI FC300S": 6.16, + "DJI FC300C": 6.31, "DJI FC300X": 6.2, "DJI FC350": 6.17, "DJI FC330": 6.25, From c0e55c61989b6525b3c509b937b1ab2dc293ea67 Mon Sep 17 00:00:00 2001 From: Tyler Baker Date: Tue, 14 Feb 2017 15:01:56 -0800 Subject: [PATCH 35/75] vlfeat: check HAS_CPUID before use of _vl_x86cpu_info_init Check HAS_CPUID before using _vl_x86cpu_info_init to avoid compilation errors when building on other architectures. Signed-off-by: Tyler Baker --- opensfm/src/third_party/vlfeat/vl/host.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensfm/src/third_party/vlfeat/vl/host.c b/opensfm/src/third_party/vlfeat/vl/host.c index ef97cdcae..499db6484 100644 --- a/opensfm/src/third_party/vlfeat/vl/host.c +++ b/opensfm/src/third_party/vlfeat/vl/host.c @@ -441,6 +441,7 @@ _vl_cpuid (vl_int32* info, int function) #endif +#if defined(HAS_CPUID) void _vl_x86cpu_info_init (VlX86CpuInfo *self) { @@ -463,6 +464,7 @@ _vl_x86cpu_info_init (VlX86CpuInfo *self) self->hasAVX = info[2] & (1 << 28) ; } } +#endif char * _vl_x86cpu_info_to_string_copy (VlX86CpuInfo const *self) From 30116a056179ae7f1eaea8927a5c0fed73298e18 Mon Sep 17 00:00:00 2001 From: Tyler Baker Date: Tue, 14 Feb 2017 11:44:43 -0800 Subject: [PATCH 36/75] cmake: VL_DISABLE_SSE2 when building on other architectures SSE2 is an x86 specific instruction set, so disable it when building on other architectures. Signed-off-by: Tyler Baker --- opensfm/src/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opensfm/src/CMakeLists.txt b/opensfm/src/CMakeLists.txt index e0f9a1625..a275b0d4e 100644 --- a/opensfm/src/CMakeLists.txt +++ b/opensfm/src/CMakeLists.txt @@ -77,6 +77,10 @@ target_link_libraries(akaze ${OpenCV_LIBS}) # VLFeat include_directories(third_party/vlfeat) file(GLOB VLFEAT_SRCS third_party/vlfeat/vl/*.c third_party/vlfeat/vl/*.h) +if (NOT CMAKE_SYSTEM_PROCESSOR MATCHES + "(x86)|(X86)|(x86_64)|(X86_64)|(amd64)|(AMD64)") + add_definitions(-DVL_DISABLE_SSE2) +endif () add_definitions(-DVL_DISABLE_AVX) add_library(vl ${VLFEAT_SRCS}) From 752a9fb8bbc2638d4de204301ac0f00de338d01f Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 1 Mar 2017 11:14:55 +0100 Subject: [PATCH 37/75] Test fisheye only when using OpenCV 3 --- opensfm/test/test_types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opensfm/test/test_types.py b/opensfm/test/test_types.py index c49a61539..c14eef0e7 100644 --- a/opensfm/test/test_types.py +++ b/opensfm/test/test_types.py @@ -1,5 +1,6 @@ import numpy as np +from opensfm import context from opensfm import types """ @@ -103,6 +104,8 @@ def test_perspective_camera_projection(): def test_fisheye_camera_projection(): """Test fisheye projection--backprojection loop.""" + if not context.OPENCV3: + return camera = types.FisheyeCamera() camera.width = 800 camera.height = 600 From faad6d40dc760828d6ddfd218386616aad9767e4 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 8 Mar 2017 22:11:56 +0100 Subject: [PATCH 38/75] Use area interpolation for downsampling --- opensfm/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/features.py b/opensfm/features.py index 03ae5582d..1cc6ca572 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -22,7 +22,7 @@ def resized_image(image, config): size = np.array(image.shape[0:2]) if 0 < feature_process_size < size.max(): new_size = size * feature_process_size / size.max() - return cv2.resize(image, dsize=(new_size[1], new_size[0])) + return cv2.resize(image, dsize=(new_size[1], new_size[0]), interpolation=cv2.INTER_AREA) else: return image From 41d174efc63eafe97c7691b1283d71c6fb137689 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 8 Mar 2017 22:18:24 +0100 Subject: [PATCH 39/75] Format --- data/berlin/config.yaml | 1 + opensfm/features.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/data/berlin/config.yaml b/data/berlin/config.yaml index 8b092a971..863426711 100644 --- a/data/berlin/config.yaml +++ b/data/berlin/config.yaml @@ -5,3 +5,4 @@ processes: 8 # Number of threads to use depthmap_min_consistent_views: 2 # Min number of views that should reconstruct a point for it to be valid depthmap_save_debug_files: yes # Save debug files with partial reconstruction results +feature_process_size: 1024 diff --git a/opensfm/features.py b/opensfm/features.py index 1cc6ca572..b712db7e1 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -1,13 +1,8 @@ -# -*- coding: utf-8 -*- +"""Tools to extract features.""" -import os, sys -import tempfile import time import logging -from subprocess import call import numpy as np -import json -import uuid import cv2 import csfm @@ -18,14 +13,17 @@ def resized_image(image, config): - feature_process_size = config.get('feature_process_size', -1) - size = np.array(image.shape[0:2]) - if 0 < feature_process_size < size.max(): - new_size = size * feature_process_size / size.max() - return cv2.resize(image, dsize=(new_size[1], new_size[0]), interpolation=cv2.INTER_AREA) + """Resize image to feature_process_size.""" + max_size = config.get('feature_process_size', -1) + h, w, _ = image.shape + size = max(w, h) + if 0 < max_size < size: + dsize = w * max_size / size, h * max_size / size + return cv2.resize(image, dsize=dsize, interpolation=cv2.INTER_AREA) else: return image + def root_feature(desc, l2_normalization=False): if l2_normalization: s2 = np.linalg.norm(desc, axis=1) @@ -34,6 +32,7 @@ def root_feature(desc, l2_normalization=False): desc = np.sqrt(desc.T/s).T return desc + def root_feature_surf(desc, l2_normalization=False, partial=False): """ Experimental square root mapping of surf-like feature, only work for 64-dim surf now @@ -54,6 +53,7 @@ def root_feature_surf(desc, l2_normalization=False, partial=False): desc[:, ii] = desc_sub*desc_sub_sign return desc + def normalized_image_coordinates(pixel_coords, width, height): size = max(width, height) p = np.empty((len(pixel_coords), 2)) @@ -61,6 +61,7 @@ def normalized_image_coordinates(pixel_coords, width, height): p[:, 1] = (pixel_coords[:, 1] + 0.5 - height / 2.0) / size return p + def denormalized_image_coordinates(norm_coords, width, height): size = max(width, height) p = np.empty((len(norm_coords), 2)) @@ -128,6 +129,7 @@ def extract_features_sift(image, config): points = np.array([(i.pt[0], i.pt[1], i.size, i.angle) for i in points]) return points, desc + def extract_features_surf(image, config): surf_hessian_threshold = config.get('surf_hessian_threshold', 3000) if context.OPENCV3: @@ -171,6 +173,7 @@ def extract_features_surf(image, config): points = np.array([(i.pt[0], i.pt[1], i.size, i.angle) for i in points]) return points, desc + def akaze_descriptor_type(name): d = csfm.AkazeDescriptorType.__dict__ if name in d: @@ -179,6 +182,7 @@ def akaze_descriptor_type(name): logger.debug('Wrong akaze descriptor type') return d['MSURF'] + def extract_features_akaze(image, config): options = csfm.AKAZEOptions() options.omax = config.get('akaze_omax', 4) @@ -206,6 +210,7 @@ def extract_features_akaze(image, config): points = points.astype(float) return points, desc + def extract_features_hahog(image, config): t = time.time() points, desc = csfm.hahog(image.astype(np.float32) / 255, # VlFeat expects pixel values between 0, 1 @@ -226,9 +231,12 @@ def extract_features_hahog(image, config): logger.debug('Found {0} points in {1}s'.format( len(points), time.time()-t )) return points, desc + def extract_features(color_image, config, mask=None): assert len(color_image.shape) == 3 color_image = resized_image(color_image, config) + cv2.imshow('color_image', color_image) + cv2.waitKey(0) image = cv2.cvtColor(color_image, cv2.COLOR_RGB2GRAY) feature_type = config.get('feature_type','SIFT').upper() From f80cc8ef634d971f789bd063512bf51c18c2ecb7 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 8 Mar 2017 23:28:19 +0100 Subject: [PATCH 40/75] Remove log plots --- opensfm/features.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/opensfm/features.py b/opensfm/features.py index b712db7e1..949e04000 100644 --- a/opensfm/features.py +++ b/opensfm/features.py @@ -235,8 +235,6 @@ def extract_features_hahog(image, config): def extract_features(color_image, config, mask=None): assert len(color_image.shape) == 3 color_image = resized_image(color_image, config) - cv2.imshow('color_image', color_image) - cv2.waitKey(0) image = cv2.cvtColor(color_image, cv2.COLOR_RGB2GRAY) feature_type = config.get('feature_type','SIFT').upper() From 979f14dc840449a5561259edab21735eb0c66913 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 16 Mar 2017 11:46:55 +0100 Subject: [PATCH 41/75] Add doc for GCP --- doc/source/gcp.rst | 49 ++++++++++++++++++++++++++++++++++++++++++++ doc/source/using.rst | 3 +++ 2 files changed, 52 insertions(+) create mode 100644 doc/source/gcp.rst diff --git a/doc/source/gcp.rst b/doc/source/gcp.rst new file mode 100644 index 000000000..134d7e8e1 --- /dev/null +++ b/doc/source/gcp.rst @@ -0,0 +1,49 @@ + +Ground Control Points +--------------------- + +When EXIF data contains GPS location, it is used by OpenSfM to georeference the reconstruction. Additionally, it is possible to use ground control points. + +Ground control points (GCP) are landmarks visible on the images for which the geospatial position (latitude, longitude and altitude) is known. A single GCP can be observed in one or more images. + +OpenSfM uses GCP in two steps of the reconstruction process: alignment and bundle adjustment. In the alignment step, points are used to globaly move the reconstruction so that the observed GCP align with their GPS position. Two or more observations for each GCP are required for it to be used during the aligment step. + +In the bundle adjustment step, GCP observations are used as a constraint to refine the reconstruction. In this step, all ground control points are used. No minimum number of observation is required. + +File format +``````````` +GCPs can be specified by adding a text file named ``gcp_list.txt`` at the root folder of the dataset. The format of the file should be as follows. + + - The first line should contain the name of the projection used for the geo coordinates. + + - The following lines should con should contain the data for each ground control point observation. One per line and in the format + :: + + Where `` `` are the geospatial coordinates of the GCP and `` `` are the pixel coordinates where the GCP is observed. + + +Supported projections +````````````````````` +The geospatial coordinates can be specified in one the following formats. + +- WGS84: This is the standard latitude, longitude coordinates used by most GPS devices. In this case, `` = longitude``, `` = latitude`` and `` = altitude`` + +- `UTM`_: UTM projections can be specified using a string projection string such as ``WGS84 UTM 32N``, where 32 is the region and N is . In this case, `` = E``, `` = N`` and `` = altitude`` + +- `proj4`_: Any valid proj4 format string can be used. For example, for UTM 32N we can use ``+proj=utm +zone=32 +north +ellps=WGS84 +datum=WGS84 +units=m +no_defs`` + +.. _WGS84: https://en.wikipedia.org/wiki/World_Geodetic_System +.. _UTM: https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system +.. _proj4: http://proj4.org/ + +Example +``````` +:: + + WGS84 + 13.400740745 52.519134104 12.0792090446 2335.0 1416.7 01.jpg + 13.400740745 52.519134104 12.0792090446 2639.1 938.0 02.jpg + 13.400502446 52.519251158 16.7021233002 766.0 1133.1 01.jpg + +This file defines 2 GCP whose coordinates are specified in the WGS84 standard. The first one is observed in both ``01.jpg`` and ``02.jpg``, while the second one is only observed in ``01.jpg`` + diff --git a/doc/source/using.rst b/doc/source/using.rst index d40756f89..effa7eedf 100644 --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -114,3 +114,6 @@ Configuration ------------- TODO explain config.yaml and the available parameters + + +.. include:: gcp.rst From abd15913ea056d2a70cb291f65d85214e052dd41 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 16 Mar 2017 11:51:23 +0100 Subject: [PATCH 42/75] Run doc server at 8001 --- doc/Makefile | 2 +- doc/source/building.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 39db9881e..137f03184 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -58,7 +58,7 @@ html: @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." livehtml: html - sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + sphinx-autobuild -b html -p 8001 $(ALLSPHINXOPTS) $(BUILDDIR)/html dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml diff --git a/doc/source/building.rst b/doc/source/building.rst index 52a65b9e1..e6a82620f 100644 --- a/doc/source/building.rst +++ b/doc/source/building.rst @@ -70,3 +70,12 @@ When running OpenSfM on top of OpenCV version 3.0 the `OpenCV Contrib`_ modules .. _Ceres solver: http://ceres-solver.org/ .. _Boost Python: http://www.boost.org/ .. _Networkx: https://github.com/networkx/networkx + + +Building the documentation +-------------------------- +To build the documentation and browse it locally use +:: + cd doc + make livehtml +and browse `http://localhost:8001/ `_ From 88bb9672f54fcfcd37907856c0ba2e786204bf44 Mon Sep 17 00:00:00 2001 From: Sina Samangooei Date: Thu, 16 Mar 2017 12:25:28 +0000 Subject: [PATCH 43/75] added missing packages in setup.py and added a better named run_all for global installation --- bin/opensfm_run_all | 12 ++++++++++++ setup.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 bin/opensfm_run_all diff --git a/bin/opensfm_run_all b/bin/opensfm_run_all new file mode 100755 index 000000000..831677255 --- /dev/null +++ b/bin/opensfm_run_all @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +$DIR/opensfm extract_metadata $1 +$DIR/opensfm detect_features $1 +$DIR/opensfm match_features $1 +$DIR/opensfm create_tracks $1 +$DIR/opensfm reconstruct $1 +$DIR/opensfm mesh $1 diff --git a/setup.py b/setup.py index 35216ea13..b319c8736 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,8 @@ def mkdir_p(path): url='https://github.com/mapillary/OpenSfM', author='Mapillary', license='BSD', - packages=['opensfm'], + packages=['opensfm', 'opensfm.commands'], + scripts=['bin/opensfm_run_all', 'bin/opensfm'], package_data={ 'opensfm': ['csfm.so', 'data/sensor_data.json'] }, From a25f0805f161a246e0d46b0c11f98aaf1231c271 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Mon, 20 Mar 2017 16:02:15 +0100 Subject: [PATCH 44/75] Add doc on coordinate systems --- doc/source/geometry.rst | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/doc/source/geometry.rst b/doc/source/geometry.rst index d0ae5f2f5..c24ee9fc3 100644 --- a/doc/source/geometry.rst +++ b/doc/source/geometry.rst @@ -10,7 +10,43 @@ TODO Coordinate Systems ------------------ -TODO +Normalized Image Coordinates +```````````````````````````` + +The 2d position of a point in images is stored in what we will call *normalized image coordinates*. The origin is in the middle of the image. The x coordinate grows to the right and y grows downwards. The larger dimension of the image is 1. + +This means, for example, that all the pixels in an image with aspect ratio 4:3 will be contained in the intervals ``[-0.5, 0.5]`` and ``[3/4 * (-0.5), 3/4 * 0.5]`` for the X and Y axis respectively. + +:: + + +-----------------------------+ + | | + | | + | | + | + -------------> + | | (0, 0) | (0.5, 0) + | | | + | | | + +-----------------------------+ + | + v + (0, 0.5) + + +World Coordinates +````````````````` +The position of the reconstructed 3D points is stored in *world coordinates*. In general, this is an arbitrary euclidean reference frame. + +When GPS data is available, a topocentric reference frame is used for the world coordinates reference. This is a reference frame that with the origin somewhere near the ground, the X axis pointing to the east, the Y axis pointing to the north and the Z axis pointing to the zenith. The latitude, longitude, and altitude of the origin are stored in the ``reference_lla.json`` file. + +When GPS data is not available, the reconstruction process makes its best to rotate the world reference frame so that the vertical direction is Z and the ground is near the `z = 0` plane. It does so by assuming that the images are taken from similar altitudes and that the up vector of the images corresponds to the up vector of the world. + + +Camera Coordinates +`````````````````` +The *camera coordinate* reference frame has the origin at the camera's optical center, the X axis is pointing to the right of the camera the Y axis is pointing down and the Z axis is pointing to the front. A point in front of the camera has positive Z camera coordinate. + +The pose of a camera is determined by the rotation and translation that converts world coordinates to camera coordinates. Camera Models From d301856e920af2f999267e53032ecf1b18154813 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Mon, 20 Mar 2017 16:15:15 +0100 Subject: [PATCH 45/75] Add pixel coordinates doc --- doc/source/geometry.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/geometry.rst b/doc/source/geometry.rst index c24ee9fc3..6f0bd753b 100644 --- a/doc/source/geometry.rst +++ b/doc/source/geometry.rst @@ -32,6 +32,14 @@ This means, for example, that all the pixels in an image with aspect ratio 4:3 w v (0, 0.5) +Normalized coordinates are independent of the resolution of the image and give better numerical stability for some multi-view geometry algorithms than pixel coordinates. + + +Pixel Coordinates +````````````````` + +Many OpenCV functions that work with images use *pixel coordinates*. In that reference frame, the origin is at the center of the top-left pixel, x grow by one for every pixel to the right and y grows by one for every pixel downwards. The bottom-right pixels is therefore at ``(width -1, height - 1)``. + World Coordinates ````````````````` From 7f084bc91612a0063dcada5e41b3e938a38a5825 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Fri, 24 Mar 2017 15:41:45 +0100 Subject: [PATCH 46/75] Handle binary Akaze descriptors --- opensfm/src/akaze.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/opensfm/src/akaze.cc b/opensfm/src/akaze.cc index 749a2100d..22af173b8 100644 --- a/opensfm/src/akaze.cc +++ b/opensfm/src/akaze.cc @@ -42,8 +42,14 @@ bp::object akaze(PyObject *image, bp::list retn; npy_intp keys_shape[2] = {keys.rows, keys.cols}; retn.append(bpn_array_from_data(2, keys_shape, keys.ptr(0))); - npy_intp desc_shape[2] = {desc.rows, desc.cols}; - retn.append(bpn_array_from_data(2, desc_shape, desc.ptr(0))); + + if (options.descriptor == MLDB_UPRIGHT || options.descriptor == MLDB) { + npy_intp desc_shape[2] = {desc.rows, desc.cols}; + retn.append(bpn_array_from_data(2, desc_shape, desc.ptr(0))); + } else { + npy_intp desc_shape[2] = {desc.rows, desc.cols}; + retn.append(bpn_array_from_data(2, desc_shape, desc.ptr(0))); + } return retn; } From d30b5394adc8ec895ee9153525dc959a85093610 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Fri, 24 Mar 2017 15:43:07 +0100 Subject: [PATCH 47/75] Compute flann index only when flann matching is selected --- doc/source/geometry.rst | 2 +- opensfm/commands/detect_features.py | 6 ++++-- opensfm/dataset.py | 10 ---------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/doc/source/geometry.rst b/doc/source/geometry.rst index 6f0bd753b..22209603e 100644 --- a/doc/source/geometry.rst +++ b/doc/source/geometry.rst @@ -38,7 +38,7 @@ Normalized coordinates are independent of the resolution of the image and give b Pixel Coordinates ````````````````` -Many OpenCV functions that work with images use *pixel coordinates*. In that reference frame, the origin is at the center of the top-left pixel, x grow by one for every pixel to the right and y grows by one for every pixel downwards. The bottom-right pixels is therefore at ``(width -1, height - 1)``. +Many OpenCV functions that work with images use *pixel coordinates*. In that reference frame, the origin is at the center of the top-left pixel, x grow by one for every pixel to the right and y grows by one for every pixel downwards. The bottom-right pixel is therefore at ``(width -1, height - 1)``. World Coordinates diff --git a/opensfm/commands/detect_features.py b/opensfm/commands/detect_features.py index 80ccca050..cc2879c3e 100644 --- a/opensfm/commands/detect_features.py +++ b/opensfm/commands/detect_features.py @@ -60,5 +60,7 @@ def detect(args): f_pre = f_sorted[-preemptive_max:] data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) - index = features.build_flann_index(f_sorted, data.config) - data.save_feature_index(image, index) + + if data.config.get('matcher_type', 'FLANN') == 'FLANN': + index = features.build_flann_index(f_sorted, data.config) + data.save_feature_index(image, index) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index f6dbdad08..36437a4d1 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -303,16 +303,6 @@ def load_preemtive_features(self, image): def save_preemptive_features(self, image, points, descriptors): self.__save_features(self.__preemptive_features_file(image), image, points, descriptors) - def matcher_type(self): - """Return the type of matcher - """ - matcher_type = self.config.get('matcher_type', 'BruteForce') - if 'BruteForce' in matcher_type: - if self.feature_type() == 'akaze' and (self.config.get('akaze_descriptor', 5) >= 4): - matcher_type = 'BruteForce-Hamming' - self.config['matcher_type'] = matcher_type - return matcher_type # BruteForce, BruteForce-L1, BruteForce-Hamming - def __matches_path(self): """Return path of matches directory""" return os.path.join(self.data_path, 'matches') From ce0e3d8743fdc42bbaf18461caa8bd7514a6661f Mon Sep 17 00:00:00 2001 From: Alexey Kachayev Date: Mon, 27 Mar 2017 21:56:01 +0300 Subject: [PATCH 48/75] Update documentation to use `virtualenv` README file now suggests using `virtualenv` to create isolated `python` environment to manage installed dependencies separately from system-level packages. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9207ac607..85697b9ca 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ Install OpenCV using brew install opencv brew install homebrew/science/ceres-solver brew install boost-python - sudo pip install -r requirements.txt And install OpenGV using @@ -63,6 +62,11 @@ When running OpenSfM on top of [OpenCV][] 3.0 the [OpenCV Contrib][] modules are ## Building + sudo pip install virtualenv + virtualenv . + source bin/activate + pip install -r requirements.txt + pip install scipy==0.19.0 python setup.py build From 46a163a2bac53eeec1c626284d313a174da65441 Mon Sep 17 00:00:00 2001 From: Alexey Kachayev Date: Tue, 28 Mar 2017 12:50:49 +0300 Subject: [PATCH 49/75] Use ENV name for virtualenv not to mess up with OpenSfM bin files --- .gitignore | 3 +++ README.md | 5 ++--- requirements.txt | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 92e34e201..31fa17aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ data/berlin/* !data/berlin/images eval + +# Ignore virtualenv files +env/* diff --git a/README.md b/README.md index 85697b9ca..d6a09938d 100644 --- a/README.md +++ b/README.md @@ -63,10 +63,9 @@ When running OpenSfM on top of [OpenCV][] 3.0 the [OpenCV Contrib][] modules are ## Building sudo pip install virtualenv - virtualenv . - source bin/activate + virtualenv env + source env/bin/activate pip install -r requirements.txt - pip install scipy==0.19.0 python setup.py build diff --git a/requirements.txt b/requirements.txt index 2289cf936..6a070ead0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ exifread gpxpy networkx numpy +scipy pytest python-dateutil PyYAML From ae6c0ec4437e87dfa591762e57517d146e90fef4 Mon Sep 17 00:00:00 2001 From: Alexey Kachayev Date: Tue, 28 Mar 2017 13:45:09 +0300 Subject: [PATCH 50/75] Specify library versions in requirements file --- requirements.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6a070ead0..4e5540677 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -exifread -gpxpy -networkx -numpy -scipy -pytest -python-dateutil -PyYAML -xmltodict -pyproj +exifread==2.1.2 +gpxpy==1.1.2 +networkx==1.11 +numpy==1.12.1 +pytest==3.0.7 +python-dateutil==2.6.0 +pyproj==1.9.5.1 +PyYAML==3.12 +scipy==0.19.0 +xmltodict==0.10.2 From 1d44100ad05d99ebd0aeda639ca159c3a1414fcb Mon Sep 17 00:00:00 2001 From: Alexey Kachayev Date: Tue, 28 Mar 2017 14:28:32 +0300 Subject: [PATCH 51/75] Do not specify versions for numpy & scipy to try to avoid problems on linux --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4e5540677..9da24d9a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ exifread==2.1.2 gpxpy==1.1.2 networkx==1.11 -numpy==1.12.1 +numpy pytest==3.0.7 python-dateutil==2.6.0 pyproj==1.9.5.1 PyYAML==3.12 -scipy==0.19.0 +scipy xmltodict==0.10.2 From 68dc4b856e33c4c8d27b50960dd7cf0a3305347a Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 29 Mar 2017 11:12:18 +0200 Subject: [PATCH 52/75] Add scipy requirement --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2289cf936..2f6735c39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ exifread gpxpy networkx numpy +pyproj pytest python-dateutil PyYAML +scipy xmltodict -pyproj From 7a0260dca8316a334e818f88e85f46210d9e7cd9 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 4 Dec 2016 11:17:30 +0000 Subject: [PATCH 53/75] Store and return best neighbor index. --- opensfm/dense.py | 17 +++++++--- opensfm/src/csfm.cc | 1 + opensfm/src/depthmap.cc | 57 ++++++++++++++++++--------------- opensfm/src/depthmap_wrapper.cc | 16 +++++---- 4 files changed, 54 insertions(+), 37 deletions(-) diff --git a/opensfm/dense.py b/opensfm/dense.py index 1668328e5..0aee56116 100644 --- a/opensfm/dense.py +++ b/opensfm/dense.py @@ -67,7 +67,7 @@ def compute_depthmap(arguments): de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations']) de.set_min_patch_sd(data.config['depthmap_min_patch_sd']) add_views_to_depth_estimator(data, reconstruction, neighbors[shot.id], de) - depth, plane, score = de.compute_patch_match() + depth, plane, score, nbour = de.compute_patch_match() good_score = score > data.config['depthmap_min_correlation_score'] depth = depth * (depth < max_depth) * good_score @@ -83,16 +83,21 @@ def compute_depthmap(arguments): if data.config.get('interactive'): import matplotlib.pyplot as plt - plt.subplot(2, 2, 1) + plt.figure() + plt.suptitle("Shot: " + shot.id + ", neighbors: " + ', '.join(neighbors[shot.id][1:])) + plt.subplot(2, 3, 1) plt.imshow(image) - plt.subplot(2, 2, 2) + plt.subplot(2, 3, 2) plt.imshow(color_plane_normals(plane)) - plt.subplot(2, 2, 3) + plt.subplot(2, 3, 3) plt.imshow(depth) plt.colorbar() - plt.subplot(2, 2, 4) + plt.subplot(2, 3, 4) plt.imshow(score) plt.colorbar() + plt.subplot(2, 3, 5) + plt.imshow(nbour) + plt.colorbar() plt.show() @@ -124,6 +129,8 @@ def clean_depthmap(arguments): if data.config.get('interactive'): import matplotlib.pyplot as plt + plt.figure() + plt.suptitle("Shot: " + shot.id) plt.subplot(2, 2, 1) plt.imshow(raw_depth) plt.colorbar() diff --git a/opensfm/src/csfm.cc b/opensfm/src/csfm.cc index c528d300b..187f347d3 100644 --- a/opensfm/src/csfm.cc +++ b/opensfm/src/csfm.cc @@ -1,5 +1,6 @@ #include #include +#include #include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index c39cc0dc2..3bf8ef4e0 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -176,10 +176,11 @@ class DepthmapEstimator { min_patch_variance_ = sd * sd; } - void ComputeBruteForce(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void ComputeBruteForce(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); + *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_8U, cv::Scalar(0)); int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { @@ -188,42 +189,44 @@ class DepthmapEstimator { float depth = 1 / (1 / min_depth_ + d * (1 / max_depth_ - 1 / min_depth_) / (num_depth_planes_ - 1)); cv::Vec3f normal(0, 0, -1); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneCandidate(best_depth, best_plane, best_score, i, j, plane); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); } } } } - void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); + *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_8U, cv::Scalar(0)); - RandomInitialization(best_depth, best_plane, best_score); - ComputeIgnoreMask(best_depth, best_plane, best_score); + RandomInitialization(best_depth, best_plane, best_score, best_nbour); + ComputeIgnoreMask(best_depth, best_plane, best_score, best_nbour); for (int i = 0; i < patchmatch_iterations_; ++i) { - PatchMatchForwardPass(best_depth, best_plane, best_score); - PatchMatchBackwardPass(best_depth, best_plane, best_score); + PatchMatchForwardPass(best_depth, best_plane, best_score, best_nbour); + PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nbour); } } - void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { float depth = UniformRand(min_depth_, max_depth_); cv::Vec3f normal(UniformRand(-1, 1), UniformRand(-1, 1), -1); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - float score = ComputePlaneScore(i, j, plane); + std::pair score = ComputePlaneScore(i, j, plane); best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; - best_score->at(i, j) = score; + best_score->at(i, j) = score.first; + best_nbour->at(i, j) = score.second; } } } - void ComputeIgnoreMask(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void ComputeIgnoreMask(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { @@ -232,6 +235,7 @@ class DepthmapEstimator { best_depth->at(i, j) = 0; best_plane->at(i, j) = cv::Vec3f(0, 0, 0); best_score->at(i, j) = 0; + best_nbour->at(i, j) = 0; } } } @@ -250,27 +254,27 @@ class DepthmapEstimator { } - void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { int neighbors[2][2] = {{-1, 0}, {0, -1}}; int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nbour, i, j, neighbors); } } } - void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score) { + void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { int neighbors[2][2] = {{0, 1}, {1, 0}}; int hpz = (patch_size_ - 1) / 2; for (int i = best_depth->rows - hpz - 1; i >= hpz; --i) { for (int j = best_depth->cols - hpz - 1; j >= hpz; --j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nbour, i, j, neighbors); } } } - void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, + void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, int i, int j, int neighbors[2][2]) { // Ignore pixels with depth == 0. @@ -281,7 +285,7 @@ class DepthmapEstimator { // Check neighbor's planes. for (int k = 0; k < 2; ++k) { cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); - CheckPlaneCandidate(best_depth, best_plane, best_score, i, j, plane); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); } // Check random planes. @@ -297,32 +301,35 @@ class DepthmapEstimator { -1.0f); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneCandidate(best_depth, best_plane, best_score, i, j, plane); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); depth_range *= 0.5; normal_range *= 0.5; } } - void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, + void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, int i, int j, const cv::Vec3f &plane) { - float score = ComputePlaneScore(i, j, plane); - if (score > best_score->at(i, j)) { - best_score->at(i, j) = score; + std::pair score = ComputePlaneScore(i, j, plane); + if (score.first > best_score->at(i, j)) { + best_score->at(i, j) = score.first; best_plane->at(i, j) = plane; best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); + best_nbour->at(i, j) = score.second; } } - float ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { + std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { float best_score = -1.0f; - for (int other = 1; other < images_.size(); ++other) { + unsigned char best_nbour = 0; + for (std::size_t other = 1; other < images_.size(); ++other) { float score = ComputePlaneImageScore(i, j, plane, other); if (score > best_score) { best_score = score; + best_nbour = static_cast(other); } } - return best_score; + return std::make_pair(best_score, best_nbour); } float ComputePlaneImageScoreUnoptimized(int i, int j, diff --git a/opensfm/src/depthmap_wrapper.cc b/opensfm/src/depthmap_wrapper.cc index 8fc0694cd..0c53aa72b 100644 --- a/opensfm/src/depthmap_wrapper.cc +++ b/opensfm/src/depthmap_wrapper.cc @@ -32,26 +32,28 @@ class DepthmapEstimatorWrapper { } bp::object ComputePatchMatch() { - cv::Mat depth, plane, score; - de_.ComputePatchMatch(&depth, &plane, &score); - return ComputeReturnValues(depth, plane, score); + cv::Mat depth, plane, score, nbour; + de_.ComputePatchMatch(&depth, &plane, &score, &nbour); + return ComputeReturnValues(depth, plane, score, nbour); } bp::object ComputeBruteForce() { - cv::Mat depth, plane, score; - de_.ComputeBruteForce(&depth, &plane, &score); - return ComputeReturnValues(depth, plane, score); + cv::Mat depth, plane, score, nbour; + de_.ComputeBruteForce(&depth, &plane, &score, &nbour); + return ComputeReturnValues(depth, plane, score, nbour); } bp::object ComputeReturnValues(const cv::Mat &depth, const cv::Mat &plane, - const cv::Mat &score) { + const cv::Mat &score, + const cv::Mat &nbour) { bp::list retn; npy_intp shape[2] = {depth.rows, depth.cols}; npy_intp plane_shape[3] = {depth.rows, depth.cols, 3}; retn.append(bpn_array_from_data(2, shape, depth.ptr(0))); retn.append(bpn_array_from_data(3, plane_shape, plane.ptr(0))); retn.append(bpn_array_from_data(2, shape, score.ptr(0))); + retn.append(bpn_array_from_data(2, shape, nbour.ptr(0))); return retn; } From 12835bfc019952b2189e7ae69e7390e1fa5b100e Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Mon, 7 Nov 2016 18:15:18 +0000 Subject: [PATCH 54/75] Fix pytest setup-cfg section name warning. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ef7356404..ee6817871 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ -[pytest] +[tool:pytest] testpaths = opensfm addopts = --doctest-modules From f31246db239b1a771ab87a339c6715c15f76b551 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 4 Dec 2016 19:38:32 +0000 Subject: [PATCH 55/75] Initialize with single random neighbor. --- opensfm/src/csfm.cc | 1 + opensfm/src/depthmap.cc | 34 ++++++++++++++++++++------------- opensfm/src/depthmap_wrapper.cc | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/opensfm/src/csfm.cc b/opensfm/src/csfm.cc index 187f347d3..f82d32565 100644 --- a/opensfm/src/csfm.cc +++ b/opensfm/src/csfm.cc @@ -2,6 +2,7 @@ #include #include #include +#include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 3bf8ef4e0..52681789f 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -135,7 +135,6 @@ float UniformRand(float a, float b) { return a + (b - a) * float(rand()) / RAND_MAX; } - class DepthmapEstimator { public: DepthmapEstimator() @@ -145,6 +144,8 @@ class DepthmapEstimator { , num_depth_planes_(50) , patchmatch_iterations_(3) , min_patch_variance_(5 * 5) + , rng_{std::random_device{}()} + , uni_(0, 0) {} void AddView(const double *pK, @@ -160,6 +161,10 @@ class DepthmapEstimator { Qs_.emplace_back(Rs_.back() * Rs_.front().t()); as_.emplace_back(Qs_.back() * ts_.front() - ts_.back()); images_.emplace_back(cv::Mat(height, width, CV_8U, (void *)pimage).clone()); + std::size_t size = images_.size(); + int a = (size > 1) ? 1 : 0; + int b = (size > 1) ? size - 1 : 0; + uni_.param(std::uniform_int_distribution::param_type(a, b)); } void SetDepthRange(double min_depth, double max_depth, int num_depth_planes) { @@ -180,7 +185,7 @@ class DepthmapEstimator { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_8U, cv::Scalar(0)); + *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { @@ -199,7 +204,7 @@ class DepthmapEstimator { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_8U, cv::Scalar(0)); + *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); RandomInitialization(best_depth, best_plane, best_score, best_nbour); ComputeIgnoreMask(best_depth, best_plane, best_score, best_nbour); @@ -217,11 +222,12 @@ class DepthmapEstimator { float depth = UniformRand(min_depth_, max_depth_); cv::Vec3f normal(UniformRand(-1, 1), UniformRand(-1, 1), -1); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - std::pair score = ComputePlaneScore(i, j, plane); + int other = uni_(rng_); + float score = ComputePlaneImageScore(i, j, plane, other); best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; - best_score->at(i, j) = score.first; - best_nbour->at(i, j) = score.second; + best_score->at(i, j) = score; + best_nbour->at(i, j) = other; } } } @@ -235,7 +241,7 @@ class DepthmapEstimator { best_depth->at(i, j) = 0; best_plane->at(i, j) = cv::Vec3f(0, 0, 0); best_score->at(i, j) = 0; - best_nbour->at(i, j) = 0; + best_nbour->at(i, j) = 0; } } } @@ -310,23 +316,23 @@ class DepthmapEstimator { void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, int i, int j, const cv::Vec3f &plane) { - std::pair score = ComputePlaneScore(i, j, plane); + std::pair score = ComputePlaneScore(i, j, plane); if (score.first > best_score->at(i, j)) { best_score->at(i, j) = score.first; best_plane->at(i, j) = plane; best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); - best_nbour->at(i, j) = score.second; + best_nbour->at(i, j) = score.second; } } - std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { + std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { float best_score = -1.0f; - unsigned char best_nbour = 0; - for (std::size_t other = 1; other < images_.size(); ++other) { + int best_nbour = 0; + for (int other = 1; other < images_.size(); ++other) { float score = ComputePlaneImageScore(i, j, plane, other); if (score > best_score) { best_score = score; - best_nbour = static_cast(other); + best_nbour = other; } } return std::make_pair(best_score, best_nbour); @@ -397,6 +403,8 @@ class DepthmapEstimator { int num_depth_planes_; int patchmatch_iterations_; float min_patch_variance_; + std::mt19937 rng_; + std::uniform_int_distribution uni_; }; diff --git a/opensfm/src/depthmap_wrapper.cc b/opensfm/src/depthmap_wrapper.cc index 0c53aa72b..6294e08c4 100644 --- a/opensfm/src/depthmap_wrapper.cc +++ b/opensfm/src/depthmap_wrapper.cc @@ -53,7 +53,7 @@ class DepthmapEstimatorWrapper { retn.append(bpn_array_from_data(2, shape, depth.ptr(0))); retn.append(bpn_array_from_data(3, plane_shape, plane.ptr(0))); retn.append(bpn_array_from_data(2, shape, score.ptr(0))); - retn.append(bpn_array_from_data(2, shape, nbour.ptr(0))); + retn.append(bpn_array_from_data(2, shape, nbour.ptr(0))); return retn; } From 366e36c34730528d98030c0e5ea817f9ee868dc7 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 4 Dec 2016 20:41:28 +0000 Subject: [PATCH 56/75] Randomly sample neighbors. Use only current neighbor when checking random planes. Propagate neighbor when checking neighboring pixel planes. Randomly sample another neighbor after all other tests. --- opensfm/src/depthmap.cc | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 52681789f..37691df03 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -291,12 +291,18 @@ class DepthmapEstimator { // Check neighbor's planes. for (int k = 0; k < 2; ++k) { cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); + int nbour = best_nbour->at(i + neighbors[k][0], j + neighbors[k][1]); + if (nbour == 0) { + continue; + } + + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, nbour); } // Check random planes. float depth_range = (1 / max_depth_ - 1 / min_depth_) / 20; float normal_range = 0.5; + int current_nbour = best_nbour->at(i, j); for (int k = 0; k < 6; ++k) { float current_depth = best_depth->at(i, j); float depth = 1 / (1 / current_depth + UniformRand(-depth_range, depth_range)); @@ -307,11 +313,24 @@ class DepthmapEstimator { -1.0f); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, current_nbour); depth_range *= 0.5; normal_range *= 0.5; } + + if (images_.size() <= 2) { + return; + } + + // TODO: handle excluding current neighbour index correctly when drawing from uniform integer distribution + int other_nbour = uni_(rng_); + while (other_nbour == current_nbour) { + other_nbour = uni_(rng_); + } + + cv::Vec3f plane = best_plane->at(i, j); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, other_nbour); } void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, @@ -325,6 +344,17 @@ class DepthmapEstimator { } } + void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, + int i, int j, const cv::Vec3f &plane, int nbour) { + float score = ComputePlaneImageScore(i, j, plane, nbour); + if (score > best_score->at(i, j)) { + best_score->at(i, j) = score; + best_plane->at(i, j) = plane; + best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); + best_nbour->at(i, j) = nbour; + } + } + std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { float best_score = -1.0f; int best_nbour = 0; From 7ec103419beb2139b240930ef3a08cab9474039b Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sat, 18 Mar 2017 12:03:33 +0000 Subject: [PATCH 57/75] Include headers for depth where used. --- opensfm/src/csfm.cc | 2 -- opensfm/src/depthmap.cc | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/opensfm/src/csfm.cc b/opensfm/src/csfm.cc index f82d32565..c528d300b 100644 --- a/opensfm/src/csfm.cc +++ b/opensfm/src/csfm.cc @@ -1,8 +1,6 @@ #include #include -#include #include -#include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 37691df03..abdbb5038 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -1,7 +1,8 @@ +#include +#include #include - namespace csfm { bool IsInsideImage(const cv::Mat &image, int i, int j) { From 40c0118b8c505fdb4e89a5780902434fad08c950 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 19 Mar 2017 11:17:02 +0000 Subject: [PATCH 58/75] Save neighbour matrix and list in dataset. --- opensfm/dataset.py | 6 +++--- opensfm/dense.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 36437a4d1..4c4e6e288 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -113,14 +113,14 @@ def _depthmap_file(self, image, suffix): def raw_depthmap_exists(self, image): return os.path.isfile(self._depthmap_file(image, 'raw.npz')) - def save_raw_depthmap(self, image, depth, plane, score): + def save_raw_depthmap(self, image, depth, plane, score, nbour, nbours): io.mkdir_p(self._depthmap_path()) filepath = self._depthmap_file(image, 'raw.npz') - np.savez_compressed(filepath, depth=depth, plane=plane, score=score) + np.savez_compressed(filepath, depth=depth, plane=plane, score=score, nbour=nbour, nbours=nbours) def load_raw_depthmap(self, image): o = np.load(self._depthmap_file(image, 'raw.npz')) - return o['depth'], o['plane'], o['score'] + return o['depth'], o['plane'], o['score'], o['nbour'], o['nbours'] def clean_depthmap_exists(self, image): return os.path.isfile(self._depthmap_file(image, 'clean.npz')) diff --git a/opensfm/dense.py b/opensfm/dense.py index 0aee56116..2a6df3c5e 100644 --- a/opensfm/dense.py +++ b/opensfm/dense.py @@ -72,7 +72,7 @@ def compute_depthmap(arguments): depth = depth * (depth < max_depth) * good_score # Save and display results - data.save_raw_depthmap(shot.id, depth, plane, score) + data.save_raw_depthmap(shot.id, depth, plane, score, nbour, neighbors[shot.id][1:]) if data.config['depthmap_save_debug_files']: image = data.undistorted_image_as_array(shot.id) @@ -117,7 +117,7 @@ def clean_depthmap(arguments): depth = dc.clean() # Save and display results - raw_depth, raw_plane, raw_score = data.load_raw_depthmap(shot.id) + raw_depth, raw_plane, raw_score, raw_nbour, nbours = data.load_raw_depthmap(shot.id) data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score) if data.config['depthmap_save_debug_files']: @@ -197,7 +197,7 @@ def add_views_to_depth_cleaner(data, reconstruction, neighbors, dc): shot = reconstruction.shots[neighbor] if not data.raw_depthmap_exists(shot.id): continue - depth, plane, score = data.load_raw_depthmap(shot.id) + depth, plane, score, nbour, nbours = data.load_raw_depthmap(shot.id) height, width = depth.shape K = shot.camera.get_K_in_pixel_coordinates(width, height) R = shot.pose.get_rotation_matrix() From 64b7a4e839619a869247a8daf559f24fef3f1e06 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 19 Mar 2017 11:17:33 +0000 Subject: [PATCH 59/75] Rename image plane check overload for clarity. --- opensfm/src/depthmap.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index abdbb5038..3a6848776 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -297,7 +297,7 @@ class DepthmapEstimator { continue; } - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, nbour); } // Check random planes. @@ -314,7 +314,7 @@ class DepthmapEstimator { -1.0f); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, current_nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, current_nbour); depth_range *= 0.5; normal_range *= 0.5; @@ -324,14 +324,13 @@ class DepthmapEstimator { return; } - // TODO: handle excluding current neighbour index correctly when drawing from uniform integer distribution int other_nbour = uni_(rng_); while (other_nbour == current_nbour) { other_nbour = uni_(rng_); } cv::Vec3f plane = best_plane->at(i, j); - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, other_nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, other_nbour); } void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, @@ -345,8 +344,8 @@ class DepthmapEstimator { } } - void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, - int i, int j, const cv::Vec3f &plane, int nbour) { + void CheckPlaneImageCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, + int i, int j, const cv::Vec3f &plane, int nbour) { float score = ComputePlaneImageScore(i, j, plane, nbour); if (score > best_score->at(i, j)) { best_score->at(i, j) = score; From 1fa98d96c92d14b5f02be2a999e1dd2072cb90a0 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Sun, 19 Mar 2017 11:21:38 +0000 Subject: [PATCH 60/75] Add script for plotting saved raw depthmaps. --- bin/plot_depthmaps | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 bin/plot_depthmaps diff --git a/bin/plot_depthmaps b/bin/plot_depthmaps new file mode 100644 index 000000000..d286f1321 --- /dev/null +++ b/bin/plot_depthmaps @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import argparse +import matplotlib.pyplot as pl +import numpy as np +from textwrap import wrap + +from opensfm import dataset +from opensfm import io + + +def plot_depthmap(im, title, depth, plane, score, nbour): + ax = pl.subplot2grid((2, 3), (0, 0), rowspan=2) + ax_title = ax.set_title(title) + ax_title.set_y(1.05) + pl.imshow(im) + ax = pl.subplot(2, 3, 2) + ax_title = ax.set_title("Depth") + ax_title.set_y(1.08) + pl.imshow(depth) + pl.colorbar() + ax = pl.subplot(2, 3, 3) + ax_title = ax.set_title("Score") + ax_title.set_y(1.08) + pl.imshow(score) + pl.colorbar() + ax = pl.subplot(2, 3, 5) + ax_title = ax.set_title("Neighbour") + ax_title.set_y(1.08) + pl.imshow(nbour) + pl.colorbar() + ax = pl.subplot(2, 3, 6) + ax_title = ax.set_title("Plane normal") + ax_title.set_y(1.02) + pl.imshow(color_plane_normals(plane)) + + +def color_plane_normals(plane): + l = np.linalg.norm(plane, axis=2) + normal = plane / l[..., np.newaxis] + normal[..., 1] *= -1 # Reverse Y because it points down + normal[..., 2] *= -1 # Reverse Z because standard colormap does so + return ((normal + 1) * 128).astype(np.uint8) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Compute reconstruction') + parser.add_argument('dataset', + help='path to the dataset to be processed') + parser.add_argument('--image', + help='name of the image to show') + parser.add_argument('--save-figs', + help='save figures istead of showing them', + action='store_true') + args = parser.parse_args() + + data = dataset.DataSet(args.dataset) + + images = [args.image] if args.image else data.images() + for image in images: + depth, plane, score, nbour, nbours = data.load_raw_depthmap(image) + + print "Plotting depthmap for {0}".format(image) + pl.close("all") + pl.figure(figsize=(30, 16), dpi=90, facecolor='w', edgecolor='k') + title = "Shot: " + image + "\n" + "\n".join(wrap("Neighbors: " + ', '.join(nbours), 80)) + plot_depthmap(data.image_as_array(image), title, depth, plane, score, nbour) + pl.tight_layout() + + if args.save_figs: + p = args.dataset + '/plot_depthmaps' + io.mkdir_p(p) + pl.savefig(p + '/' + image + '.png') + pl.close() + else: + pl.show() From e584debf26ba5d6429141ffdf31008cced1d5445 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Thu, 30 Mar 2017 07:08:38 +0000 Subject: [PATCH 61/75] Rename neighbor variables. --- bin/plot_depthmaps | 12 +++--- opensfm/dataset.py | 6 +-- opensfm/dense.py | 12 +++--- opensfm/src/depthmap.cc | 72 ++++++++++++++++----------------- opensfm/src/depthmap_wrapper.cc | 16 ++++---- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/bin/plot_depthmaps b/bin/plot_depthmaps index d286f1321..965e5cbef 100644 --- a/bin/plot_depthmaps +++ b/bin/plot_depthmaps @@ -9,7 +9,7 @@ from opensfm import dataset from opensfm import io -def plot_depthmap(im, title, depth, plane, score, nbour): +def plot_depthmap(im, title, depth, plane, score, nghbr): ax = pl.subplot2grid((2, 3), (0, 0), rowspan=2) ax_title = ax.set_title(title) ax_title.set_y(1.05) @@ -25,9 +25,9 @@ def plot_depthmap(im, title, depth, plane, score, nbour): pl.imshow(score) pl.colorbar() ax = pl.subplot(2, 3, 5) - ax_title = ax.set_title("Neighbour") + ax_title = ax.set_title("Neighbor") ax_title.set_y(1.08) - pl.imshow(nbour) + pl.imshow(nghbr) pl.colorbar() ax = pl.subplot(2, 3, 6) ax_title = ax.set_title("Plane normal") @@ -58,13 +58,13 @@ if __name__ == "__main__": images = [args.image] if args.image else data.images() for image in images: - depth, plane, score, nbour, nbours = data.load_raw_depthmap(image) + depth, plane, score, nghbr, nghbrs = data.load_raw_depthmap(image) print "Plotting depthmap for {0}".format(image) pl.close("all") pl.figure(figsize=(30, 16), dpi=90, facecolor='w', edgecolor='k') - title = "Shot: " + image + "\n" + "\n".join(wrap("Neighbors: " + ', '.join(nbours), 80)) - plot_depthmap(data.image_as_array(image), title, depth, plane, score, nbour) + title = "Shot: " + image + "\n" + "\n".join(wrap("Neighbors: " + ', '.join(nghbrs), 80)) + plot_depthmap(data.image_as_array(image), title, depth, plane, score, nghbr) pl.tight_layout() if args.save_figs: diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 4c4e6e288..62dd86752 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -113,14 +113,14 @@ def _depthmap_file(self, image, suffix): def raw_depthmap_exists(self, image): return os.path.isfile(self._depthmap_file(image, 'raw.npz')) - def save_raw_depthmap(self, image, depth, plane, score, nbour, nbours): + def save_raw_depthmap(self, image, depth, plane, score, nghbr, nghbrs): io.mkdir_p(self._depthmap_path()) filepath = self._depthmap_file(image, 'raw.npz') - np.savez_compressed(filepath, depth=depth, plane=plane, score=score, nbour=nbour, nbours=nbours) + np.savez_compressed(filepath, depth=depth, plane=plane, score=score, nghbr=nghbr, nghbrs=nghbrs) def load_raw_depthmap(self, image): o = np.load(self._depthmap_file(image, 'raw.npz')) - return o['depth'], o['plane'], o['score'], o['nbour'], o['nbours'] + return o['depth'], o['plane'], o['score'], o['nghbr'], o['nghbrs'] def clean_depthmap_exists(self, image): return os.path.isfile(self._depthmap_file(image, 'clean.npz')) diff --git a/opensfm/dense.py b/opensfm/dense.py index 2a6df3c5e..5f77cffa9 100644 --- a/opensfm/dense.py +++ b/opensfm/dense.py @@ -67,12 +67,12 @@ def compute_depthmap(arguments): de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations']) de.set_min_patch_sd(data.config['depthmap_min_patch_sd']) add_views_to_depth_estimator(data, reconstruction, neighbors[shot.id], de) - depth, plane, score, nbour = de.compute_patch_match() + depth, plane, score, nghbr = de.compute_patch_match() good_score = score > data.config['depthmap_min_correlation_score'] depth = depth * (depth < max_depth) * good_score # Save and display results - data.save_raw_depthmap(shot.id, depth, plane, score, nbour, neighbors[shot.id][1:]) + data.save_raw_depthmap(shot.id, depth, plane, score, nghbr, neighbors[shot.id][1:]) if data.config['depthmap_save_debug_files']: image = data.undistorted_image_as_array(shot.id) @@ -96,7 +96,7 @@ def compute_depthmap(arguments): plt.imshow(score) plt.colorbar() plt.subplot(2, 3, 5) - plt.imshow(nbour) + plt.imshow(nghbr) plt.colorbar() plt.show() @@ -117,7 +117,7 @@ def clean_depthmap(arguments): depth = dc.clean() # Save and display results - raw_depth, raw_plane, raw_score, raw_nbour, nbours = data.load_raw_depthmap(shot.id) + raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = data.load_raw_depthmap(shot.id) data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score) if data.config['depthmap_save_debug_files']: @@ -197,7 +197,7 @@ def add_views_to_depth_cleaner(data, reconstruction, neighbors, dc): shot = reconstruction.shots[neighbor] if not data.raw_depthmap_exists(shot.id): continue - depth, plane, score, nbour, nbours = data.load_raw_depthmap(shot.id) + depth, plane, score, nghbr, nghbrs = data.load_raw_depthmap(shot.id) height, width = depth.shape K = shot.camera.get_K_in_pixel_coordinates(width, height) R = shot.pose.get_rotation_matrix() @@ -219,7 +219,7 @@ def compute_depth_range(graph, reconstruction, shot): def find_neighboring_images(shot, common_tracks, reconstruction, num_neighbors=5): - """Find neighbouring images based on common tracks.""" + """Find neighboring images based on common tracks.""" theta_min = np.pi / 60 theta_max = np.pi / 6 ns = [] diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 3a6848776..581846d2f 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -182,11 +182,11 @@ class DepthmapEstimator { min_patch_variance_ = sd * sd; } - void ComputeBruteForce(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void ComputeBruteForce(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); + *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { @@ -195,28 +195,28 @@ class DepthmapEstimator { float depth = 1 / (1 / min_depth_ + d * (1 / max_depth_ - 1 / min_depth_) / (num_depth_planes_ - 1)); cv::Vec3f normal(0, 0, -1); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane); + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane); } } } } - void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nbour = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); + *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); - RandomInitialization(best_depth, best_plane, best_score, best_nbour); - ComputeIgnoreMask(best_depth, best_plane, best_score, best_nbour); + RandomInitialization(best_depth, best_plane, best_score, best_nghbr); + ComputeIgnoreMask(best_depth, best_plane, best_score, best_nghbr); for (int i = 0; i < patchmatch_iterations_; ++i) { - PatchMatchForwardPass(best_depth, best_plane, best_score, best_nbour); - PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nbour); + PatchMatchForwardPass(best_depth, best_plane, best_score, best_nghbr); + PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nghbr); } } - void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { @@ -228,12 +228,12 @@ class DepthmapEstimator { best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; best_score->at(i, j) = score; - best_nbour->at(i, j) = other; + best_nghbr->at(i, j) = other; } } } - void ComputeIgnoreMask(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void ComputeIgnoreMask(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { @@ -242,7 +242,7 @@ class DepthmapEstimator { best_depth->at(i, j) = 0; best_plane->at(i, j) = cv::Vec3f(0, 0, 0); best_score->at(i, j) = 0; - best_nbour->at(i, j) = 0; + best_nghbr->at(i, j) = 0; } } } @@ -261,27 +261,27 @@ class DepthmapEstimator { } - void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { int neighbors[2][2] = {{-1, 0}, {0, -1}}; int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nbour, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors); } } } - void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour) { + void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { int neighbors[2][2] = {{0, 1}, {1, 0}}; int hpz = (patch_size_ - 1) / 2; for (int i = best_depth->rows - hpz - 1; i >= hpz; --i) { for (int j = best_depth->cols - hpz - 1; j >= hpz; --j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nbour, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors); } } } - void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, + void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, int i, int j, int neighbors[2][2]) { // Ignore pixels with depth == 0. @@ -292,18 +292,18 @@ class DepthmapEstimator { // Check neighbor's planes. for (int k = 0; k < 2; ++k) { cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); - int nbour = best_nbour->at(i + neighbors[k][0], j + neighbors[k][1]); - if (nbour == 0) { + int nghbr = best_nghbr->at(i + neighbors[k][0], j + neighbors[k][1]); + if (nghbr == 0) { continue; } - CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, nghbr); } // Check random planes. float depth_range = (1 / max_depth_ - 1 / min_depth_) / 20; float normal_range = 0.5; - int current_nbour = best_nbour->at(i, j); + int current_nghbr = best_nghbr->at(i, j); for (int k = 0; k < 6; ++k) { float current_depth = best_depth->at(i, j); float depth = 1 / (1 / current_depth + UniformRand(-depth_range, depth_range)); @@ -314,7 +314,7 @@ class DepthmapEstimator { -1.0f); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, current_nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, current_nghbr); depth_range *= 0.5; normal_range *= 0.5; @@ -324,48 +324,48 @@ class DepthmapEstimator { return; } - int other_nbour = uni_(rng_); - while (other_nbour == current_nbour) { - other_nbour = uni_(rng_); + int other_nghbr = uni_(rng_); + while (other_nghbr == current_nghbr) { + other_nghbr = uni_(rng_); } cv::Vec3f plane = best_plane->at(i, j); - CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nbour, i, j, plane, other_nbour); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, other_nghbr); } - void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, + void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, int i, int j, const cv::Vec3f &plane) { std::pair score = ComputePlaneScore(i, j, plane); if (score.first > best_score->at(i, j)) { best_score->at(i, j) = score.first; best_plane->at(i, j) = plane; best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); - best_nbour->at(i, j) = score.second; + best_nghbr->at(i, j) = score.second; } } - void CheckPlaneImageCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nbour, - int i, int j, const cv::Vec3f &plane, int nbour) { - float score = ComputePlaneImageScore(i, j, plane, nbour); + void CheckPlaneImageCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, + int i, int j, const cv::Vec3f &plane, int nghbr) { + float score = ComputePlaneImageScore(i, j, plane, nghbr); if (score > best_score->at(i, j)) { best_score->at(i, j) = score; best_plane->at(i, j) = plane; best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); - best_nbour->at(i, j) = nbour; + best_nghbr->at(i, j) = nghbr; } } std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { float best_score = -1.0f; - int best_nbour = 0; + int best_nghbr = 0; for (int other = 1; other < images_.size(); ++other) { float score = ComputePlaneImageScore(i, j, plane, other); if (score > best_score) { best_score = score; - best_nbour = other; + best_nghbr = other; } } - return std::make_pair(best_score, best_nbour); + return std::make_pair(best_score, best_nghbr); } float ComputePlaneImageScoreUnoptimized(int i, int j, diff --git a/opensfm/src/depthmap_wrapper.cc b/opensfm/src/depthmap_wrapper.cc index 6294e08c4..2f370d6f0 100644 --- a/opensfm/src/depthmap_wrapper.cc +++ b/opensfm/src/depthmap_wrapper.cc @@ -32,28 +32,28 @@ class DepthmapEstimatorWrapper { } bp::object ComputePatchMatch() { - cv::Mat depth, plane, score, nbour; - de_.ComputePatchMatch(&depth, &plane, &score, &nbour); - return ComputeReturnValues(depth, plane, score, nbour); + cv::Mat depth, plane, score, nghbr; + de_.ComputePatchMatch(&depth, &plane, &score, &nghbr); + return ComputeReturnValues(depth, plane, score, nghbr); } bp::object ComputeBruteForce() { - cv::Mat depth, plane, score, nbour; - de_.ComputeBruteForce(&depth, &plane, &score, &nbour); - return ComputeReturnValues(depth, plane, score, nbour); + cv::Mat depth, plane, score, nghbr; + de_.ComputeBruteForce(&depth, &plane, &score, &nghbr); + return ComputeReturnValues(depth, plane, score, nghbr); } bp::object ComputeReturnValues(const cv::Mat &depth, const cv::Mat &plane, const cv::Mat &score, - const cv::Mat &nbour) { + const cv::Mat &nghbr) { bp::list retn; npy_intp shape[2] = {depth.rows, depth.cols}; npy_intp plane_shape[3] = {depth.rows, depth.cols, 3}; retn.append(bpn_array_from_data(2, shape, depth.ptr(0))); retn.append(bpn_array_from_data(3, plane_shape, plane.ptr(0))); retn.append(bpn_array_from_data(2, shape, score.ptr(0))); - retn.append(bpn_array_from_data(2, shape, nbour.ptr(0))); + retn.append(bpn_array_from_data(2, shape, nghbr.ptr(0))); return retn; } From 8e341c4c2f2b51f77c5a65d020d9c22ae9672a34 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Thu, 30 Mar 2017 10:25:05 +0000 Subject: [PATCH 62/75] Add method for running patch match with sampling. Make it possible to run either regular patch match or patch match with single view sampling. --- opensfm/src/csfm.cc | 1 + opensfm/src/depthmap.cc | 83 +++++++++++++++++++++++---------- opensfm/src/depthmap_wrapper.cc | 6 +++ 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/opensfm/src/csfm.cc b/opensfm/src/csfm.cc index c528d300b..c7223a7db 100644 --- a/opensfm/src/csfm.cc +++ b/opensfm/src/csfm.cc @@ -165,6 +165,7 @@ BOOST_PYTHON_MODULE(csfm) { .def("set_min_patch_sd", &csfm::DepthmapEstimatorWrapper::SetMinPatchSD) .def("add_view", &csfm::DepthmapEstimatorWrapper::AddView) .def("compute_patch_match", &csfm::DepthmapEstimatorWrapper::ComputePatchMatch) + .def("compute_patch_match_sample", &csfm::DepthmapEstimatorWrapper::ComputePatchMatchSample) .def("compute_brute_force", &csfm::DepthmapEstimatorWrapper::ComputeBruteForce) ; diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 581846d2f..7571f22fd 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -183,10 +183,7 @@ class DepthmapEstimator { } void ComputeBruteForce(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { - *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); - *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); + AssignMatrices(best_depth, best_plane, best_score, best_nghbr); int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { @@ -202,11 +199,7 @@ class DepthmapEstimator { } void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { - *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); - *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); - *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); - + AssignMatrices(best_depth, best_plane, best_score, best_nghbr); RandomInitialization(best_depth, best_plane, best_score, best_nghbr); ComputeIgnoreMask(best_depth, best_plane, best_score, best_nghbr); @@ -216,19 +209,45 @@ class DepthmapEstimator { } } - void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { + void ComputePatchMatchSample(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { + AssignMatrices(best_depth, best_plane, best_score, best_nghbr); + RandomInitialization(best_depth, best_plane, best_score, best_nghbr, true); + ComputeIgnoreMask(best_depth, best_plane, best_score, best_nghbr); + + for (int i = 0; i < patchmatch_iterations_; ++i) { + PatchMatchForwardPass(best_depth, best_plane, best_score, best_nghbr, true); + PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nghbr, true); + } + } + + void AssignMatrices(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { + *best_depth = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); + *best_plane = cv::Mat(images_[0].rows, images_[0].cols, CV_32FC3, 0.0f); + *best_score = cv::Mat(images_[0].rows, images_[0].cols, CV_32F, 0.0f); + *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); + } + + void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, bool sample = false) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { float depth = UniformRand(min_depth_, max_depth_); cv::Vec3f normal(UniformRand(-1, 1), UniformRand(-1, 1), -1); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - int other = uni_(rng_); - float score = ComputePlaneImageScore(i, j, plane, other); + int nghbr; + float score; + if (sample) { + nghbr = uni_(rng_); + score = ComputePlaneImageScore(i, j, plane, nghbr); + } else { + std::pair result = ComputePlaneScore(i, j, plane); + nghbr = result.second; + score = result.first; + } best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; best_score->at(i, j) = score; - best_nghbr->at(i, j) = other; + best_nghbr->at(i, j) = nghbr; } } } @@ -261,17 +280,19 @@ class DepthmapEstimator { } - void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { + void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, + bool sample = false) { int neighbors[2][2] = {{-1, 0}, {0, -1}}; int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors, sample); } } } - void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { + void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, + bool sample = false) { int neighbors[2][2] = {{0, 1}, {1, 0}}; int hpz = (patch_size_ - 1) / 2; for (int i = best_depth->rows - hpz - 1; i >= hpz; --i) { @@ -283,24 +304,31 @@ class DepthmapEstimator { void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, int i, int j, - int neighbors[2][2]) { + int neighbors[2][2], + bool sample = false) { // Ignore pixels with depth == 0. if (best_depth->at(i, j) == 0.0f) { return; } - // Check neighbor's planes. + // Check neighbors and their planes for adjacent pixels. for (int k = 0; k < 2; ++k) { - cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); - int nghbr = best_nghbr->at(i + neighbors[k][0], j + neighbors[k][1]); - if (nghbr == 0) { + // Do not propagate ignored adjacent pixels. + if (best_depth->at(i + neighbors[k][0], j + neighbors[k][1]) == 0.0f) { continue; } - CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, nghbr); + cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); + + if (sample) { + int nghbr = best_nghbr->at(i + neighbors[k][0], j + neighbors[k][1]); + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, nghbr); + } else { + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane); + } } - // Check random planes. + // Check random planes for current neighbor. float depth_range = (1 / max_depth_ - 1 / min_depth_) / 20; float normal_range = 0.5; int current_nghbr = best_nghbr->at(i, j); @@ -314,16 +342,21 @@ class DepthmapEstimator { -1.0f); cv::Vec3f plane = PlaneFromDepthAndNormal(j, i, Ks_[0], depth, normal); - CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, current_nghbr); + if (sample) { + CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, current_nghbr); + } else { + CheckPlaneCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane); + } depth_range *= 0.5; normal_range *= 0.5; } - if (images_.size() <= 2) { + if (!sample || images_.size() <= 2) { return; } + // Check random other neighbor for current plane. int other_nghbr = uni_(rng_); while (other_nghbr == current_nghbr) { other_nghbr = uni_(rng_); diff --git a/opensfm/src/depthmap_wrapper.cc b/opensfm/src/depthmap_wrapper.cc index 2f370d6f0..c027ba359 100644 --- a/opensfm/src/depthmap_wrapper.cc +++ b/opensfm/src/depthmap_wrapper.cc @@ -37,6 +37,12 @@ class DepthmapEstimatorWrapper { return ComputeReturnValues(depth, plane, score, nghbr); } + bp::object ComputePatchMatchSample() { + cv::Mat depth, plane, score, nghbr; + de_.ComputePatchMatchSample(&depth, &plane, &score, &nghbr); + return ComputeReturnValues(depth, plane, score, nghbr); + } + bp::object ComputeBruteForce() { cv::Mat depth, plane, score, nghbr; de_.ComputeBruteForce(&depth, &plane, &score, &nghbr); From ee7987d63ac34b23f2020233a47174ca901009f8 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Thu, 30 Mar 2017 10:53:26 +0000 Subject: [PATCH 63/75] Add config param for choosing depthmap algorithm. --- opensfm/config.py | 1 + opensfm/dense.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/opensfm/config.py b/opensfm/config.py index d183c2cd2..a3b1c45d0 100644 --- a/opensfm/config.py +++ b/opensfm/config.py @@ -99,6 +99,7 @@ nav_rotation_threshold: 30 # Maximum general rotation in degrees between cameras for steps # Params for depth estimation +depthmap_method: PATCH_MATCH # Raw depthmap computationg algorithm (PATCH_MATCH, BRUTE_FORCE, PATCH_MATCH_SAMPLE) depthmap_resolution: 640 # Resolution of the depth maps depthmap_num_neighbors: 10 # Number of neighboring views depthmap_num_matching_views: 2 # Number of neighboring views used for each depthmaps diff --git a/opensfm/dense.py b/opensfm/dense.py index 5f77cffa9..19cda3597 100644 --- a/opensfm/dense.py +++ b/opensfm/dense.py @@ -56,18 +56,26 @@ def parallel_run(function, arguments, num_processes): def compute_depthmap(arguments): """Compute depthmap for a single shot.""" data, reconstruction, neighbors, min_depth, max_depth, shot = arguments + method = data.config['depthmap_method'] if data.raw_depthmap_exists(shot.id): logger.info("Using precomputed raw depthmap {}".format(shot.id)) return - logger.info("Computing depthmap for image {}".format(shot.id)) + logger.info("Computing depthmap for image {0} with {1}".format(shot.id, method)) de = csfm.DepthmapEstimator() de.set_depth_range(min_depth, max_depth, 100) de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations']) de.set_min_patch_sd(data.config['depthmap_min_patch_sd']) add_views_to_depth_estimator(data, reconstruction, neighbors[shot.id], de) - depth, plane, score, nghbr = de.compute_patch_match() + + if (method == 'BRUTE_FORCE'): + depth, plane, score, nghbr = de.compute_brute_force() + elif (method == 'PATCH_MATCH_SAMPLE'): + depth, plane, score, nghbr = de.compute_patch_match_sample() + else: + depth, plane, score, nghbr = de.compute_patch_match() + good_score = score > data.config['depthmap_min_correlation_score'] depth = depth * (depth < max_depth) * good_score From 89629eddf04b39055c786d026b4e6d993667b578 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Fri, 31 Mar 2017 07:47:35 +0000 Subject: [PATCH 64/75] Use references instead of returning pair. --- opensfm/src/depthmap.cc | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 7571f22fd..42a137faa 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -1,5 +1,4 @@ #include -#include #include @@ -240,9 +239,9 @@ class DepthmapEstimator { nghbr = uni_(rng_); score = ComputePlaneImageScore(i, j, plane, nghbr); } else { - std::pair result = ComputePlaneScore(i, j, plane); - nghbr = result.second; - score = result.first; + float score; + int nghbr; + ComputePlaneScore(i, j, plane, score, nghbr); } best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; @@ -297,7 +296,7 @@ class DepthmapEstimator { int hpz = (patch_size_ - 1) / 2; for (int i = best_depth->rows - hpz - 1; i >= hpz; --i) { for (int j = best_depth->cols - hpz - 1; j >= hpz; --j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors, sample); } } } @@ -368,12 +367,14 @@ class DepthmapEstimator { void CheckPlaneCandidate(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, int i, int j, const cv::Vec3f &plane) { - std::pair score = ComputePlaneScore(i, j, plane); - if (score.first > best_score->at(i, j)) { - best_score->at(i, j) = score.first; + float score; + int nghbr; + ComputePlaneScore(i, j, plane, score, nghbr); + if (score > best_score->at(i, j)) { + best_score->at(i, j) = score; best_plane->at(i, j) = plane; best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); - best_nghbr->at(i, j) = score.second; + best_nghbr->at(i, j) = nghbr; } } @@ -388,17 +389,16 @@ class DepthmapEstimator { } } - std::pair ComputePlaneScore(int i, int j, const cv::Vec3f &plane) { - float best_score = -1.0f; - int best_nghbr = 0; + void ComputePlaneScore(int i, int j, const cv::Vec3f &plane, float &score, int &nghbr) { + score = -1.0f; + nghbr = 0; for (int other = 1; other < images_.size(); ++other) { - float score = ComputePlaneImageScore(i, j, plane, other); - if (score > best_score) { - best_score = score; - best_nghbr = other; + float image_score = ComputePlaneImageScore(i, j, plane, other); + if (image_score > score) { + score = image_score; + nghbr = other; } } - return std::make_pair(best_score, best_nghbr); } float ComputePlaneImageScoreUnoptimized(int i, int j, From ddf710cf72e9849f1fdd8f97a11ed0d616e5b785 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Fri, 31 Mar 2017 09:21:24 +0000 Subject: [PATCH 65/75] Throw if depthmap method is not supported. Throw to make it clear what depthmap methods are supported. Do not use defualt param values. Rename pixel neighbor to adjacent for clarity. --- opensfm/dense.py | 5 ++++- opensfm/src/depthmap.cc | 33 ++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/opensfm/dense.py b/opensfm/dense.py index 19cda3597..6dac650ad 100644 --- a/opensfm/dense.py +++ b/opensfm/dense.py @@ -71,10 +71,13 @@ def compute_depthmap(arguments): if (method == 'BRUTE_FORCE'): depth, plane, score, nghbr = de.compute_brute_force() + elif (method == 'PATCH_MATCH'): + depth, plane, score, nghbr = de.compute_patch_match() elif (method == 'PATCH_MATCH_SAMPLE'): depth, plane, score, nghbr = de.compute_patch_match_sample() else: - depth, plane, score, nghbr = de.compute_patch_match() + raise ValueError('Unknown depthmap method type ' \ + '(must be BRUTE_FORCE, PATCH_MATCH or PATCH_MATCH_SAMPLE)') good_score = score > data.config['depthmap_min_correlation_score'] depth = depth * (depth < max_depth) * good_score diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 42a137faa..8dd735e3c 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -199,12 +199,12 @@ class DepthmapEstimator { void ComputePatchMatch(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr) { AssignMatrices(best_depth, best_plane, best_score, best_nghbr); - RandomInitialization(best_depth, best_plane, best_score, best_nghbr); + RandomInitialization(best_depth, best_plane, best_score, best_nghbr, false); ComputeIgnoreMask(best_depth, best_plane, best_score, best_nghbr); for (int i = 0; i < patchmatch_iterations_; ++i) { - PatchMatchForwardPass(best_depth, best_plane, best_score, best_nghbr); - PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nghbr); + PatchMatchForwardPass(best_depth, best_plane, best_score, best_nghbr, false); + PatchMatchBackwardPass(best_depth, best_plane, best_score, best_nghbr, false); } } @@ -226,7 +226,7 @@ class DepthmapEstimator { *best_nghbr = cv::Mat(images_[0].rows, images_[0].cols, CV_32S, cv::Scalar(0)); } - void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, bool sample = false) { + void RandomInitialization(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, bool sample) { int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { @@ -280,31 +280,31 @@ class DepthmapEstimator { void PatchMatchForwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, - bool sample = false) { - int neighbors[2][2] = {{-1, 0}, {0, -1}}; + bool sample) { + int adjacent[2][2] = {{-1, 0}, {0, -1}}; int hpz = (patch_size_ - 1) / 2; for (int i = hpz; i < best_depth->rows - hpz; ++i) { for (int j = hpz; j < best_depth->cols - hpz; ++j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors, sample); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, adjacent, sample); } } } void PatchMatchBackwardPass(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, - bool sample = false) { - int neighbors[2][2] = {{0, 1}, {1, 0}}; + bool sample) { + int adjacent[2][2] = {{0, 1}, {1, 0}}; int hpz = (patch_size_ - 1) / 2; for (int i = best_depth->rows - hpz - 1; i >= hpz; --i) { for (int j = best_depth->cols - hpz - 1; j >= hpz; --j) { - PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, neighbors, sample); + PatchMatchUpdatePixel(best_depth, best_plane, best_score, best_nghbr, i, j, adjacent, sample); } } } void PatchMatchUpdatePixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, int i, int j, - int neighbors[2][2], - bool sample = false) { + int adjacent[2][2], + bool sample) { // Ignore pixels with depth == 0. if (best_depth->at(i, j) == 0.0f) { return; @@ -312,15 +312,18 @@ class DepthmapEstimator { // Check neighbors and their planes for adjacent pixels. for (int k = 0; k < 2; ++k) { + int i_adjacent = i + adjacent[k][0]; + int j_adjacent = j + adjacent[k][1]; + // Do not propagate ignored adjacent pixels. - if (best_depth->at(i + neighbors[k][0], j + neighbors[k][1]) == 0.0f) { + if (best_depth->at(i_adjacent, j_adjacent) == 0.0f) { continue; } - cv::Vec3f plane = best_plane->at(i + neighbors[k][0], j + neighbors[k][1]); + cv::Vec3f plane = best_plane->at(i_adjacent, j_adjacent); if (sample) { - int nghbr = best_nghbr->at(i + neighbors[k][0], j + neighbors[k][1]); + int nghbr = best_nghbr->at(i_adjacent, j_adjacent); CheckPlaneImageCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane, nghbr); } else { CheckPlaneCandidate(best_depth, best_plane, best_score, best_nghbr, i, j, plane); From 9bd7e4ae06e9250724d5df464ec36b88b7f3c6c6 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Fri, 31 Mar 2017 11:48:55 +0000 Subject: [PATCH 66/75] Refactor pixel assignment to method. --- opensfm/src/depthmap.cc | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 8dd735e3c..2a5cffd7c 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -243,10 +243,7 @@ class DepthmapEstimator { int nghbr; ComputePlaneScore(i, j, plane, score, nghbr); } - best_depth->at(i, j) = depth; - best_plane->at(i, j) = plane; - best_score->at(i, j) = score; - best_nghbr->at(i, j) = nghbr; + AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, depth, plane, score, nghbr); } } } @@ -257,10 +254,7 @@ class DepthmapEstimator { for (int j = hpz; j < best_depth->cols - hpz; ++j) { float variance = PatchVariance(i, j); if (variance < min_patch_variance_) { - best_depth->at(i, j) = 0; - best_plane->at(i, j) = cv::Vec3f(0, 0, 0); - best_score->at(i, j) = 0; - best_nghbr->at(i, j) = 0; + AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, 0.0f, cv::Vec3f(0, 0, 0), 0.0f, 0); } } } @@ -374,10 +368,8 @@ class DepthmapEstimator { int nghbr; ComputePlaneScore(i, j, plane, score, nghbr); if (score > best_score->at(i, j)) { - best_score->at(i, j) = score; - best_plane->at(i, j) = plane; - best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); - best_nghbr->at(i, j) = nghbr; + float depth = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); + AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, depth, plane, score, nghbr); } } @@ -385,11 +377,17 @@ class DepthmapEstimator { int i, int j, const cv::Vec3f &plane, int nghbr) { float score = ComputePlaneImageScore(i, j, plane, nghbr); if (score > best_score->at(i, j)) { - best_score->at(i, j) = score; + float depth = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); + AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, depth, plane, score, nghbr); + } + } + + void AssignPixel(cv::Mat *best_depth, cv::Mat *best_plane, cv::Mat *best_score, cv::Mat *best_nghbr, + int i, int j, const float depth, const cv::Vec3f &plane, const float score, const int nghbr) { + best_depth->at(i, j) = depth; best_plane->at(i, j) = plane; - best_depth->at(i, j) = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); + best_score->at(i, j) = score; best_nghbr->at(i, j) = nghbr; - } } void ComputePlaneScore(int i, int j, const cv::Vec3f &plane, float &score, int &nghbr) { From 079608da6284af05a431a09b91766f631038a284 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Tue, 4 Apr 2017 10:53:42 +0000 Subject: [PATCH 67/75] Use pointers for return values. --- opensfm/src/depthmap.cc | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/opensfm/src/depthmap.cc b/opensfm/src/depthmap.cc index 2a5cffd7c..debe28722 100644 --- a/opensfm/src/depthmap.cc +++ b/opensfm/src/depthmap.cc @@ -239,9 +239,7 @@ class DepthmapEstimator { nghbr = uni_(rng_); score = ComputePlaneImageScore(i, j, plane, nghbr); } else { - float score; - int nghbr; - ComputePlaneScore(i, j, plane, score, nghbr); + ComputePlaneScore(i, j, plane, &score, &nghbr); } AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, depth, plane, score, nghbr); } @@ -366,7 +364,7 @@ class DepthmapEstimator { int i, int j, const cv::Vec3f &plane) { float score; int nghbr; - ComputePlaneScore(i, j, plane, score, nghbr); + ComputePlaneScore(i, j, plane, &score, &nghbr); if (score > best_score->at(i, j)) { float depth = DepthOfPlaneBackprojection(j, i, Ks_[0], plane); AssignPixel(best_depth, best_plane, best_score, best_nghbr, i, j, depth, plane, score, nghbr); @@ -390,14 +388,14 @@ class DepthmapEstimator { best_nghbr->at(i, j) = nghbr; } - void ComputePlaneScore(int i, int j, const cv::Vec3f &plane, float &score, int &nghbr) { - score = -1.0f; - nghbr = 0; + void ComputePlaneScore(int i, int j, const cv::Vec3f &plane, float *score, int *nghbr) { + *score = -1.0f; + *nghbr = 0; for (int other = 1; other < images_.size(); ++other) { float image_score = ComputePlaneImageScore(i, j, plane, other); - if (image_score > score) { - score = image_score; - nghbr = other; + if (image_score > *score) { + *score = image_score; + *nghbr = other; } } } From 24ee416c77b2e309e14ac3c1030474b08c01956c Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Tue, 11 Apr 2017 13:02:51 +0200 Subject: [PATCH 68/75] Catch division by zero when parsing EXIF Fixes #85 --- opensfm/exif.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opensfm/exif.py b/opensfm/exif.py index f20fa41c8..e20d29bc7 100644 --- a/opensfm/exif.py +++ b/opensfm/exif.py @@ -31,7 +31,10 @@ def get_float_tag(tags, key): def get_frac_tag(tags, key): if key in tags: - return eval_frac(tags[key].values[0]) + try: + return eval_frac(tags[key].values[0]) + except ZeroDivisionError: + return None else: return None From a01c0bb171b98fdc2c922142f80d48e9d44b397d Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Thu, 13 Apr 2017 01:05:20 +0200 Subject: [PATCH 69/75] Save features in the features folder. Do not use the name of the features for naming the features folder and files --- opensfm/dataset.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 62dd86752..96383ab76 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -220,27 +220,16 @@ def feature_type(self): if self.config.get('feature_root', False): feature_name = 'root_' + feature_name return feature_name - def descriptor_type(self): - """Return the type of the descriptor (if exists) - """ - if self.feature_type() == 'akaze': - return self.config.get('akaze_descriptor', '') - else: - return '' - def __feature_path(self): """Return path of feature descriptors and FLANN indices directory""" - __feature_path = self.feature_type() - if len(self.descriptor_type()) > 0: - __feature_path += '_' + self.descriptor_type() - return os.path.join(self.data_path, __feature_path) + return os.path.join(self.data_path, "features") def __feature_file(self, image): """ Return path of feature file for specified image :param image: Image name, with extension (i.e. 123.jpg) """ - return os.path.join(self.__feature_path(), image + '.' + self.feature_type() + '.npz') + return os.path.join(self.__feature_path(), image + '.npz') def __save_features(self, filepath, image, points, descriptors, colors=None): io.mkdir_p(self.__feature_path()) @@ -278,7 +267,7 @@ def __feature_index_file(self, image): Return path of FLANN index file for specified image :param image: Image name, with extension (i.e. 123.jpg) """ - return os.path.join(self.__feature_path(), image + '.' + self.feature_type() + '.flann') + return os.path.join(self.__feature_path(), image + '.flann') def load_feature_index(self, image, features): index = cv2.flann.Index() if context.OPENCV3 else cv2.flann_Index() @@ -294,7 +283,7 @@ def __preemptive_features_file(self, image): for specified image :param image: Image name, with extension (i.e. 123.jpg) """ - return os.path.join(self.__feature_path(), image + '_preemptive.' + self.feature_type() + '.npz') + return os.path.join(self.__feature_path(), image + '_preemptive' + '.npz') def load_preemtive_features(self, image): s = np.load(self.__preemptive_features_file(image)) From fe5c5ffaf1189ae04b48542f0ec0fc5b6841aef4 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Wed, 19 Apr 2017 22:32:38 +0200 Subject: [PATCH 70/75] Add commands description --- doc/source/_static/mathjax_conf.js | 2 +- doc/source/conf.py | 6 +-- doc/source/dataset.rst | 19 +++++++- doc/source/dense.rst | 2 +- doc/source/gcp.rst | 14 +++--- doc/source/index.rst | 4 -- doc/source/using.rst | 70 ++++++++++++++++++++++++++---- 7 files changed, 92 insertions(+), 25 deletions(-) diff --git a/doc/source/_static/mathjax_conf.js b/doc/source/_static/mathjax_conf.js index 95e4dc947..3b20bff7d 100644 --- a/doc/source/_static/mathjax_conf.js +++ b/doc/source/_static/mathjax_conf.js @@ -1,6 +1,6 @@ MathJax.Hub.Config({ "HTML-CSS": { availableFonts: ["TeX"], - scale: 90 + scale: 100 } }); diff --git a/doc/source/conf.py b/doc/source/conf.py index dadacf9e1..7bf13f8dd 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -54,7 +54,7 @@ # General information about the project. project = u'OpenSfM' -copyright = u'2016, Mapillary' +copyright = u'2017, Mapillary' author = u'Mapillary' # The version info for the project you're documenting, acts as replacement for @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = u'0.0' +version = u'0.1' # The full version, including alpha/beta/rc tags. -release = u'0.0.0' +release = u'0.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/dataset.rst b/doc/source/dataset.rst index 809dc5005..8ef6e74fe 100644 --- a/doc/source/dataset.rst +++ b/doc/source/dataset.rst @@ -4,4 +4,21 @@ Dataset Structure ================= -TODO +:: + + project/ + ├── config.yaml + ├── images/ + ├── masks/ + ├── gcp_list.txt + ├── metadata/ + ├── features/ + ├── matches/ + ├── tracks.tsv + ├── reconstruction.json + ├── reconstruction.meshed.json + ├── undistorted/ + ├── undistorted_tracks.json + ├── undistorted_reconstruction.json + └── depthmaps/ + └── merged.ply diff --git a/doc/source/dense.rst b/doc/source/dense.rst index afc2a4f30..c97a1d213 100644 --- a/doc/source/dense.rst +++ b/doc/source/dense.rst @@ -120,7 +120,7 @@ And the linear approximation around :math:`(x_0, y_0)` is Undistortion ------------ -The dense module assumes that images are taken with perspective projection and no radial distortion. For perspective images, undistorted versions can be generated by taking into account the computed distortion parameters, :math:`k1` and :math: `k2`. +The dense module assumes that images are taken with perspective projection and no radial distortion. For perspective images, undistorted versions can be generated by taking into account the computed distortion parameters, :math:`k1` and :math:`k2`. Equirectangular images (360 panoramas) however can not be unwarped into a single persepective view. We need to generate multiple perspective views to cover the field of view of a panorama. diff --git a/doc/source/gcp.rst b/doc/source/gcp.rst index 134d7e8e1..f3658890d 100644 --- a/doc/source/gcp.rst +++ b/doc/source/gcp.rst @@ -14,19 +14,20 @@ File format ``````````` GCPs can be specified by adding a text file named ``gcp_list.txt`` at the root folder of the dataset. The format of the file should be as follows. - - The first line should contain the name of the projection used for the geo coordinates. +- The first line should contain the name of the projection used for the geo coordinates. + +- The following lines should con should contain the data for each ground control point observation. One per line and in the format:: - - The following lines should con should contain the data for each ground control point observation. One per line and in the format - :: - Where `` `` are the geospatial coordinates of the GCP and `` `` are the pixel coordinates where the GCP is observed. + + Where `` `` are the geospatial coordinates of the GCP and `` `` are the pixel coordinates where the GCP is observed. Supported projections ````````````````````` The geospatial coordinates can be specified in one the following formats. -- WGS84: This is the standard latitude, longitude coordinates used by most GPS devices. In this case, `` = longitude``, `` = latitude`` and `` = altitude`` +- `WGS84`_: This is the standard latitude, longitude coordinates used by most GPS devices. In this case, `` = longitude``, `` = latitude`` and `` = altitude`` - `UTM`_: UTM projections can be specified using a string projection string such as ``WGS84 UTM 32N``, where 32 is the region and N is . In this case, `` = E``, `` = N`` and `` = altitude`` @@ -38,12 +39,11 @@ The geospatial coordinates can be specified in one the following formats. Example ``````` -:: +This file defines 2 GCP whose coordinates are specified in the WGS84 standard. The first one is observed in both ``01.jpg`` and ``02.jpg``, while the second one is only observed in ``01.jpg`` :: WGS84 13.400740745 52.519134104 12.0792090446 2335.0 1416.7 01.jpg 13.400740745 52.519134104 12.0792090446 2639.1 938.0 02.jpg 13.400502446 52.519251158 16.7021233002 766.0 1133.1 01.jpg -This file defines 2 GCP whose coordinates are specified in the WGS84 standard. The first one is observed in both ``01.jpg`` and ``02.jpg``, while the second one is only observed in ``01.jpg`` diff --git a/doc/source/index.rst b/doc/source/index.rst index 013e2afcc..527ec71ad 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,10 +3,6 @@ OpenSfM ======= -Only tests here for now. - -test contents: - .. toctree:: :maxdepth: 2 diff --git a/doc/source/using.rst b/doc/source/using.rst index effa7eedf..3689c3b15 100644 --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -79,35 +79,89 @@ Here is the usage page of ``bin/opensfm``, which lists the available commands extract_metadata ```````````````` -TODO + +This commands extracts EXIF metadata from the images an stores them in the ``exif`` folder and the ``camera_models.json`` file. + +The following data is extracted for each image: + +- ``width`` and ``height``: image size in pixels + +- ``gps`` ``latitude``, ``longitude``, ``altitude`` and ``dop``: The GPS coordinates of the camera at capture time and the corresponding Degree Of Precission). This is used to geolocate the reconstruction. + +- ``capture_time``: The capture time. Used to choose candidate matching images when the option ``matching_time_neighbors`` is set. + +- ``camera orientation``: The EXIF orientation tag (see this `exif orientation documentation`_). Used to orient the reconstruction straigh up. + +- ``projection_type``: The camera projection type. It is extracted from the GPano_ metadata and used to determine which projection to use for each camera. Supported types are `perspective`, `equirectangular` and `fisheye`. + +- ``focal_ratio``: The focal length provided by the EXIF metadata divided by the sensor width. This is used as initialization and prior for the camera focal length parameter. + +- ``make`` and ``model``: The camera make and model. Used to build the camera ID. + +- ``camera``: The camera ID string. Used to identify a camera. When multiple images have the same camera ID string, they will be assumed to be taken with the same camera and will share its parameters. + + +Once the metadata for all images has been extracted, a list of camera models is created and stored in ``camera_models.json``. A camera is created for each diferent camera ID string found on the images. + +For each camera the following data is stored: + +- ``width`` and ``height``: image size in pixels +- ``projection_type``: the camera projection type +- ``focal``: The initial estimation of the focal length (as a multiple of the sensor width). +- ``k1`` and ``k2``: The initial estimation of the radial distortion parameters. Only used for `perspective` and `fisheye` projection models. +- ``focal_prior``: The focal length prior. The final estimated focal length will be forced to be similar to it. +- ``k1_prior`` and ``k2_prior``: The radial distortion parameters prior. + + +Providing your own camera parameters +'''''''''''''''''''''''''''''''''''' + +By default, the camera parameters are taken from the EXIF metadata but it is also possible to override the default parameters. To do so, place a file named ``camera_models_override.json`` in the project folder. This file should have the same structure as ``camera_models.json``. When running the ``extract_metadata`` command, the parameters of any camera present in the ``camera_models_overrides.json`` file will be copied to ``camera_models.json`` overriding the default ones. + +Simplest way to create the ``camera_models_overrides.json`` file is to rename ``camera_models.json`` and modify the parameters. You will need to rerun the ``extract_metadata`` command after that. + + +.. _`exif orientation documentation`: http://sylvana.net/jpegcrop/exif_orientation.html +.. _GPano: TODO(pau): link to Google Pano metadata documentation + detect_features ``````````````` -TODO +This command detect feature points in the images and stores them in the `feature` folder. + match_features `````````````` -TODO +This command matches feature points between images and stores them in the `matches` folder. It first determines the list of image pairs to run, and then run the matching process for each pair to find corresponding feature points. + +Since there are a lot of possible image pairs, the process can be very slow. It can be speeded up by restricting the list of pairs to match. The pairs can be restricted by GPS distance, capture time or file name order. + create_tracks ````````````` -TODO +This command links the matches between pairs of images to build feature point tracks. The tracks are stored in the `tracks.csv` file. A track is a set of feature points from different images that have been recognized to correspond to the same pysical point. + reconstruct ``````````` -TODO +This command runs the incremental reconstruction process. The goal of the reconstruction process is to find the 3D position of tracks (the `structure`) together with the position of the cameras (the `motion`). The computed reconstruction is stored in the ``reconstruction.json`` file. + mesh ```` -TODO +This process computes a rough triangular mesh of the scene seen by each images. Such mesh is used for simulating smooth motions between images in the web viewer. The reconstruction with the mesh added is stored in ``reconstruction.meshed.json`` file. + +Note that the only difference between ``reconstruction.json`` and ``reconstruction.meshed.json`` is that the later contains the triangular meshes. If you don't need that, you only need the former file and there's no need to run this command. + undistort ````````` -TODO +This command creates undistorted version of the reconstruction, tracks and images. The undistorted version can later be used for computing depth maps. + compute_depthmaps ````````````````` -TODO +This commands computes a dense point cloud of the scene by computing and merging depthmaps. It requires an undistorted reconstructions. The resulting depthmaps are stored in the ``depthmaps`` folder and the merged point cloud is stored in ``depthmaps/merged.ply`` Configuration From 77d21a4396a5e8c046ed495eb8744b27ec59acc6 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 14 May 2017 17:48:25 +0200 Subject: [PATCH 71/75] Update openmvs Interface.h Close #169 --- opensfm/src/openmvs_exporter.h | 2 +- opensfm/src/third_party/openmvs/Interface.h | 227 +++++++++++++----- .../src/third_party/openmvs/README.opensfm | 2 +- 3 files changed, 175 insertions(+), 56 deletions(-) diff --git a/opensfm/src/openmvs_exporter.h b/opensfm/src/openmvs_exporter.h index 8dd7108d7..ce277ff61 100644 --- a/opensfm/src/openmvs_exporter.h +++ b/opensfm/src/openmvs_exporter.h @@ -73,7 +73,7 @@ class OpenMVSExporter { } void Export(std::string filename) { - ARCHIVE::SerializeSave(scene_, filename); + MVS::ARCHIVE::SerializeSave(scene_, filename); } private: diff --git a/opensfm/src/third_party/openmvs/Interface.h b/opensfm/src/third_party/openmvs/Interface.h index 5fa6dc4b8..eefe98aec 100644 --- a/opensfm/src/third_party/openmvs/Interface.h +++ b/opensfm/src/third_party/openmvs/Interface.h @@ -9,6 +9,14 @@ // D E F I N E S /////////////////////////////////////////////////// +#define MVSI_PROJECT_ID "MVSI" // identifies the project stream +#define MVSI_PROJECT_VER ((uint32_t)2) // identifies the version of a project stream + +// set a default namespace name is none given +#ifndef _INTERFACE_NAMESPACE +#define _INTERFACE_NAMESPACE MVS +#endif + // uncomment to enable custom OpenCV data types // (should be uncommented if OpenCV is not available) #if !defined(_USE_OPENCV) && !defined(_USE_CUSTOM_CV) @@ -27,46 +35,58 @@ namespace cv { // simple cv::Matx -template +template class Matx { public: - typedef _Tp Type; + typedef Type value_type; inline Matx() {} #ifdef _USE_EIGEN EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Type,m*n) typedef Eigen::Matrix1?Eigen::RowMajor:Eigen::Default)> EMat; typedef Eigen::Map CEMatMap; typedef Eigen::Map EMatMap; - inline Matx(const EMat& rhs) { operator EMatMap () = rhs; } - inline Matx& operator = (const EMat& rhs) { operator EMatMap () = rhs; return *this; } + template + inline Matx(const Eigen::EigenBase& rhs) { operator EMatMap () = rhs; } + template + inline Matx& operator = (const Eigen::EigenBase& rhs) { operator EMatMap () = rhs; return *this; } inline operator CEMatMap() const { return CEMatMap((const Type*)val); } inline operator EMatMap () { return EMatMap((Type*)val); } #endif + static Matx eye() { + Matx M; + memset(M.val, 0, sizeof(Type)*m*n); + const int shortdim(m < n ? m : n); + for (int i = 0; i < shortdim; i++) + M(i,i) = 1; + return M; + } Type operator()(int r, int c) const { return val[r*n+c]; } Type& operator()(int r, int c) { return val[r*n+c]; } public: - _Tp val[m*n]; + Type val[m*n]; }; // simple cv::Matx -template +template class Point3_ { public: - typedef _Tp Type; + typedef Type value_type; inline Point3_() {} #ifdef _USE_EIGEN EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Type,3) typedef Eigen::Matrix EVec; typedef Eigen::Map EVecMap; - inline Point3_(const EVec& rhs) { operator EVecMap () = rhs; } - inline Point3_& operator = (const EVec& rhs) { operator EVecMap () = rhs; return *this; } + template + inline Point3_(const Eigen::EigenBase& rhs) { operator EVecMap () = rhs; } + template + inline Point3_& operator = (const Eigen::EigenBase& rhs) { operator EVecMap () = rhs; return *this; } inline operator const EVecMap () const { return EVecMap((Type*)this); } inline operator EVecMap () { return EVecMap((Type*)this); } #endif public: - _Tp x, y, z; + Type x, y, z; }; } // namespace cv @@ -74,62 +94,110 @@ class Point3_ /*----------------------------------------------------------------*/ +namespace _INTERFACE_NAMESPACE { + // custom serialization namespace ARCHIVE { -struct ArchiveSave; -struct ArchiveLoad; - -template -bool Save(ArchiveSave& a, const _Tp& obj) { - const_cast<_Tp&>(obj).serialize(a, 0); - return true; -} -template -bool Load(ArchiveLoad& a, _Tp& obj) { - obj.serialize(a, 0); - return true; -} - - // Basic serialization types struct ArchiveSave { std::ostream& stream; - ArchiveSave(std::ostream& _stream) : stream(_stream) {} + uint32_t version; + ArchiveSave(std::ostream& _stream, uint32_t _version) + : stream(_stream), version(_version) {} template - ArchiveSave& operator & (const _Tp& obj) { - Save(*this, obj); - return *this; - } + ArchiveSave& operator & (const _Tp& obj); }; struct ArchiveLoad { std::istream& stream; - ArchiveLoad(std::istream& _stream) : stream(_stream) {} + uint32_t version; + ArchiveLoad(std::istream& _stream, uint32_t _version) + : stream(_stream), version(_version) {} template - ArchiveLoad& operator & (_Tp& obj) { - Load(*this, obj); - return *this; - } + ArchiveLoad& operator & (_Tp& obj); }; +template +bool Save(ArchiveSave& a, const _Tp& obj) { + const_cast<_Tp&>(obj).serialize(a, a.version); + return true; +} +template +bool Load(ArchiveLoad& a, _Tp& obj) { + obj.serialize(a, a.version); + return true; +} + +template +ArchiveSave& ArchiveSave::operator & (const _Tp& obj) { + Save(*this, obj); + return *this; +} +template +ArchiveLoad& ArchiveLoad::operator & (_Tp& obj) { + Load(*this, obj); + return *this; +} // Main exporter & importer template -bool SerializeSave(const _Tp& obj, const std::string& fileName) { +bool SerializeSave(const _Tp& obj, const std::string& fileName, uint32_t version=MVSI_PROJECT_VER) { + // open the output stream std::ofstream stream(fileName, std::ofstream::binary); if (!stream.is_open()) return false; - ARCHIVE::ArchiveSave serializer(stream); + // write header + if (version > 0) { + // save project ID + stream.write(MVSI_PROJECT_ID, 4); + // save project version + stream.write((const char*)&version, sizeof(uint32_t)); + // reserve some bytes + const uint32_t reserved(0); + stream.write((const char*)&reserved, sizeof(uint32_t)); + } + // serialize out the current state + ARCHIVE::ArchiveSave serializer(stream, version); serializer & obj; return true; } template -bool SerializeLoad(_Tp& obj, const std::string& fileName) { +bool SerializeLoad(_Tp& obj, const std::string& fileName, uint32_t* pVersion=NULL) { + // open the input stream std::ifstream stream(fileName, std::ifstream::binary); if (!stream.is_open()) return false; - ARCHIVE::ArchiveLoad serializer(stream); + // read header + uint32_t version(0); + // load project header ID + char szHeader[4]; + stream.read(szHeader, 4); + if (!stream) + return false; + if (strncmp(szHeader, MVSI_PROJECT_ID, 4) != 0) { + // try to load as the first version that didn't have a header + const size_t size(fileName.size()); + if (size <= 4) + return false; + std::string ext(fileName.substr(size-4)); + std::transform(ext.begin(), ext.end(), ext.begin(), ::towlower); + if (ext != ".mvs") + return false; + stream.seekg(0, std::ifstream::beg); + } else { + // load project version + stream.read((char*)&version, sizeof(uint32_t)); + if (!stream || version > MVSI_PROJECT_VER) + return false; + // skip reserved bytes + uint32_t reserved; + stream.read((char*)&reserved, sizeof(uint32_t)); + } + // serialize in the current state + ARCHIVE::ArchiveLoad serializer(stream, version); serializer & obj; + if (pVersion) + *pVersion = version; return true; } @@ -221,17 +289,16 @@ bool Load(ArchiveLoad& a, std::vector<_Tp>& v) { /*----------------------------------------------------------------*/ -namespace MVS { - // interface used to export/import MVS input data; -// MAX(width,height) is used for normalization +// - MAX(width,height) is used for normalization +// - row-major order is used for storing the matrices struct Interface { typedef cv::Point3_ Pos3f; typedef cv::Point3_ Pos3d; typedef cv::Matx Mat33d; - typedef uint8_t Color; - typedef cv::Point3_ Col3; // x=B, y=G, z=R + typedef cv::Matx Mat44d; + typedef cv::Point3_ Col3; // x=B, y=G, z=R /*----------------------------------------------------------------*/ // structure describing a mobile platform with cameras attached to it @@ -239,13 +306,22 @@ struct Interface // structure describing a camera mounted on a platform struct Camera { std::string name; // camera's name - Mat33d K; // camera's normalized intrinsics matrix + uint32_t width, height; // image resolution in pixels for all images sharing this camera (optional) + Mat33d K; // camera's intrinsics matrix (normalized if image resolution not specified) Mat33d R; // camera's rotation matrix relative to the platform Pos3d C; // camera's translation vector relative to the platform + Camera() : width(0), height(0) {} + bool HasResolution() const { return width > 0 && height > 0; } + bool IsNormalized() const { return !HasResolution(); } + template - void serialize(Archive& ar, const unsigned int /*version*/) { + void serialize(Archive& ar, const unsigned int version) { ar & name; + if (version > 0) { + ar & width; + ar & height; + } ar & K; ar & R; ar & C; @@ -325,8 +401,37 @@ struct Interface typedef std::vector VertexArr; /*----------------------------------------------------------------*/ + // structure describing a 3D line + struct Line { + // structure describing one view for a given 3D feature + struct View { + uint32_t imageID; // image ID corresponding to this view + float confidence; // view's confidence (0 - not available) + + template + void serialize(Archive& ar, const unsigned int /*version*/) { + ar & imageID; + ar & confidence; + } + }; + typedef std::vector ViewArr; + + Pos3f pt1; // 3D line segment end-point + Pos3f pt2; // 3D line segment end-point + ViewArr views; // list of all available views for this 3D feature + + template + void serialize(Archive& ar, const unsigned int /*version*/) { + ar & pt1; + ar & pt2; + ar & views; + } + }; + typedef std::vector LineArr; + /*----------------------------------------------------------------*/ + // structure describing a 3D point's normal (optional) - struct VertexNormal { + struct Normal { Pos3f n; // 3D feature normal template @@ -334,11 +439,11 @@ struct Interface ar & n; } }; - typedef std::vector VertexNormalArr; + typedef std::vector NormalArr; /*----------------------------------------------------------------*/ // structure describing a 3D point's color (optional) - struct VertexColor { + struct Color { Col3 c; // 3D feature color template @@ -346,26 +451,40 @@ struct Interface ar & c; } }; - typedef std::vector VertexColorArr; + typedef std::vector ColorArr; /*----------------------------------------------------------------*/ PlatformArr platforms; // array of platforms ImageArr images; // array of images VertexArr vertices; // array of reconstructed 3D points - VertexNormalArr verticesNormal; // array of reconstructed 3D points' normal (optional) - VertexColorArr verticesColor; // array of reconstructed 3D points' color (optional) + NormalArr verticesNormal; // array of reconstructed 3D points' normal (optional) + ColorArr verticesColor; // array of reconstructed 3D points' color (optional) + LineArr lines; // array of reconstructed 3D lines + NormalArr linesNormal; // array of reconstructed 3D lines' normal (optional) + ColorArr linesColor; // array of reconstructed 3D lines' color (optional) + Mat44d transform; // transformation used to convert from absolute to relative coordinate system (optional) + + Interface() : transform(Mat44d::eye()) {} template - void serialize(Archive& ar, const unsigned int /*version*/) { + void serialize(Archive& ar, const unsigned int version) { ar & platforms; ar & images; ar & vertices; ar & verticesNormal; ar & verticesColor; + if (version > 0) { + ar & lines; + ar & linesNormal; + ar & linesColor; + if (version > 1) { + ar & transform; + } + } } }; /*----------------------------------------------------------------*/ -} // namespace MVS +} // namespace _INTERFACE_NAMESPACE #endif // _INTERFACE_MVS_H_ diff --git a/opensfm/src/third_party/openmvs/README.opensfm b/opensfm/src/third_party/openmvs/README.opensfm index 82c73570d..036a4e399 100644 --- a/opensfm/src/third_party/openmvs/README.opensfm +++ b/opensfm/src/third_party/openmvs/README.opensfm @@ -1,7 +1,7 @@ Project: openMVS URL: https://github.com/cdcseacave/openMVS License: BSD -Version: 0.7, 2016-04-06 +Version: a3b360016660a1397f6eb6c070c2c19bbb4c7590, 2017-05-01 Local modifications: * Imported only the Interface.h file From 0d878401d3a8e5c7c2203002eca9d10dbb067fbc Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 14 May 2017 18:19:59 +0200 Subject: [PATCH 72/75] Open binary files in binary mode. Fixes #171 --- opensfm/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensfm/dataset.py b/opensfm/dataset.py index 96383ab76..bbf40125a 100644 --- a/opensfm/dataset.py +++ b/opensfm/dataset.py @@ -66,7 +66,7 @@ def __image_file(self, image): return self.image_files[image] def load_image(self, image): - return open(self.__image_file(image)) + return open(self.__image_file(image), 'rb') def image_as_array(self, image): """Return image pixels as 3-dimensional numpy array (R G B order)""" @@ -205,12 +205,12 @@ def load_exif(self, image): :param image: Image name, with extension (i.e. 123.jpg) """ - with open(self.__exif_file(image), 'r') as fin: + with open(self.__exif_file(image), 'rb') as fin: return json.load(fin) def save_exif(self, image, data): io.mkdir_p(self.__exif_path()) - with open(self.__exif_file(image), 'w') as fout: + with open(self.__exif_file(image), 'wb') as fout: io.json_dump(data, fout) def feature_type(self): From c230ce47429e93dac6ac79f163303b26eb398baf Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 14 May 2017 22:00:39 +0200 Subject: [PATCH 73/75] Remove old run_all (replaced by opensfm_run_all) --- README.md | 2 +- bin/run_all | 12 ------------ bin/run_eval | 2 +- doc/source/using.rst | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) delete mode 100755 bin/run_all diff --git a/README.md b/README.md index d6a09938d..5d320de84 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ An example dataset is available at `data/berlin`. 1. Put some images in `data/DATASET_NAME/images/` 2. Put config.yaml in `data/DATASET_NAME/config.yaml` - 3. Go to the root of the project and run `bin/run_all data/DATASET_NAME` + 3. Go to the root of the project and run `bin/opensfm_run_all data/DATASET_NAME` 4. Start an http server from the root with `python -m SimpleHTTPServer` 5. Browse `http://localhost:8000/viewer/reconstruction.html#file=/data/DATASET_NAME/reconstruction.meshed.json`. diff --git a/bin/run_all b/bin/run_all deleted file mode 100755 index 831677255..000000000 --- a/bin/run_all +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -$DIR/opensfm extract_metadata $1 -$DIR/opensfm detect_features $1 -$DIR/opensfm match_features $1 -$DIR/opensfm create_tracks $1 -$DIR/opensfm reconstruct $1 -$DIR/opensfm mesh $1 diff --git a/bin/run_eval b/bin/run_eval index e7e69b812..2ae07724c 100755 --- a/bin/run_eval +++ b/bin/run_eval @@ -17,7 +17,7 @@ from opensfm import types def run_dataset(run_root, name): folder = run_root + '/' + name - call(['bin/run_all', folder]) + call(['bin/opensfm_run_all', folder]) with open(run_root + '/index.html', 'a') as fout: s = ''' diff --git a/doc/source/using.rst b/doc/source/using.rst index 3689c3b15..7b6d11221 100644 --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -10,7 +10,7 @@ Quickstart An example dataset is available at ``data/berlin``. You can reconstruct it using by running :: - bin/run_all data/berlin + bin/opensfm_run_all data/berlin This will run the entire SfM pipeline and produce the file ``data/berlin/reconstruction.meshed.json`` as output. To visualize the result you can start a HTTP server running :: From 32dc83f4cb493ead05341279b371e1c5fe829d24 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Mon, 15 May 2017 23:03:54 +0200 Subject: [PATCH 74/75] Better resolution fit when undistorting panoramas --- opensfm/commands/undistort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/commands/undistort.py b/opensfm/commands/undistort.py index f46a7ee10..e6666119d 100644 --- a/opensfm/commands/undistort.py +++ b/opensfm/commands/undistort.py @@ -50,7 +50,7 @@ def undistort_images(self, graph, reconstruction, data): urec.add_shot(shot) elif shot.camera.projection_type in ['equirectangular', 'spherical']: original = data.image_as_array(shot.id) - width = int(data.config['depthmap_resolution']) + width = 4 * int(data.config['depthmap_resolution']) height = width / 2 image = cv2.resize(original, (width, height), interpolation=cv2.INTER_AREA) shots = perspective_views_of_a_panorama(shot, width) From 833f74c9663117a957ccc40ef16bcce7fffde111 Mon Sep 17 00:00:00 2001 From: Brook Roberts Date: Thu, 8 Jun 2017 15:15:33 +0100 Subject: [PATCH 75/75] Convert shot.id to string as required by bundle adjuster --- opensfm/reconstruction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensfm/reconstruction.py b/opensfm/reconstruction.py index dc822dd17..de4a8cce6 100644 --- a/opensfm/reconstruction.py +++ b/opensfm/reconstruction.py @@ -158,7 +158,7 @@ def bundle_single_view(graph, reconstruction, shot_id, config): if config['bundle_use_gps']: g = shot.metadata.gps_position - ba.add_position_prior(shot.id, g[0], g[1], g[2], + ba.add_position_prior(str(shot.id), g[0], g[1], g[2], shot.metadata.gps_dop) ba.set_loss_function(config.get('loss_function', 'SoftLOneLoss'), @@ -229,7 +229,7 @@ def bundle_local(graph, reconstruction, gcp, central_shot_id, config): for shot_id in interior: shot = reconstruction.shots[shot_id] g = shot.metadata.gps_position - ba.add_position_prior(shot.id, g[0], g[1], g[2], + ba.add_position_prior(str(shot.id), g[0], g[1], g[2], shot.metadata.gps_dop) if config['bundle_use_gcp'] and gcp: