1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. 14 // Copyright (C) 2009, Willow Garage Inc., all rights reserved. 15 // Third party copyrights are property of their respective owners. 16 // 17 // Redistribution and use in source and binary forms, with or without modification, 18 // are permitted provided that the following conditions are met: 19 // 20 // * Redistribution's of source code must retain the above copyright notice, 21 // this list of conditions and the following disclaimer. 22 // 23 // * Redistribution's in binary form must reproduce the above copyright notice, 24 // this list of conditions and the following disclaimer in the documentation 25 // and/or other materials provided with the distribution. 26 // 27 // * The name of the copyright holders may not be used to endorse or promote products 28 // derived from this software without specific prior written permission. 29 // 30 // This software is provided by the copyright holders and contributors "as is" and 31 // any express or implied warranties, including, but not limited to, the implied 32 // warranties of merchantability and fitness for a particular purpose are disclaimed. 33 // In no event shall the Intel Corporation or contributors be liable for any direct, 34 // indirect, incidental, special, exemplary, or consequential damages 35 // (including, but not limited to, procurement of substitute goods or services; 36 // loss of use, data, or profits; or business interruption) however caused 37 // and on any theory of liability, whether in contract, strict liability, 38 // or tort (including negligence or otherwise) arising in any way out of 39 // the use of this software, even if advised of the possibility of such damage. 40 // 41 //M*/ 42 43 #ifndef OPENCV_STITCHING_MOTION_ESTIMATORS_HPP 44 #define OPENCV_STITCHING_MOTION_ESTIMATORS_HPP 45 46 #include "opencv2/core.hpp" 47 #include "matchers.hpp" 48 #include "util.hpp" 49 #include "camera.hpp" 50 51 namespace cv { 52 namespace detail { 53 54 //! @addtogroup stitching_rotation 55 //! @{ 56 57 /** @brief Rotation estimator base class. 58 59 It takes features of all images, pairwise matches between all images and estimates rotations of all 60 cameras. 61 62 @note The coordinate system origin is implementation-dependent, but you can always normalize the 63 rotations in respect to the first camera, for instance. : 64 */ 65 class CV_EXPORTS Estimator 66 { 67 public: ~Estimator()68 virtual ~Estimator() {} 69 70 /** @brief Estimates camera parameters. 71 72 @param features Features of images 73 @param pairwise_matches Pairwise matches of images 74 @param cameras Estimated camera parameters 75 @return True in case of success, false otherwise 76 */ operator ()(const std::vector<ImageFeatures> & features,const std::vector<MatchesInfo> & pairwise_matches,std::vector<CameraParams> & cameras)77 bool operator ()(const std::vector<ImageFeatures> &features, 78 const std::vector<MatchesInfo> &pairwise_matches, 79 std::vector<CameraParams> &cameras) 80 { return estimate(features, pairwise_matches, cameras); } 81 82 protected: 83 /** @brief This method must implement camera parameters estimation logic in order to make the wrapper 84 detail::Estimator::operator()_ work. 85 86 @param features Features of images 87 @param pairwise_matches Pairwise matches of images 88 @param cameras Estimated camera parameters 89 @return True in case of success, false otherwise 90 */ 91 virtual bool estimate(const std::vector<ImageFeatures> &features, 92 const std::vector<MatchesInfo> &pairwise_matches, 93 std::vector<CameraParams> &cameras) = 0; 94 }; 95 96 /** @brief Homography based rotation estimator. 97 */ 98 class CV_EXPORTS HomographyBasedEstimator : public Estimator 99 { 100 public: HomographyBasedEstimator(bool is_focals_estimated=false)101 HomographyBasedEstimator(bool is_focals_estimated = false) 102 : is_focals_estimated_(is_focals_estimated) {} 103 104 private: 105 virtual bool estimate(const std::vector<ImageFeatures> &features, 106 const std::vector<MatchesInfo> &pairwise_matches, 107 std::vector<CameraParams> &cameras) CV_OVERRIDE; 108 109 bool is_focals_estimated_; 110 }; 111 112 /** @brief Affine transformation based estimator. 113 114 This estimator uses pairwise transformations estimated by matcher to estimate 115 final transformation for each camera. 116 117 @sa cv::detail::HomographyBasedEstimator 118 */ 119 class CV_EXPORTS AffineBasedEstimator : public Estimator 120 { 121 private: 122 virtual bool estimate(const std::vector<ImageFeatures> &features, 123 const std::vector<MatchesInfo> &pairwise_matches, 124 std::vector<CameraParams> &cameras) CV_OVERRIDE; 125 }; 126 127 /** @brief Base class for all camera parameters refinement methods. 128 */ 129 class CV_EXPORTS BundleAdjusterBase : public Estimator 130 { 131 public: refinementMask() const132 const Mat refinementMask() const { return refinement_mask_.clone(); } setRefinementMask(const Mat & mask)133 void setRefinementMask(const Mat &mask) 134 { 135 CV_Assert(mask.type() == CV_8U && mask.size() == Size(3, 3)); 136 refinement_mask_ = mask.clone(); 137 } 138 confThresh() const139 double confThresh() const { return conf_thresh_; } setConfThresh(double conf_thresh)140 void setConfThresh(double conf_thresh) { conf_thresh_ = conf_thresh; } 141 termCriteria()142 TermCriteria termCriteria() { return term_criteria_; } setTermCriteria(const TermCriteria & term_criteria)143 void setTermCriteria(const TermCriteria& term_criteria) { term_criteria_ = term_criteria; } 144 145 protected: 146 /** @brief Construct a bundle adjuster base instance. 147 148 @param num_params_per_cam Number of parameters per camera 149 @param num_errs_per_measurement Number of error terms (components) per match 150 */ BundleAdjusterBase(int num_params_per_cam,int num_errs_per_measurement)151 BundleAdjusterBase(int num_params_per_cam, int num_errs_per_measurement) 152 : num_images_(0), total_num_matches_(0), 153 num_params_per_cam_(num_params_per_cam), 154 num_errs_per_measurement_(num_errs_per_measurement), 155 features_(0), pairwise_matches_(0), conf_thresh_(0) 156 { 157 setRefinementMask(Mat::ones(3, 3, CV_8U)); 158 setConfThresh(1.); 159 setTermCriteria(TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 1000, DBL_EPSILON)); 160 } 161 162 // Runs bundle adjustment 163 virtual bool estimate(const std::vector<ImageFeatures> &features, 164 const std::vector<MatchesInfo> &pairwise_matches, 165 std::vector<CameraParams> &cameras) CV_OVERRIDE; 166 167 /** @brief Sets initial camera parameter to refine. 168 169 @param cameras Camera parameters 170 */ 171 virtual void setUpInitialCameraParams(const std::vector<CameraParams> &cameras) = 0; 172 /** @brief Gets the refined camera parameters. 173 174 @param cameras Refined camera parameters 175 */ 176 virtual void obtainRefinedCameraParams(std::vector<CameraParams> &cameras) const = 0; 177 /** @brief Calculates error vector. 178 179 @param err Error column-vector of length total_num_matches \* num_errs_per_measurement 180 */ 181 virtual void calcError(Mat &err) = 0; 182 /** @brief Calculates the cost function jacobian. 183 184 @param jac Jacobian matrix of dimensions 185 (total_num_matches \* num_errs_per_measurement) x (num_images \* num_params_per_cam) 186 */ 187 virtual void calcJacobian(Mat &jac) = 0; 188 189 // 3x3 8U mask, where 0 means don't refine respective parameter, != 0 means refine 190 Mat refinement_mask_; 191 192 int num_images_; 193 int total_num_matches_; 194 195 int num_params_per_cam_; 196 int num_errs_per_measurement_; 197 198 const ImageFeatures *features_; 199 const MatchesInfo *pairwise_matches_; 200 201 // Threshold to filter out poorly matched image pairs 202 double conf_thresh_; 203 204 //Levenberg-Marquardt algorithm termination criteria 205 TermCriteria term_criteria_; 206 207 // Camera parameters matrix (CV_64F) 208 Mat cam_params_; 209 210 // Connected images pairs 211 std::vector<std::pair<int,int> > edges_; 212 }; 213 214 215 /** @brief Stub bundle adjuster that does nothing. 216 */ 217 class CV_EXPORTS NoBundleAdjuster : public BundleAdjusterBase 218 { 219 public: NoBundleAdjuster()220 NoBundleAdjuster() : BundleAdjusterBase(0, 0) {} 221 222 private: estimate(const std::vector<ImageFeatures> &,const std::vector<MatchesInfo> &,std::vector<CameraParams> &)223 bool estimate(const std::vector<ImageFeatures> &, const std::vector<MatchesInfo> &, 224 std::vector<CameraParams> &) CV_OVERRIDE 225 { 226 return true; 227 } setUpInitialCameraParams(const std::vector<CameraParams> &)228 void setUpInitialCameraParams(const std::vector<CameraParams> &) CV_OVERRIDE {} obtainRefinedCameraParams(std::vector<CameraParams> &) const229 void obtainRefinedCameraParams(std::vector<CameraParams> &) const CV_OVERRIDE {} calcError(Mat &)230 void calcError(Mat &) CV_OVERRIDE {} calcJacobian(Mat &)231 void calcJacobian(Mat &) CV_OVERRIDE {} 232 }; 233 234 235 /** @brief Implementation of the camera parameters refinement algorithm which minimizes sum of the reprojection 236 error squares 237 238 It can estimate focal length, aspect ratio, principal point. 239 You can affect only on them via the refinement mask. 240 */ 241 class CV_EXPORTS BundleAdjusterReproj : public BundleAdjusterBase 242 { 243 public: BundleAdjusterReproj()244 BundleAdjusterReproj() : BundleAdjusterBase(7, 2) {} 245 246 private: 247 void setUpInitialCameraParams(const std::vector<CameraParams> &cameras) CV_OVERRIDE; 248 void obtainRefinedCameraParams(std::vector<CameraParams> &cameras) const CV_OVERRIDE; 249 void calcError(Mat &err) CV_OVERRIDE; 250 void calcJacobian(Mat &jac) CV_OVERRIDE; 251 252 Mat err1_, err2_; 253 }; 254 255 256 /** @brief Implementation of the camera parameters refinement algorithm which minimizes sum of the distances 257 between the rays passing through the camera center and a feature. : 258 259 It can estimate focal length. It ignores the refinement mask for now. 260 */ 261 class CV_EXPORTS BundleAdjusterRay : public BundleAdjusterBase 262 { 263 public: BundleAdjusterRay()264 BundleAdjusterRay() : BundleAdjusterBase(4, 3) {} 265 266 private: 267 void setUpInitialCameraParams(const std::vector<CameraParams> &cameras) CV_OVERRIDE; 268 void obtainRefinedCameraParams(std::vector<CameraParams> &cameras) const CV_OVERRIDE; 269 void calcError(Mat &err) CV_OVERRIDE; 270 void calcJacobian(Mat &jac) CV_OVERRIDE; 271 272 Mat err1_, err2_; 273 }; 274 275 276 /** @brief Bundle adjuster that expects affine transformation 277 represented in homogeneous coordinates in R for each camera param. Implements 278 camera parameters refinement algorithm which minimizes sum of the reprojection 279 error squares 280 281 It estimates all transformation parameters. Refinement mask is ignored. 282 283 @sa AffineBasedEstimator AffineBestOf2NearestMatcher BundleAdjusterAffinePartial 284 */ 285 class CV_EXPORTS BundleAdjusterAffine : public BundleAdjusterBase 286 { 287 public: BundleAdjusterAffine()288 BundleAdjusterAffine() : BundleAdjusterBase(6, 2) {} 289 290 private: 291 void setUpInitialCameraParams(const std::vector<CameraParams> &cameras) CV_OVERRIDE; 292 void obtainRefinedCameraParams(std::vector<CameraParams> &cameras) const CV_OVERRIDE; 293 void calcError(Mat &err) CV_OVERRIDE; 294 void calcJacobian(Mat &jac) CV_OVERRIDE; 295 296 Mat err1_, err2_; 297 }; 298 299 300 /** @brief Bundle adjuster that expects affine transformation with 4 DOF 301 represented in homogeneous coordinates in R for each camera param. Implements 302 camera parameters refinement algorithm which minimizes sum of the reprojection 303 error squares 304 305 It estimates all transformation parameters. Refinement mask is ignored. 306 307 @sa AffineBasedEstimator AffineBestOf2NearestMatcher BundleAdjusterAffine 308 */ 309 class CV_EXPORTS BundleAdjusterAffinePartial : public BundleAdjusterBase 310 { 311 public: BundleAdjusterAffinePartial()312 BundleAdjusterAffinePartial() : BundleAdjusterBase(4, 2) {} 313 314 private: 315 void setUpInitialCameraParams(const std::vector<CameraParams> &cameras) CV_OVERRIDE; 316 void obtainRefinedCameraParams(std::vector<CameraParams> &cameras) const CV_OVERRIDE; 317 void calcError(Mat &err) CV_OVERRIDE; 318 void calcJacobian(Mat &jac) CV_OVERRIDE; 319 320 Mat err1_, err2_; 321 }; 322 323 324 enum WaveCorrectKind 325 { 326 WAVE_CORRECT_HORIZ, 327 WAVE_CORRECT_VERT 328 }; 329 330 /** @brief Tries to make panorama more horizontal (or vertical). 331 332 @param rmats Camera rotation matrices. 333 @param kind Correction kind, see detail::WaveCorrectKind. 334 */ 335 void CV_EXPORTS waveCorrect(std::vector<Mat> &rmats, WaveCorrectKind kind); 336 337 338 ////////////////////////////////////////////////////////////////////////////// 339 // Auxiliary functions 340 341 // Returns matches graph representation in DOT language 342 String CV_EXPORTS matchesGraphAsString(std::vector<String> &pathes, std::vector<MatchesInfo> &pairwise_matches, 343 float conf_threshold); 344 345 std::vector<int> CV_EXPORTS leaveBiggestComponent( 346 std::vector<ImageFeatures> &features, 347 std::vector<MatchesInfo> &pairwise_matches, 348 float conf_threshold); 349 350 void CV_EXPORTS findMaxSpanningTree( 351 int num_images, const std::vector<MatchesInfo> &pairwise_matches, 352 Graph &span_tree, std::vector<int> ¢ers); 353 354 //! @} stitching_rotation 355 356 } // namespace detail 357 } // namespace cv 358 359 #endif // OPENCV_STITCHING_MOTION_ESTIMATORS_HPP 360