1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved. 14 // Third party copyrights are property of their respective owners. 15 // 16 // Redistribution and use in source and binary forms, with or without modification, 17 // are permitted provided that the following conditions are met: 18 // 19 // * Redistribution's of source code must retain the above copyright notice, 20 // this list of conditions and the following disclaimer. 21 // 22 // * Redistribution's in binary form must reproduce the above copyright notice, 23 // this list of conditions and the following disclaimer in the documentation 24 // and/or other materials provided with the distribution. 25 // 26 // * The name of the copyright holders may not be used to endorse or promote products 27 // derived from this software without specific prior written permission. 28 // 29 // This software is provided by the copyright holders and contributors "as is" and 30 // any express or implied warranties, including, but not limited to, the implied 31 // warranties of merchantability and fitness for a particular purpose are disclaimed. 32 // In no event shall the Intel Corporation or contributors be liable for any direct, 33 // indirect, incidental, special, exemplary, or consequential damages 34 // (including, but not limited to, procurement of substitute goods or services; 35 // loss of use, data, or profits; or business interruption) however caused 36 // and on any theory of liability, whether in contract, strict liability, 37 // or tort (including negligence or otherwise) arising in any way out of 38 // the use of this software, even if advised of the possibility of such damage. 39 // 40 //M*/ 41 42 #ifndef OPENCV_DNN_DNN_HPP 43 #define OPENCV_DNN_DNN_HPP 44 45 #include <vector> 46 #include <opencv2/core.hpp> 47 48 #if !defined CV_DOXYGEN && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS 49 #define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v11 { 50 #define CV__DNN_EXPERIMENTAL_NS_END } 51 namespace cv { namespace dnn { namespace experimental_dnn_34_v11 { } using namespace experimental_dnn_34_v11; }} 52 #else 53 #define CV__DNN_EXPERIMENTAL_NS_BEGIN 54 #define CV__DNN_EXPERIMENTAL_NS_END 55 #endif 56 57 #include <opencv2/dnn/dict.hpp> 58 59 namespace cv { 60 namespace dnn { 61 CV__DNN_EXPERIMENTAL_NS_BEGIN 62 //! @addtogroup dnn 63 //! @{ 64 65 typedef std::vector<int> MatShape; 66 67 /** 68 * @brief Enum of computation backends supported by layers. 69 * @see Net::setPreferableBackend 70 */ 71 enum Backend 72 { 73 //! DNN_BACKEND_DEFAULT equals to DNN_BACKEND_INFERENCE_ENGINE if 74 //! OpenCV is built with Intel's Inference Engine library or 75 //! DNN_BACKEND_OPENCV otherwise. 76 DNN_BACKEND_DEFAULT, 77 DNN_BACKEND_HALIDE, 78 DNN_BACKEND_INFERENCE_ENGINE, 79 DNN_BACKEND_OPENCV 80 }; 81 82 /** 83 * @brief Enum of target devices for computations. 84 * @see Net::setPreferableTarget 85 */ 86 enum Target 87 { 88 DNN_TARGET_CPU, 89 DNN_TARGET_OPENCL, 90 DNN_TARGET_OPENCL_FP16, 91 DNN_TARGET_MYRIAD, 92 //! FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin. 93 DNN_TARGET_FPGA 94 }; 95 96 CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends(); 97 CV_EXPORTS std::vector<Target> getAvailableTargets(Backend be); 98 99 /** @brief This class provides all data needed to initialize layer. 100 * 101 * It includes dictionary with scalar params (which can be read by using Dict interface), 102 * blob params #blobs and optional meta information: #name and #type of layer instance. 103 */ 104 class CV_EXPORTS LayerParams : public Dict 105 { 106 public: 107 //TODO: Add ability to name blob params 108 std::vector<Mat> blobs; //!< List of learned parameters stored as blobs. 109 110 String name; //!< Name of the layer instance (optional, can be used internal purposes). 111 String type; //!< Type name which was used for creating layer by layer factory (optional). 112 }; 113 114 /** 115 * @brief Derivatives of this class encapsulates functions of certain backends. 116 */ 117 class BackendNode 118 { 119 public: 120 BackendNode(int backendId); 121 122 virtual ~BackendNode(); //!< Virtual destructor to make polymorphism. 123 124 int backendId; //!< Backend identifier. 125 }; 126 127 /** 128 * @brief Derivatives of this class wraps cv::Mat for different backends and targets. 129 */ 130 class BackendWrapper 131 { 132 public: 133 BackendWrapper(int backendId, int targetId); 134 135 /** 136 * @brief Wrap cv::Mat for specific backend and target. 137 * @param[in] targetId Target identifier. 138 * @param[in] m cv::Mat for wrapping. 139 * 140 * Make CPU->GPU data transfer if it's require for the target. 141 */ 142 BackendWrapper(int targetId, const cv::Mat& m); 143 144 /** 145 * @brief Make wrapper for reused cv::Mat. 146 * @param[in] base Wrapper of cv::Mat that will be reused. 147 * @param[in] shape Specific shape. 148 * 149 * Initialize wrapper from another one. It'll wrap the same host CPU 150 * memory and mustn't allocate memory on device(i.e. GPU). It might 151 * has different shape. Use in case of CPU memory reusing for reuse 152 * associated memory on device too. 153 */ 154 BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape); 155 156 virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism. 157 158 /** 159 * @brief Transfer data to CPU host memory. 160 */ 161 virtual void copyToHost() = 0; 162 163 /** 164 * @brief Indicate that an actual data is on CPU. 165 */ 166 virtual void setHostDirty() = 0; 167 168 int backendId; //!< Backend identifier. 169 int targetId; //!< Target identifier. 170 }; 171 172 class CV_EXPORTS ActivationLayer; 173 174 /** @brief This interface class allows to build new Layers - are building blocks of networks. 175 * 176 * Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs. 177 * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros. 178 */ 179 class CV_EXPORTS_W Layer : public Algorithm 180 { 181 public: 182 183 //! List of learned parameters must be stored here to allow read them by using Net::getParam(). 184 CV_PROP_RW std::vector<Mat> blobs; 185 186 /** @brief Computes and sets internal parameters according to inputs, outputs and blobs. 187 * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead 188 * @param[in] input vector of already allocated input blobs 189 * @param[out] output vector of already allocated output blobs 190 * 191 * If this method is called after network has allocated all memory for input and output blobs 192 * and before inferencing. 193 */ 194 CV_DEPRECATED_EXTERNAL 195 virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output); 196 197 /** @brief Computes and sets internal parameters according to inputs, outputs and blobs. 198 * @param[in] inputs vector of already allocated input blobs 199 * @param[out] outputs vector of already allocated output blobs 200 * 201 * If this method is called after network has allocated all memory for input and output blobs 202 * and before inferencing. 203 */ 204 CV_WRAP virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs); 205 206 /** @brief Given the @p input blobs, computes the output @p blobs. 207 * @deprecated Use Layer::forward(InputArrayOfArrays, OutputArrayOfArrays, OutputArrayOfArrays) instead 208 * @param[in] input the input blobs. 209 * @param[out] output allocated output blobs, which will store results of the computation. 210 * @param[out] internals allocated internal blobs 211 */ 212 CV_DEPRECATED_EXTERNAL 213 virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals); 214 215 /** @brief Given the @p input blobs, computes the output @p blobs. 216 * @param[in] inputs the input blobs. 217 * @param[out] outputs allocated output blobs, which will store results of the computation. 218 * @param[out] internals allocated internal blobs 219 */ 220 virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals); 221 222 /** @brief Given the @p input blobs, computes the output @p blobs. 223 * @param[in] inputs the input blobs. 224 * @param[out] outputs allocated output blobs, which will store results of the computation. 225 * @param[out] internals allocated internal blobs 226 */ 227 void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals); 228 229 /** @brief 230 * @overload 231 * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead 232 */ 233 CV_DEPRECATED_EXTERNAL 234 void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs); 235 236 /** @brief 237 * @overload 238 * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead 239 */ 240 CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs); 241 242 /** @brief Allocates layer and computes output. 243 * @deprecated This method will be removed in the future release. 244 */ 245 CV_DEPRECATED CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs, 246 CV_IN_OUT std::vector<Mat> &internals); 247 248 /** @brief Returns index of input blob into the input array. 249 * @param inputName label of input blob 250 * 251 * Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation. 252 * This method maps label of input blob to its index into input vector. 253 */ 254 virtual int inputNameToIndex(String inputName); 255 /** @brief Returns index of output blob in output array. 256 * @see inputNameToIndex() 257 */ 258 CV_WRAP virtual int outputNameToIndex(const String& outputName); 259 260 /** 261 * @brief Ask layer if it support specific backend for doing computations. 262 * @param[in] backendId computation backend identifier. 263 * @see Backend 264 */ 265 virtual bool supportBackend(int backendId); 266 267 /** 268 * @brief Returns Halide backend node. 269 * @param[in] inputs Input Halide buffers. 270 * @see BackendNode, BackendWrapper 271 * 272 * Input buffers should be exactly the same that will be used in forward invocations. 273 * Despite we can use Halide::ImageParam based on input shape only, 274 * it helps prevent some memory management issues (if something wrong, 275 * Halide tests will be failed). 276 */ 277 virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs); 278 279 virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs); 280 281 /** 282 * @brief Automatic Halide scheduling based on layer hyper-parameters. 283 * @param[in] node Backend node with Halide functions. 284 * @param[in] inputs Blobs that will be used in forward invocations. 285 * @param[in] outputs Blobs that will be used in forward invocations. 286 * @param[in] targetId Target identifier 287 * @see BackendNode, Target 288 * 289 * Layer don't use own Halide::Func members because we can have applied 290 * layers fusing. In this way the fused function should be scheduled. 291 */ 292 virtual void applyHalideScheduler(Ptr<BackendNode>& node, 293 const std::vector<Mat*> &inputs, 294 const std::vector<Mat> &outputs, 295 int targetId) const; 296 297 /** 298 * @brief Implement layers fusing. 299 * @param[in] node Backend node of bottom layer. 300 * @see BackendNode 301 * 302 * Actual for graph-based backends. If layer attached successfully, 303 * returns non-empty cv::Ptr to node of the same backend. 304 * Fuse only over the last function. 305 */ 306 virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node); 307 308 /** 309 * @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case. 310 * @param[in] layer The subsequent activation layer. 311 * 312 * Returns true if the activation layer has been attached successfully. 313 */ 314 virtual bool setActivation(const Ptr<ActivationLayer>& layer); 315 316 /** 317 * @brief Try to fuse current layer with a next one 318 * @param[in] top Next layer to be fused. 319 * @returns True if fusion was performed. 320 */ 321 virtual bool tryFuse(Ptr<Layer>& top); 322 323 /** 324 * @brief Returns parameters of layers with channel-wise multiplication and addition. 325 * @param[out] scale Channel-wise multipliers. Total number of values should 326 * be equal to number of channels. 327 * @param[out] shift Channel-wise offsets. Total number of values should 328 * be equal to number of channels. 329 * 330 * Some layers can fuse their transformations with further layers. 331 * In example, convolution + batch normalization. This way base layer 332 * use weights from layer after it. Fused layer is skipped. 333 * By default, @p scale and @p shift are empty that means layer has no 334 * element-wise multiplications or additions. 335 */ 336 virtual void getScaleShift(Mat& scale, Mat& shift) const; 337 338 /** 339 * @brief "Deattaches" all the layers, attached to particular layer. 340 */ 341 virtual void unsetAttached(); 342 343 virtual bool getMemoryShapes(const std::vector<MatShape> &inputs, 344 const int requiredOutputs, 345 std::vector<MatShape> &outputs, 346 std::vector<MatShape> &internals) const; getFLOPS(const std::vector<MatShape> & inputs,const std::vector<MatShape> & outputs) const347 virtual int64 getFLOPS(const std::vector<MatShape> &inputs, 348 const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;} 349 350 CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes. 351 CV_PROP String type; //!< Type name which was used for creating layer by layer factory. 352 CV_PROP int preferableTarget; //!< prefer target for layer forwarding 353 354 Layer(); 355 explicit Layer(const LayerParams ¶ms); //!< Initializes only #name, #type and #blobs fields. 356 void setParamsFrom(const LayerParams ¶ms); //!< Initializes only #name, #type and #blobs fields. 357 virtual ~Layer(); 358 }; 359 360 /** @brief This class allows to create and manipulate comprehensive artificial neural networks. 361 * 362 * Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances, 363 * and edges specify relationships between layers inputs and outputs. 364 * 365 * Each network layer has unique integer id and unique string name inside its network. 366 * LayerId can store either layer name or layer id. 367 * 368 * This class supports reference counting of its instances, i. e. copies point to the same instance. 369 */ 370 class CV_EXPORTS_W_SIMPLE Net 371 { 372 public: 373 374 CV_WRAP Net(); //!< Default constructor. 375 CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore. 376 377 /** @brief Create a network from Intel's Model Optimizer intermediate representation. 378 * @param[in] xml XML configuration file with network's topology. 379 * @param[in] bin Binary file with trained weights. 380 * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine 381 * backend. 382 */ 383 CV_WRAP static Net readFromModelOptimizer(const String& xml, const String& bin); 384 385 /** Returns true if there are no layers in the network. */ 386 CV_WRAP bool empty() const; 387 388 /** @brief Adds new layer to the net. 389 * @param name unique name of the adding layer. 390 * @param type typename of the adding layer (type must be registered in LayerRegister). 391 * @param params parameters which will be used to initialize the creating layer. 392 * @returns unique identifier of created layer, or -1 if a failure will happen. 393 */ 394 int addLayer(const String &name, const String &type, LayerParams ¶ms); 395 /** @brief Adds new layer and connects its first input to the first output of previously added layer. 396 * @see addLayer() 397 */ 398 int addLayerToPrev(const String &name, const String &type, LayerParams ¶ms); 399 400 /** @brief Converts string name of the layer to the integer identifier. 401 * @returns id of the layer, or -1 if the layer wasn't found. 402 */ 403 CV_WRAP int getLayerId(const String &layer); 404 405 CV_WRAP std::vector<String> getLayerNames() const; 406 407 /** @brief Container for strings and integers. */ 408 typedef DictValue LayerId; 409 410 /** @brief Returns pointer to layer with specified id or name which the network use. */ 411 CV_WRAP Ptr<Layer> getLayer(LayerId layerId); 412 413 /** @brief Returns pointers to input layers of specific layer. */ 414 std::vector<Ptr<Layer> > getLayerInputs(LayerId layerId); // FIXIT: CV_WRAP 415 416 /** @brief Connects output of the first layer to input of the second layer. 417 * @param outPin descriptor of the first layer output. 418 * @param inpPin descriptor of the second layer input. 419 * 420 * Descriptors have the following template <DFN><layer_name>[.input_number]</DFN>: 421 * - the first part of the template <DFN>layer_name</DFN> is sting name of the added layer. 422 * If this part is empty then the network input pseudo layer will be used; 423 * - the second optional part of the template <DFN>input_number</DFN> 424 * is either number of the layer input, either label one. 425 * If this part is omitted then the first layer input will be used. 426 * 427 * @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex() 428 */ 429 CV_WRAP void connect(String outPin, String inpPin); 430 431 /** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer. 432 * @param outLayerId identifier of the first layer 433 * @param outNum number of the first layer output 434 * @param inpLayerId identifier of the second layer 435 * @param inpNum number of the second layer input 436 */ 437 void connect(int outLayerId, int outNum, int inpLayerId, int inpNum); 438 439 /** @brief Sets outputs names of the network input pseudo layer. 440 * 441 * Each net always has special own the network input pseudo layer with id=0. 442 * This layer stores the user blobs only and don't make any computations. 443 * In fact, this layer provides the only way to pass user data into the network. 444 * As any other layer, this layer can label its outputs and this function provides an easy way to do this. 445 */ 446 CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames); 447 448 /** @brief Runs forward pass to compute output of layer with name @p outputName. 449 * @param outputName name for layer which output is needed to get 450 * @return blob for first output of specified layer. 451 * @details By default runs forward pass for the whole network. 452 */ 453 CV_WRAP Mat forward(const String& outputName = String()); 454 455 /** @brief Runs forward pass to compute output of layer with name @p outputName. 456 * @param outputBlobs contains all output blobs for specified layer. 457 * @param outputName name for layer which output is needed to get 458 * @details If @p outputName is empty, runs forward pass for the whole network. 459 */ 460 CV_WRAP void forward(OutputArrayOfArrays outputBlobs, const String& outputName = String()); 461 462 /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames. 463 * @param outputBlobs contains blobs for first outputs of specified layers. 464 * @param outBlobNames names for layers which outputs are needed to get 465 */ 466 CV_WRAP void forward(OutputArrayOfArrays outputBlobs, 467 const std::vector<String>& outBlobNames); 468 469 /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames. 470 * @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames. 471 * @param outBlobNames names for layers which outputs are needed to get 472 */ 473 CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector<std::vector<Mat> >& outputBlobs, 474 const std::vector<String>& outBlobNames); 475 476 /** 477 * @brief Compile Halide layers. 478 * @param[in] scheduler Path to YAML file with scheduling directives. 479 * @see setPreferableBackend 480 * 481 * Schedule layers that support Halide backend. Then compile them for 482 * specific target. For layers that not represented in scheduling file 483 * or if no manual scheduling used at all, automatic scheduling will be applied. 484 */ 485 CV_WRAP void setHalideScheduler(const String& scheduler); 486 487 /** 488 * @brief Ask network to use specific computation backend where it supported. 489 * @param[in] backendId backend identifier. 490 * @see Backend 491 * 492 * If OpenCV is compiled with Intel's Inference Engine library, DNN_BACKEND_DEFAULT 493 * means DNN_BACKEND_INFERENCE_ENGINE. Otherwise it equals to DNN_BACKEND_OPENCV. 494 */ 495 CV_WRAP void setPreferableBackend(int backendId); 496 497 /** 498 * @brief Ask network to make computations on specific target device. 499 * @param[in] targetId target identifier. 500 * @see Target 501 * 502 * List of supported combinations backend / target: 503 * | | DNN_BACKEND_OPENCV | DNN_BACKEND_INFERENCE_ENGINE | DNN_BACKEND_HALIDE | 504 * |------------------------|--------------------|------------------------------|--------------------| 505 * | DNN_TARGET_CPU | + | + | + | 506 * | DNN_TARGET_OPENCL | + | + | + | 507 * | DNN_TARGET_OPENCL_FP16 | + | + | | 508 * | DNN_TARGET_MYRIAD | | + | | 509 * | DNN_TARGET_FPGA | | + | | 510 */ 511 CV_WRAP void setPreferableTarget(int targetId); 512 513 /** @brief Sets the new input value for the network 514 * @param blob A new blob. Should have CV_32F or CV_8U depth. 515 * @param name A name of input layer. 516 * @param scalefactor An optional normalization scale. 517 * @param mean An optional mean subtraction values. 518 * @see connect(String, String) to know format of the descriptor. 519 * 520 * If scale or mean values are specified, a final input blob is computed 521 * as: 522 * \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f] 523 */ 524 CV_WRAP void setInput(InputArray blob, const String& name = "", 525 double scalefactor = 1.0, const Scalar& mean = Scalar()); 526 527 /** @brief Sets the new value for the learned param of the layer. 528 * @param layer name or id of the layer. 529 * @param numParam index of the layer parameter in the Layer::blobs array. 530 * @param blob the new value. 531 * @see Layer::blobs 532 * @note If shape of the new blob differs from the previous shape, 533 * then the following forward pass may fail. 534 */ 535 CV_WRAP void setParam(LayerId layer, int numParam, const Mat &blob); 536 537 /** @brief Returns parameter blob of the layer. 538 * @param layer name or id of the layer. 539 * @param numParam index of the layer parameter in the Layer::blobs array. 540 * @see Layer::blobs 541 */ 542 CV_WRAP Mat getParam(LayerId layer, int numParam = 0); 543 544 /** @brief Returns indexes of layers with unconnected outputs. 545 */ 546 CV_WRAP std::vector<int> getUnconnectedOutLayers() const; 547 548 /** @brief Returns names of layers with unconnected outputs. 549 */ 550 CV_WRAP std::vector<String> getUnconnectedOutLayersNames() const; 551 552 /** @brief Returns input and output shapes for all layers in loaded model; 553 * preliminary inferencing isn't necessary. 554 * @param netInputShapes shapes for all input blobs in net input layer. 555 * @param layersIds output parameter for layer IDs. 556 * @param inLayersShapes output parameter for input layers shapes; 557 * order is the same as in layersIds 558 * @param outLayersShapes output parameter for output layers shapes; 559 * order is the same as in layersIds 560 */ 561 CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes, 562 CV_OUT std::vector<int>& layersIds, 563 CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes, 564 CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const; 565 566 /** @overload */ 567 CV_WRAP void getLayersShapes(const MatShape& netInputShape, 568 CV_OUT std::vector<int>& layersIds, 569 CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes, 570 CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const; 571 572 /** @brief Returns input and output shapes for layer with specified 573 * id in loaded model; preliminary inferencing isn't necessary. 574 * @param netInputShape shape input blob in net input layer. 575 * @param layerId id for layer. 576 * @param inLayerShapes output parameter for input layers shapes; 577 * order is the same as in layersIds 578 * @param outLayerShapes output parameter for output layers shapes; 579 * order is the same as in layersIds 580 */ 581 void getLayerShapes(const MatShape& netInputShape, 582 const int layerId, 583 CV_OUT std::vector<MatShape>& inLayerShapes, 584 CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP 585 586 /** @overload */ 587 void getLayerShapes(const std::vector<MatShape>& netInputShapes, 588 const int layerId, 589 CV_OUT std::vector<MatShape>& inLayerShapes, 590 CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP 591 592 /** @brief Computes FLOP for whole loaded model with specified input shapes. 593 * @param netInputShapes vector of shapes for all net inputs. 594 * @returns computed FLOP. 595 */ 596 CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const; 597 /** @overload */ 598 CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const; 599 /** @overload */ 600 CV_WRAP int64 getFLOPS(const int layerId, 601 const std::vector<MatShape>& netInputShapes) const; 602 /** @overload */ 603 CV_WRAP int64 getFLOPS(const int layerId, 604 const MatShape& netInputShape) const; 605 606 /** @brief Returns list of types for layer used in model. 607 * @param layersTypes output parameter for returning types. 608 */ 609 CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const; 610 611 /** @brief Returns count of layers of specified type. 612 * @param layerType type. 613 * @returns count of layers 614 */ 615 CV_WRAP int getLayersCount(const String& layerType) const; 616 617 /** @brief Computes bytes number which are required to store 618 * all weights and intermediate blobs for model. 619 * @param netInputShapes vector of shapes for all net inputs. 620 * @param weights output parameter to store resulting bytes for weights. 621 * @param blobs output parameter to store resulting bytes for intermediate blobs. 622 */ 623 void getMemoryConsumption(const std::vector<MatShape>& netInputShapes, 624 CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP 625 /** @overload */ 626 CV_WRAP void getMemoryConsumption(const MatShape& netInputShape, 627 CV_OUT size_t& weights, CV_OUT size_t& blobs) const; 628 /** @overload */ 629 CV_WRAP void getMemoryConsumption(const int layerId, 630 const std::vector<MatShape>& netInputShapes, 631 CV_OUT size_t& weights, CV_OUT size_t& blobs) const; 632 /** @overload */ 633 CV_WRAP void getMemoryConsumption(const int layerId, 634 const MatShape& netInputShape, 635 CV_OUT size_t& weights, CV_OUT size_t& blobs) const; 636 637 /** @brief Computes bytes number which are required to store 638 * all weights and intermediate blobs for each layer. 639 * @param netInputShapes vector of shapes for all net inputs. 640 * @param layerIds output vector to save layer IDs. 641 * @param weights output parameter to store resulting bytes for weights. 642 * @param blobs output parameter to store resulting bytes for intermediate blobs. 643 */ 644 void getMemoryConsumption(const std::vector<MatShape>& netInputShapes, 645 CV_OUT std::vector<int>& layerIds, 646 CV_OUT std::vector<size_t>& weights, 647 CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP 648 /** @overload */ 649 void getMemoryConsumption(const MatShape& netInputShape, 650 CV_OUT std::vector<int>& layerIds, 651 CV_OUT std::vector<size_t>& weights, 652 CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP 653 654 /** @brief Enables or disables layer fusion in the network. 655 * @param fusion true to enable the fusion, false to disable. The fusion is enabled by default. 656 */ 657 CV_WRAP void enableFusion(bool fusion); 658 659 /** @brief Returns overall time for inference and timings (in ticks) for layers. 660 * Indexes in returned vector correspond to layers ids. Some layers can be fused with others, 661 * in this case zero ticks count will be return for that skipped layers. 662 * @param timings vector for tick timings for all layers. 663 * @return overall ticks for model inference. 664 */ 665 CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings); 666 667 private: 668 struct Impl; 669 Ptr<Impl> impl; 670 }; 671 672 /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files. 673 * @param cfgFile path to the .cfg file with text description of the network architecture. 674 * @param darknetModel path to the .weights file with learned network. 675 * @returns Network object that ready to do forward, throw an exception in failure cases. 676 * @returns Net object. 677 */ 678 CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String()); 679 680 /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files. 681 * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture. 682 * @param bufferModel A buffer contains a content of .weights file with learned network. 683 * @returns Net object. 684 */ 685 CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg, 686 const std::vector<uchar>& bufferModel = std::vector<uchar>()); 687 688 /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files. 689 * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture. 690 * @param lenCfg Number of bytes to read from bufferCfg 691 * @param bufferModel A buffer contains a content of .weights file with learned network. 692 * @param lenModel Number of bytes to read from bufferModel 693 * @returns Net object. 694 */ 695 CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg, 696 const char *bufferModel = NULL, size_t lenModel = 0); 697 698 /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format. 699 * @param prototxt path to the .prototxt file with text description of the network architecture. 700 * @param caffeModel path to the .caffemodel file with learned network. 701 * @returns Net object. 702 */ 703 CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String()); 704 705 /** @brief Reads a network model stored in Caffe model in memory. 706 * @param bufferProto buffer containing the content of the .prototxt file 707 * @param bufferModel buffer containing the content of the .caffemodel file 708 * @returns Net object. 709 */ 710 CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto, 711 const std::vector<uchar>& bufferModel = std::vector<uchar>()); 712 713 /** @brief Reads a network model stored in Caffe model in memory. 714 * @details This is an overloaded member function, provided for convenience. 715 * It differs from the above function only in what argument(s) it accepts. 716 * @param bufferProto buffer containing the content of the .prototxt file 717 * @param lenProto length of bufferProto 718 * @param bufferModel buffer containing the content of the .caffemodel file 719 * @param lenModel length of bufferModel 720 * @returns Net object. 721 */ 722 CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto, 723 const char *bufferModel = NULL, size_t lenModel = 0); 724 725 /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format. 726 * @param model path to the .pb file with binary protobuf description of the network architecture 727 * @param config path to the .pbtxt file that contains text graph definition in protobuf format. 728 * Resulting Net object is built by text graph using weights from a binary one that 729 * let us make it more flexible. 730 * @returns Net object. 731 */ 732 CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String()); 733 734 /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format. 735 * @param bufferModel buffer containing the content of the pb file 736 * @param bufferConfig buffer containing the content of the pbtxt file 737 * @returns Net object. 738 */ 739 CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel, 740 const std::vector<uchar>& bufferConfig = std::vector<uchar>()); 741 742 /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format. 743 * @details This is an overloaded member function, provided for convenience. 744 * It differs from the above function only in what argument(s) it accepts. 745 * @param bufferModel buffer containing the content of the pb file 746 * @param lenModel length of bufferModel 747 * @param bufferConfig buffer containing the content of the pbtxt file 748 * @param lenConfig length of bufferConfig 749 */ 750 CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel, 751 const char *bufferConfig = NULL, size_t lenConfig = 0); 752 753 /** 754 * @brief Reads a network model stored in <a href="http://torch.ch">Torch7</a> framework's format. 755 * @param model path to the file, dumped from Torch by using torch.save() function. 756 * @param isBinary specifies whether the network was serialized in ascii mode or binary. 757 * @param evaluate specifies testing phase of network. If true, it's similar to evaluate() method in Torch. 758 * @returns Net object. 759 * 760 * @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language, 761 * which has various bit-length on different systems. 762 * 763 * The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object 764 * with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors. 765 * 766 * List of supported layers (i.e. object instances derived from Torch nn.Module class): 767 * - nn.Sequential 768 * - nn.Parallel 769 * - nn.Concat 770 * - nn.Linear 771 * - nn.SpatialConvolution 772 * - nn.SpatialMaxPooling, nn.SpatialAveragePooling 773 * - nn.ReLU, nn.TanH, nn.Sigmoid 774 * - nn.Reshape 775 * - nn.SoftMax, nn.LogSoftMax 776 * 777 * Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported. 778 */ 779 CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true, bool evaluate = true); 780 781 /** 782 * @brief Read deep learning network represented in one of the supported formats. 783 * @param[in] model Binary file contains trained weights. The following file 784 * extensions are expected for models from different frameworks: 785 * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/) 786 * * `*.pb` (TensorFlow, https://www.tensorflow.org/) 787 * * `*.t7` | `*.net` (Torch, http://torch.ch/) 788 * * `*.weights` (Darknet, https://pjreddie.com/darknet/) 789 * * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit) 790 * @param[in] config Text file contains network configuration. It could be a 791 * file with the following extensions: 792 * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/) 793 * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/) 794 * * `*.cfg` (Darknet, https://pjreddie.com/darknet/) 795 * * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit) 796 * @param[in] framework Explicit framework name tag to determine a format. 797 * @returns Net object. 798 * 799 * This function automatically detects an origin framework of trained model 800 * and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow, 801 * @ref readNetFromTorch or @ref readNetFromDarknet. An order of @p model and @p config 802 * arguments does not matter. 803 */ 804 CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = ""); 805 806 /** 807 * @brief Read deep learning network represented in one of the supported formats. 808 * @details This is an overloaded member function, provided for convenience. 809 * It differs from the above function only in what argument(s) it accepts. 810 * @param[in] framework Name of origin framework. 811 * @param[in] bufferModel A buffer with a content of binary file with weights 812 * @param[in] bufferConfig A buffer with a content of text file contains network configuration. 813 * @returns Net object. 814 */ 815 CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel, 816 const std::vector<uchar>& bufferConfig = std::vector<uchar>()); 817 818 /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework. 819 * @warning This function has the same limitations as readNetFromTorch(). 820 */ 821 CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true); 822 823 /** @brief Load a network from Intel's Model Optimizer intermediate representation. 824 * @param[in] xml XML configuration file with network's topology. 825 * @param[in] bin Binary file with trained weights. 826 * @returns Net object. 827 * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine 828 * backend. 829 */ 830 CV_EXPORTS_W Net readNetFromModelOptimizer(const String &xml, const String &bin); 831 832 /** @brief Reads a network model <a href="https://onnx.ai/">ONNX</a>. 833 * @param onnxFile path to the .onnx file with text description of the network architecture. 834 * @returns Network object that ready to do forward, throw an exception in failure cases. 835 */ 836 CV_EXPORTS_W Net readNetFromONNX(const String &onnxFile); 837 838 /** @brief Creates blob from .pb file. 839 * @param path to the .pb file with input tensor. 840 * @returns Mat. 841 */ 842 CV_EXPORTS_W Mat readTensorFromONNX(const String& path); 843 844 /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center, 845 * subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels. 846 * @param image input image (with 1-, 3- or 4-channels). 847 * @param size spatial size for output image 848 * @param mean scalar with mean values which are subtracted from channels. Values are intended 849 * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true. 850 * @param scalefactor multiplier for @p image values. 851 * @param swapRB flag which indicates that swap first and last channels 852 * in 3-channel image is necessary. 853 * @param crop flag which indicates whether image will be cropped after resize or not 854 * @param ddepth Depth of output blob. Choose CV_32F or CV_8U. 855 * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding 856 * dimension in @p size and another one is equal or larger. Then, crop from the center is performed. 857 * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed. 858 * @returns 4-dimensional Mat with NCHW dimensions order. 859 */ 860 CV_EXPORTS_W Mat blobFromImage(InputArray image, double scalefactor=1.0, const Size& size = Size(), 861 const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false, 862 int ddepth=CV_32F); 863 864 /** @brief Creates 4-dimensional blob from image. 865 * @details This is an overloaded member function, provided for convenience. 866 * It differs from the above function only in what argument(s) it accepts. 867 */ 868 CV_EXPORTS void blobFromImage(InputArray image, OutputArray blob, double scalefactor=1.0, 869 const Size& size = Size(), const Scalar& mean = Scalar(), 870 bool swapRB=false, bool crop=false, int ddepth=CV_32F); 871 872 873 /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and 874 * crops @p images from center, subtract @p mean values, scales values by @p scalefactor, 875 * swap Blue and Red channels. 876 * @param images input images (all with 1-, 3- or 4-channels). 877 * @param size spatial size for output image 878 * @param mean scalar with mean values which are subtracted from channels. Values are intended 879 * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true. 880 * @param scalefactor multiplier for @p images values. 881 * @param swapRB flag which indicates that swap first and last channels 882 * in 3-channel image is necessary. 883 * @param crop flag which indicates whether image will be cropped after resize or not 884 * @param ddepth Depth of output blob. Choose CV_32F or CV_8U. 885 * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding 886 * dimension in @p size and another one is equal or larger. Then, crop from the center is performed. 887 * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed. 888 * @returns 4-dimensional Mat with NCHW dimensions order. 889 */ 890 CV_EXPORTS_W Mat blobFromImages(InputArrayOfArrays images, double scalefactor=1.0, 891 Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false, 892 int ddepth=CV_32F); 893 894 /** @brief Creates 4-dimensional blob from series of images. 895 * @details This is an overloaded member function, provided for convenience. 896 * It differs from the above function only in what argument(s) it accepts. 897 */ 898 CV_EXPORTS void blobFromImages(InputArrayOfArrays images, OutputArray blob, 899 double scalefactor=1.0, Size size = Size(), 900 const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false, 901 int ddepth=CV_32F); 902 903 /** @brief Parse a 4D blob and output the images it contains as 2D arrays through a simpler data structure 904 * (std::vector<cv::Mat>). 905 * @param[in] blob_ 4 dimensional array (images, channels, height, width) in floating point precision (CV_32F) from 906 * which you would like to extract the images. 907 * @param[out] images_ array of 2D Mat containing the images extracted from the blob in floating point precision 908 * (CV_32F). They are non normalized neither mean added. The number of returned images equals the first dimension 909 * of the blob (batch size). Every image has a number of channels equals to the second dimension of the blob (depth). 910 */ 911 CV_EXPORTS_W void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_); 912 913 /** @brief Convert all weights of Caffe network to half precision floating point. 914 * @param src Path to origin model from Caffe framework contains single 915 * precision floating point weights (usually has `.caffemodel` extension). 916 * @param dst Path to destination model with updated weights. 917 * @param layersTypes Set of layers types which parameters will be converted. 918 * By default, converts only Convolutional and Fully-Connected layers' 919 * weights. 920 * 921 * @note Shrinked model has no origin float32 weights so it can't be used 922 * in origin Caffe framework anymore. However the structure of data 923 * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe. 924 * So the resulting model may be used there. 925 */ 926 CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst, 927 const std::vector<String>& layersTypes = std::vector<String>()); 928 929 /** @brief Create a text representation for a binary network stored in protocol buffer format. 930 * @param[in] model A path to binary network. 931 * @param[in] output A path to output text file to be created. 932 * 933 * @note To reduce output file size, trained weights are not included. 934 */ 935 CV_EXPORTS_W void writeTextGraph(const String& model, const String& output); 936 937 /** @brief Performs non maximum suppression given boxes and corresponding scores. 938 939 * @param bboxes a set of bounding boxes to apply NMS. 940 * @param scores a set of corresponding confidences. 941 * @param score_threshold a threshold used to filter boxes by score. 942 * @param nms_threshold a threshold used in non maximum suppression. 943 * @param indices the kept indices of bboxes after NMS. 944 * @param eta a coefficient in adaptive threshold formula: \f$nms\_threshold_{i+1}=eta\cdot nms\_threshold_i\f$. 945 * @param top_k if `>0`, keep at most @p top_k picked indices. 946 */ 947 CV_EXPORTS_W void NMSBoxes(const std::vector<Rect>& bboxes, const std::vector<float>& scores, 948 const float score_threshold, const float nms_threshold, 949 CV_OUT std::vector<int>& indices, 950 const float eta = 1.f, const int top_k = 0); 951 952 CV_EXPORTS_W void NMSBoxes(const std::vector<Rect2d>& bboxes, const std::vector<float>& scores, 953 const float score_threshold, const float nms_threshold, 954 CV_OUT std::vector<int>& indices, 955 const float eta = 1.f, const int top_k = 0); 956 957 CV_EXPORTS_AS(NMSBoxesRotated) void NMSBoxes(const std::vector<RotatedRect>& bboxes, const std::vector<float>& scores, 958 const float score_threshold, const float nms_threshold, 959 CV_OUT std::vector<int>& indices, 960 const float eta = 1.f, const int top_k = 0); 961 962 /** @brief Release a Myriad device is binded by OpenCV. 963 * 964 * Single Myriad device cannot be shared across multiple processes which uses 965 * Inference Engine's Myriad plugin. 966 */ 967 CV_EXPORTS_W void resetMyriadDevice(); 968 969 //! @} 970 CV__DNN_EXPERIMENTAL_NS_END 971 } 972 } 973 974 #include <opencv2/dnn/layer.hpp> 975 #include <opencv2/dnn/dnn.inl.hpp> 976 977 #endif /* OPENCV_DNN_DNN_HPP */ 978