1 // Copyright (c) 2021 by Rockchip Electronics Co., Ltd. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 /*-------------------------------------------
16 Includes
17 -------------------------------------------*/
18
19 #include <float.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sys/time.h>
24
25 #include "RgaUtils.h"
26 #include "im2d.h"
27 #include "rga.h"
28 #include "rknn_api.h"
29
30 #define STB_IMAGE_IMPLEMENTATION
31 #include "stb/stb_image.h"
32 #define STB_IMAGE_RESIZE_IMPLEMENTATION
33 #include <stb/stb_image_resize.h>
34
35 #define ALIGN 8
36
37 /*-------------------------------------------
38 Functions
39 -------------------------------------------*/
getCurrentTimeUs()40 static inline int64_t getCurrentTimeUs()
41 {
42 struct timeval tv;
43 gettimeofday(&tv, NULL);
44 return tv.tv_sec * 1000000 + tv.tv_usec;
45 }
46
rknn_GetTopN(float * pfProb,float * pfMaxProb,uint32_t * pMaxClass,uint32_t outputCount,uint32_t topNum)47 static int rknn_GetTopN(float* pfProb, float* pfMaxProb, uint32_t* pMaxClass, uint32_t outputCount, uint32_t topNum)
48 {
49 uint32_t i, j;
50 uint32_t top_count = outputCount > topNum ? topNum : outputCount;
51
52 for (i = 0; i < topNum; ++i) {
53 pfMaxProb[i] = -FLT_MAX;
54 pMaxClass[i] = -1;
55 }
56
57 for (j = 0; j < top_count; j++) {
58 for (i = 0; i < outputCount; i++) {
59 if ((i == *(pMaxClass + 0)) || (i == *(pMaxClass + 1)) || (i == *(pMaxClass + 2)) || (i == *(pMaxClass + 3)) ||
60 (i == *(pMaxClass + 4))) {
61 continue;
62 }
63
64 if (pfProb[i] > *(pfMaxProb + j)) {
65 *(pfMaxProb + j) = pfProb[i];
66 *(pMaxClass + j) = i;
67 }
68 }
69 }
70
71 return 1;
72 }
73
dump_tensor_attr(rknn_tensor_attr * attr)74 static void dump_tensor_attr(rknn_tensor_attr* attr)
75 {
76 printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, "
77 "zp=%d, scale=%f\n",
78 attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3],
79 attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type),
80 get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale);
81 }
82
load_image(const char * image_path,rknn_tensor_attr * input_attr)83 unsigned char* load_image(const char* image_path, rknn_tensor_attr* input_attr)
84 {
85 int req_height = 0;
86 int req_width = 0;
87 int req_channel = 0;
88
89 switch (input_attr->fmt) {
90 case RKNN_TENSOR_NHWC:
91 req_height = input_attr->dims[1];
92 req_width = input_attr->dims[2];
93 req_channel = input_attr->dims[3];
94 break;
95 case RKNN_TENSOR_NCHW:
96 req_height = input_attr->dims[2];
97 req_width = input_attr->dims[3];
98 req_channel = input_attr->dims[1];
99 break;
100 default:
101 printf("meet unsupported layout\n");
102 return NULL;
103 }
104
105 int height = 0;
106 int width = 0;
107 int channel = 0;
108
109 unsigned char* image_data = stbi_load(image_path, &width, &height, &channel, req_channel);
110 if (image_data == NULL) {
111 printf("load image failed!\n");
112 return NULL;
113 }
114
115 if (width != req_width || height != req_height) {
116 unsigned char* image_resized = (unsigned char*)STBI_MALLOC(req_width * req_height * req_channel);
117 if (!image_resized) {
118 printf("malloc image failed!\n");
119 STBI_FREE(image_data);
120 return NULL;
121 }
122 if (stbir_resize_uint8(image_data, width, height, 0, image_resized, req_width, req_height, 0, channel) != 1) {
123 printf("resize image failed!\n");
124 STBI_FREE(image_data);
125 return NULL;
126 }
127 STBI_FREE(image_data);
128 image_data = image_resized;
129 }
130
131 return image_data;
132 }
133
134 /*-------------------------------------------
135 Main Functions
136 -------------------------------------------*/
main(int argc,char * argv[])137 int main(int argc, char* argv[])
138 {
139 if (argc < 3) {
140 printf("Usage:%s model_path input_path [loop_count]\n", argv[0]);
141 return -1;
142 }
143
144 char* model_path = argv[1];
145 char* input_path = argv[2];
146
147 int loop_count = 1;
148 if (argc > 3) {
149 loop_count = atoi(argv[3]);
150 }
151
152 rknn_context ctx = 0;
153
154 // init rga context
155 rga_buffer_t src;
156 rga_buffer_t dst;
157 im_rect src_rect;
158 im_rect dst_rect;
159 memset(&src_rect, 0, sizeof(src_rect));
160 memset(&dst_rect, 0, sizeof(dst_rect));
161 memset(&src, 0, sizeof(src));
162 memset(&dst, 0, sizeof(dst));
163
164 // Load RKNN Model
165 int ret = rknn_init(&ctx, model_path, 0, 0, NULL);
166 if (ret < 0) {
167 printf("rknn_init fail! ret=%d\n", ret);
168 return -1;
169 }
170
171 // Get sdk and driver version
172 rknn_sdk_version sdk_ver;
173 ret = rknn_query(ctx, RKNN_QUERY_SDK_VERSION, &sdk_ver, sizeof(sdk_ver));
174 if (ret != RKNN_SUCC) {
175 printf("rknn_query fail! ret=%d\n", ret);
176 return -1;
177 }
178 printf("rknn_api/rknnrt version: %s, driver version: %s\n", sdk_ver.api_version, sdk_ver.drv_version);
179
180 // Get Model Input Output Info
181 rknn_input_output_num io_num;
182 ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num));
183 if (ret != RKNN_SUCC) {
184 printf("rknn_query fail! ret=%d\n", ret);
185 return -1;
186 }
187 printf("model input num: %d, output num: %d\n", io_num.n_input, io_num.n_output);
188
189 printf("input tensors:\n");
190 rknn_tensor_attr input_attrs[io_num.n_input];
191 memset(input_attrs, 0, io_num.n_input * sizeof(rknn_tensor_attr));
192 for (uint32_t i = 0; i < io_num.n_input; i++) {
193 input_attrs[i].index = i;
194 // query info
195 ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr));
196 if (ret < 0) {
197 printf("rknn_init error! ret=%d\n", ret);
198 return -1;
199 }
200 dump_tensor_attr(&input_attrs[i]);
201 }
202
203 printf("output tensors:\n");
204 rknn_tensor_attr output_attrs[io_num.n_output];
205 memset(output_attrs, 0, io_num.n_output * sizeof(rknn_tensor_attr));
206 for (uint32_t i = 0; i < io_num.n_output; i++) {
207 output_attrs[i].index = i;
208 // query info
209 ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr));
210 if (ret != RKNN_SUCC) {
211 printf("rknn_query fail! ret=%d\n", ret);
212 return -1;
213 }
214 dump_tensor_attr(&output_attrs[i]);
215 }
216
217 // Get custom string
218 rknn_custom_string custom_string;
219 ret = rknn_query(ctx, RKNN_QUERY_CUSTOM_STRING, &custom_string, sizeof(custom_string));
220 if (ret != RKNN_SUCC) {
221 printf("rknn_query fail! ret=%d\n", ret);
222 return -1;
223 }
224 printf("custom string: %s\n", custom_string.string);
225
226 unsigned char* input_data = NULL;
227 rknn_tensor_type input_type = RKNN_TENSOR_UINT8;
228 rknn_tensor_format input_layout = RKNN_TENSOR_NHWC;
229
230 int img_input_height = 0;
231 int img_input_width = 0;
232 int channel = 0;
233 input_data = stbi_load(input_path, &img_input_height, &img_input_width, &channel, 3);
234 int model_in_height = 0;
235 int model_in_width = 0;
236 int req_channel = 0;
237 switch (input_attrs[0].fmt) {
238 case RKNN_TENSOR_NHWC:
239 model_in_height = input_attrs[0].dims[1];
240 model_in_width = input_attrs[0].dims[2];
241 req_channel = input_attrs[0].dims[3];
242 break;
243 case RKNN_TENSOR_NCHW:
244 model_in_height = input_attrs[0].dims[2];
245 model_in_width = input_attrs[0].dims[3];
246 req_channel = input_attrs[0].dims[1];
247 break;
248 default:
249 printf("meet unsupported layout\n");
250 return -1;
251 }
252
253 if (!input_data) {
254 printf("load image failed!\n");
255 return -1;
256 }
257
258 src = wrapbuffer_virtualaddr((void*)input_data, img_input_width, img_input_height,
259 RK_FORMAT_RGB_888); // wstride, hstride,
260
261 // Create input tensor memory
262 rknn_tensor_mem* input_mems[1];
263 // default input type is int8 (normalize and quantize need compute in outside)
264 // if set uint8, will fuse normalize and quantize to npu
265 input_attrs[0].type = input_type;
266 // default fmt is NHWC, npu only support NHWC in zero copy mode
267 input_attrs[0].fmt = input_layout;
268 input_mems[0] = rknn_create_mem(ctx, input_attrs[0].size_with_stride);
269 // Copy input data to input tensor memory
270 // memcpy(input_mems[0]->virt_addr, input_data, input_attrs[0].size);
271
272 int wstride = model_in_width + (ALIGN - model_in_width % ALIGN) % ALIGN;
273 int hstride = model_in_height;
274 dst = wrapbuffer_fd_t(input_mems[0]->fd, model_in_width, model_in_height, wstride, hstride,
275 RK_FORMAT_RGB_888); // wstride, hstride,
276
277 ret = imcheck(src, dst, src_rect, dst_rect);
278 if (IM_STATUS_NOERROR != ret) {
279 printf("%d, check error! %s", __LINE__, imStrError((IM_STATUS)ret));
280 return -1;
281 }
282
283 IM_STATUS STATUS = imresize(src, dst);
284
285 // Create output tensor memory
286 rknn_tensor_mem* output_mems[io_num.n_output];
287 for (uint32_t i = 0; i < io_num.n_output; ++i) {
288 int output_size = output_attrs[i].n_elems * sizeof(float);
289 // default output type is depend on model, this require float32 to compute top5
290 output_attrs[i].type = RKNN_TENSOR_FLOAT32;
291 output_mems[i] = rknn_create_mem(ctx, output_size);
292 }
293
294 // Set input tensor memory
295 ret = rknn_set_io_mem(ctx, input_mems[0], &input_attrs[0]);
296 if (ret < 0) {
297 printf("rknn_set_io_mem fail! ret=%d\n", ret);
298 return -1;
299 }
300
301 // Set output tensor memory
302 for (uint32_t i = 0; i < io_num.n_output; ++i) {
303 ret = rknn_set_io_mem(ctx, output_mems[i], &output_attrs[i]);
304 if (ret < 0) {
305 printf("rknn_set_io_mem fail! ret=%d\n", ret);
306 return -1;
307 }
308 }
309
310 // Run
311 printf("Begin perf ...\n");
312 for (int i = 0; i < loop_count; ++i) {
313 int64_t start_us = getCurrentTimeUs();
314 ret = rknn_run(ctx, NULL);
315 int64_t elapse_us = getCurrentTimeUs() - start_us;
316 if (ret < 0) {
317 printf("rknn run error %d\n", ret);
318 return -1;
319 }
320 printf("%4d: Elapse Time = %.2fms, FPS = %.2f\n", i, elapse_us / 1000.f, 1000.f * 1000.f / elapse_us);
321 }
322
323 // Get top 5
324 uint32_t topNum = 5;
325 for (uint32_t i = 0; i < io_num.n_output; i++) {
326 uint32_t MaxClass[topNum];
327 float fMaxProb[topNum];
328 float* buffer = (float*)output_mems[i]->virt_addr;
329 uint32_t sz = output_attrs[i].n_elems;
330 int top_count = sz > topNum ? topNum : sz;
331
332 rknn_GetTopN(buffer, fMaxProb, MaxClass, sz, topNum);
333
334 printf("---- Top%d ----\n", top_count);
335 for (int j = 0; j < top_count; j++) {
336 printf("%8.6f - %d\n", fMaxProb[j], MaxClass[j]);
337 }
338 }
339
340 // Destroy rknn memory
341 rknn_destroy_mem(ctx, input_mems[0]);
342 for (uint32_t i = 0; i < io_num.n_output; ++i) {
343 rknn_destroy_mem(ctx, output_mems[i]);
344 }
345
346 // destroy
347 rknn_destroy(ctx);
348
349 free(input_data);
350
351 return 0;
352 }
353