1 package com.rockchip.gpadc.demo; 2 3 import android.app.Activity; 4 import android.content.SharedPreferences; 5 import android.content.res.Resources; 6 import android.graphics.Bitmap; 7 import android.graphics.Canvas; 8 import android.graphics.Paint; 9 import android.graphics.PorterDuff; 10 import android.graphics.PorterDuffXfermode; 11 import android.graphics.RectF; 12 import android.graphics.SurfaceTexture; 13 import android.graphics.Typeface; 14 import android.hardware.Camera; 15 import android.os.Bundle; 16 import android.os.Handler; 17 import android.os.Message; 18 import android.util.Log; 19 import android.view.Gravity; 20 import android.view.SurfaceHolder; 21 import android.view.SurfaceView; 22 import android.view.WindowManager; 23 import android.widget.ImageView; 24 import android.widget.TextView; 25 import android.widget.Toast; 26 27 import com.rockchip.gpadc.demo.rga.RGA; 28 import com.rockchip.gpadc.demo.yolo.InferenceWrapper; 29 30 import java.io.File; 31 import java.io.FileOutputStream; 32 import java.io.IOException; 33 import java.io.InputStream; 34 import java.lang.reflect.Method; 35 import java.text.DecimalFormat; 36 import java.util.ArrayList; 37 import java.util.List; 38 39 import static com.rockchip.gpadc.demo.rga.HALDefine.CAMERA_PREVIEW_HEIGHT; 40 import static com.rockchip.gpadc.demo.rga.HALDefine.CAMERA_PREVIEW_WIDTH; 41 import static com.rockchip.gpadc.demo.rga.HALDefine.IM_HAL_TRANSFORM_FLIP_H; 42 import static com.rockchip.gpadc.demo.rga.HALDefine.RK_FORMAT_RGBA_8888; 43 import static com.rockchip.gpadc.demo.rga.HALDefine.RK_FORMAT_YCrCb_420_SP; 44 import static com.rockchip.gpadc.demo.yolo.PostProcess.INPUT_CHANNEL; 45 import static java.lang.Thread.sleep; 46 47 48 public class CameraPreviewActivity extends Activity implements Camera.PreviewCallback { 49 50 private final String TAG = "rkyolo"; 51 private static final int MAGIC_TEXTURE_ID = 10; 52 53 TSurfaceHolderCallback mSurfaceHolderCallback = null; 54 55 private Camera mCamera0 = null; 56 private SurfaceView mSurfaceView = null; 57 public SurfaceTexture mSurfaceTexture = null; 58 private SurfaceHolder mSurfaceHolder = null; 59 public int flip = -1; // for CAMERA_FACING_BACK(camera comes with RK3588 using this mode), 60 // we do not need flip, using -1, or we need using 61 // IM_HAL_TRANSFORM_FLIP_H 62 63 private boolean mIsCameraOpened = false; 64 private int mCameraId = -1; 65 public byte textureBuffer[]; 66 67 // for inference 68 private String mModelName = "yolov5s.rknn"; 69 private String platform = "rk3588"; 70 private InferenceWrapper mInferenceWrapper; 71 private String fileDirPath; // file dir to store model cache 72 private ImageBufferQueue mImageBufferQueue; // intermedia between camera thread and inference thread 73 private InferenceResult mInferenceResult = new InferenceResult(); // detection result 74 private int mWidth; //surface width 75 private int mHeight; //surface height 76 private volatile boolean mStopInference = false; 77 78 //draw result 79 private TextView mFpsNum1; 80 private TextView mFpsNum2; 81 private TextView mFpsNum3; 82 private TextView mFpsNum4; 83 private ImageView mTrackResultView; 84 private Bitmap mTrackResultBitmap = null; 85 private Canvas mTrackResultCanvas = null; 86 private Paint mTrackResultPaint = null; 87 private Paint mTrackResultTextPaint = null; 88 89 private PorterDuffXfermode mPorterDuffXfermodeClear; 90 private PorterDuffXfermode mPorterDuffXfermodeSRC; 91 92 @Override onCreate(Bundle savedInstanceState)93 protected void onCreate(Bundle savedInstanceState) { 94 super.onCreate(savedInstanceState); 95 setContentView(R.layout.activity_main); 96 97 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); 98 99 mFpsNum1 = (TextView) findViewById(R.id.fps_num1); 100 mFpsNum2 = (TextView) findViewById(R.id.fps_num2); 101 mFpsNum3 = (TextView) findViewById(R.id.fps_num3); 102 mFpsNum4 = (TextView) findViewById(R.id.fps_num4); 103 mTrackResultView = (ImageView) findViewById(R.id.canvasView); 104 105 fileDirPath = getCacheDir().getAbsolutePath(); 106 107 platform = getPlatform(); 108 Log.d(TAG, "get soc platform:" + platform); 109 110 if (platform.equals("rk3588")) { 111 createFile(mModelName, R.raw.yolov5s_rk3588); 112 } else if (platform.equals("rk356x")) { 113 createFile(mModelName, R.raw.yolov5s_rk3566); 114 } else if (platform.equals("rk3562")) { 115 createFile(mModelName, R.raw.yolov5s_rk3562); 116 } else { 117 Toast toast = Toast.makeText(this, "Can not get platform use RK3588 instead.", Toast.LENGTH_LONG); 118 toast.setGravity(Gravity.CENTER, 0, 0); 119 toast.show(); 120 createFile(mModelName, R.raw.yolov5s_rk3588); 121 } 122 123 try { 124 mInferenceResult.init(getAssets()); 125 } catch (IOException e) { 126 e.printStackTrace(); 127 } 128 129 mInferenceWrapper = new InferenceWrapper(); 130 131 } 132 133 @Override onDestroy()134 protected void onDestroy() { 135 Log.d(TAG, "onDestroy"); 136 137 destroyPreviewView(); 138 super.onDestroy(); 139 } 140 141 @Override onPause()142 protected void onPause() { 143 Log.d(TAG, "onPause"); 144 stopTrack(); 145 stopCamera(); 146 destroyPreviewView(); 147 super.onPause(); 148 149 } 150 151 @Override onResume()152 protected void onResume() { 153 Log.d(TAG, "onResume"); 154 155 createPreviewView(); 156 super.onResume(); 157 158 } 159 createPreviewView()160 private boolean createPreviewView() { 161 mSurfaceView = findViewById(R.id.surfaceViewCamera1); 162 mSurfaceHolder = mSurfaceView.getHolder(); 163 // mSurfaceView.setZOrderMediaOverlay(true); 164 165 mSurfaceTexture = new SurfaceTexture(MAGIC_TEXTURE_ID); 166 167 mSurfaceHolderCallback = new TSurfaceHolderCallback(); 168 mSurfaceHolder.addCallback(mSurfaceHolderCallback); 169 170 return true; 171 } 172 destroyPreviewView()173 private void destroyPreviewView() { 174 if (mSurfaceHolder != null) { 175 mSurfaceHolder.removeCallback(mSurfaceHolderCallback); 176 mSurfaceHolderCallback = null; 177 mSurfaceHolder = null; 178 } 179 180 } 181 182 @Override onPreviewFrame(byte[] data, Camera camera)183 public void onPreviewFrame(byte[] data, Camera camera) { 184 mCamera0.addCallbackBuffer(data); 185 ImageBufferQueue.ImageBuffer imageBuffer = mImageBufferQueue.getFreeBuffer(); 186 187 if (imageBuffer != null) { 188 // RK_FORMAT_YCrCb_420_SP -> RK_FORMAT_RGBA_8888 189 // flip for CAMERA_FACING_FRONT 190 RGA.colorConvertAndFlip(data, RK_FORMAT_YCrCb_420_SP, 191 imageBuffer.mImage, RK_FORMAT_RGBA_8888, 192 CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT, this.flip); 193 194 mImageBufferQueue.postBuffer(imageBuffer); 195 } 196 } 197 198 private class TSurfaceHolderCallback implements SurfaceHolder.Callback { 199 200 @Override surfaceChanged(SurfaceHolder holder, int format, int width, int height)201 public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { 202 Log.d(TAG, "surfaceChanged"); 203 mWidth = width; 204 mHeight = height; 205 206 textureBuffer=new byte[CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 4]; 207 } 208 209 @Override surfaceCreated(SurfaceHolder holder)210 public void surfaceCreated(SurfaceHolder holder) { 211 Log.d(TAG, "surfaceCreated"); 212 213 startCamera(); 214 startTrack(); 215 216 } 217 218 @Override surfaceDestroyed(SurfaceHolder holder)219 public void surfaceDestroyed(SurfaceHolder holder) { 220 Log.d(TAG, "surfaceDestroyed"); 221 222 stopTrack(); 223 stopCamera(); 224 } 225 } 226 227 startCamera()228 private boolean startCamera() { 229 if (mIsCameraOpened) { 230 return true; 231 } 232 233 //(Camera.CameraInfo.CAMERA_FACING_BACK); 234 int num = Camera.getNumberOfCameras(); 235 if (num > 2) 236 mCameraId = 2; 237 else 238 mCameraId = 0; 239 Log.d(TAG, "mCameraId = " + mCameraId); 240 Camera.CameraInfo camInfo = new Camera.CameraInfo(); 241 try { 242 Camera.getCameraInfo(mCameraId, camInfo); 243 if (mCameraId != -1) { 244 mCamera0 = Camera.open(mCameraId); 245 } else { 246 mCamera0 = Camera.open(); 247 } 248 Log.d(TAG, "mCamera0 = " + mCamera0); 249 Log.d(TAG, "camera facing: " + camInfo.facing); 250 if (Camera.CameraInfo.CAMERA_FACING_FRONT == camInfo.facing) { 251 this.flip = IM_HAL_TRANSFORM_FLIP_H; 252 } 253 254 } catch (RuntimeException e) { 255 Log.w(TAG, "Unable to open camera!"); 256 Toast toast = Toast.makeText(this, "Unable to open camera!", Toast.LENGTH_LONG); 257 toast.setGravity(Gravity.CENTER, 0, 0); 258 toast.show(); 259 return false; 260 } 261 262 setCameraParameters(); 263 264 try { 265 mCamera0.setPreviewDisplay(mSurfaceHolder); 266 mCamera0.setDisplayOrientation(0); 267 int BUFFER_SIZE0 = CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 3 / 2; // NV21 268 byte[][] mPreviewData0 = new byte[][]{new byte[BUFFER_SIZE0], new byte[BUFFER_SIZE0],new byte[BUFFER_SIZE0]}; 269 //================================ 270 for (byte[] buffer : mPreviewData0) 271 mCamera0.addCallbackBuffer(buffer); 272 mCamera0.setPreviewCallbackWithBuffer(this); 273 //================================== 274 mCamera0.startPreview(); 275 } catch (Exception e) { 276 mCamera0.release(); 277 return false; 278 } 279 280 mIsCameraOpened = true; 281 282 return true; 283 } 284 stopCamera()285 private void stopCamera() { 286 if (mIsCameraOpened) { 287 mCamera0.setPreviewCallback(null); 288 mCamera0.stopPreview(); 289 mCamera0.release(); 290 mCamera0 = null; 291 mIsCameraOpened = false; 292 } 293 294 } 295 setCameraParameters()296 private void setCameraParameters() { 297 Camera.Parameters parameters; 298 boolean checkWH = false; 299 parameters = mCamera0.getParameters(); 300 int nearest_width_index = 0; 301 int nearest_width_value = 1920; 302 303 List<Camera.Size> sizes = parameters.getSupportedPreviewSizes(); 304 for (int i = 0; i < sizes.size(); i++) { 305 Camera.Size size = sizes.get(i); 306 307 if (Math.abs(size.width-CAMERA_PREVIEW_WIDTH) < nearest_width_value ) { 308 nearest_width_value = Math.abs(size.width-CAMERA_PREVIEW_WIDTH); 309 nearest_width_index = i; 310 } 311 312 if ( (size.width == CAMERA_PREVIEW_WIDTH) && (size.height == CAMERA_PREVIEW_HEIGHT)) { 313 checkWH = true; 314 } 315 316 Log.v(TAG, "Camera Supported Preview Size = " + size.width + "x" + size.height); 317 } 318 if (!checkWH) { 319 Log.e(TAG, "Camera don't support this preview Size = " + CAMERA_PREVIEW_WIDTH + "x" + CAMERA_PREVIEW_HEIGHT); 320 CAMERA_PREVIEW_WIDTH = sizes.get(nearest_width_index).width; 321 CAMERA_PREVIEW_HEIGHT = sizes.get(nearest_width_index).height; 322 } 323 324 Log.w(TAG, "Use preview Size = " + CAMERA_PREVIEW_WIDTH + "x" + CAMERA_PREVIEW_HEIGHT); 325 326 parameters.setPreviewSize(CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT); 327 328 if (parameters.isZoomSupported()) { 329 parameters.setZoom(0); 330 } 331 mCamera0.setParameters(parameters); 332 Log.i(TAG, "mCamera0 set parameters success."); 333 } 334 startTrack()335 private void startTrack() { 336 mInferenceResult.reset(); 337 mImageBufferQueue = new ImageBufferQueue(3, CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT); 338 mStopInference = false; 339 mInferenceThread = new Thread(mInferenceRunnable); 340 mInferenceThread.start(); 341 } 342 stopTrack()343 private void stopTrack() { 344 345 mStopInference = true; 346 try { 347 if (mInferenceThread != null) { 348 mInferenceThread.join(); 349 } 350 } catch (InterruptedException e) { 351 e.printStackTrace(); 352 } 353 354 if (mImageBufferQueue != null) { 355 mImageBufferQueue.release(); 356 mImageBufferQueue = null; 357 } 358 } 359 360 private Thread mInferenceThread; 361 private Runnable mInferenceRunnable = new Runnable() { 362 public void run() { 363 364 int count = 0; 365 long oldTime = System.currentTimeMillis(); 366 long currentTime; 367 368 updateMainUI(1, 0); 369 370 String paramPath = fileDirPath + "/" + mModelName; 371 372 try { 373 mInferenceWrapper.initModel(CAMERA_PREVIEW_HEIGHT, CAMERA_PREVIEW_WIDTH, INPUT_CHANNEL, paramPath); 374 } catch (Exception e) { 375 e.printStackTrace(); 376 System.exit(1); 377 } 378 379 380 while (!mStopInference) { 381 ImageBufferQueue.ImageBuffer buffer = mImageBufferQueue.getReadyBuffer(); 382 383 if (buffer == null) { 384 try { 385 // Log.w(TAG, "buffer is null."); 386 sleep(10); 387 } catch (InterruptedException e) { 388 e.printStackTrace(); 389 } 390 continue; 391 } 392 393 InferenceResult.OutputBuffer outputs = mInferenceWrapper.run(buffer.mImage); 394 395 mInferenceResult.setResult(outputs); 396 397 mImageBufferQueue.releaseBuffer(buffer); 398 399 if (++count >= 30) { 400 currentTime = System.currentTimeMillis(); 401 402 float fps = count * 1000.f / (currentTime - oldTime); 403 404 // Log.d(TAG, "current fps = " + fps); 405 406 oldTime = currentTime; 407 count = 0; 408 updateMainUI(0, fps); 409 410 } 411 412 updateMainUI(1, 0); 413 } 414 415 mInferenceWrapper.deinit(); 416 } 417 }; 418 createFile(String fileName, int id)419 private void createFile(String fileName, int id) { 420 String filePath = fileDirPath + "/" + fileName; 421 try { 422 File dir = new File(fileDirPath); 423 424 if (!dir.exists()) { 425 dir.mkdirs(); 426 } 427 428 // 目录存在，则将apk中raw中的需要的文档复制到该目录下 429 File file = new File(filePath); 430 431 if (!file.exists() || isFirstRun()) { 432 433 InputStream ins = getResources().openRawResource(id);// 通过raw得到数据资源 434 FileOutputStream fos = new FileOutputStream(file); 435 byte[] buffer = new byte[8192]; 436 int count = 0; 437 438 while ((count = ins.read(buffer)) > 0) { 439 fos.write(buffer, 0, count); 440 } 441 442 fos.close(); 443 ins.close(); 444 445 Log.d(TAG, "Create " + filePath); 446 } 447 } catch (Exception e) { 448 e.printStackTrace(); 449 } 450 } 451 isFirstRun()452 private boolean isFirstRun() { 453 SharedPreferences sharedPreferences = getSharedPreferences("setting", MODE_PRIVATE); 454 boolean isFirstRun = sharedPreferences.getBoolean("isFirstRun", true); 455 SharedPreferences.Editor editor = sharedPreferences.edit(); 456 if (isFirstRun) { 457 editor.putBoolean("isFirstRun", false); 458 editor.commit(); 459 } 460 461 return isFirstRun; 462 } 463 464 // UI线程，用于更新处理结果 465 private Handler mHandler = new Handler() 466 { 467 public void handleMessage(Message msg) 468 { 469 if (msg.what == 0) { 470 float fps = (float) msg.obj; 471 472 DecimalFormat decimalFormat = new DecimalFormat("00.00"); 473 String fpsStr = decimalFormat.format(fps); 474 mFpsNum1.setText(String.valueOf(fpsStr.charAt(0))); 475 mFpsNum2.setText(String.valueOf(fpsStr.charAt(1))); 476 mFpsNum3.setText(String.valueOf(fpsStr.charAt(3))); 477 mFpsNum4.setText(String.valueOf(fpsStr.charAt(4))); 478 } else { 479 showTrackSelectResults(); 480 } 481 } 482 }; 483 updateMainUI(int type, Object data)484 private void updateMainUI(int type, Object data) { 485 Message msg = mHandler.obtainMessage(); 486 msg.what = type; 487 msg.obj = data; 488 mHandler.sendMessage(msg); 489 } 490 sp2px(float spValue)491 public static int sp2px(float spValue) { 492 Resources r = Resources.getSystem(); 493 final float scale = r.getDisplayMetrics().scaledDensity; 494 return (int) (spValue * scale + 0.5f); 495 } 496 showTrackSelectResults()497 private void showTrackSelectResults() { 498 499 int width = CAMERA_PREVIEW_WIDTH; 500 int height = CAMERA_PREVIEW_HEIGHT; 501 502 if (mTrackResultBitmap == null) { 503 504 mTrackResultBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); 505 mTrackResultCanvas = new Canvas(mTrackResultBitmap); 506 507 //用于画线 508 mTrackResultPaint = new Paint(); 509 mTrackResultPaint.setColor(0xff06ebff); 510 mTrackResultPaint.setStrokeJoin(Paint.Join.ROUND); 511 mTrackResultPaint.setStrokeCap(Paint.Cap.ROUND); 512 mTrackResultPaint.setStrokeWidth(4); 513 mTrackResultPaint.setStyle(Paint.Style.STROKE); 514 mTrackResultPaint.setTextAlign(Paint.Align.LEFT); 515 mTrackResultPaint.setTextSize(sp2px(10)); 516 mTrackResultPaint.setTypeface(Typeface.SANS_SERIF); 517 mTrackResultPaint.setFakeBoldText(false); 518 519 //用于文字 520 mTrackResultTextPaint = new Paint(); 521 mTrackResultTextPaint.setColor(0xff06ebff); 522 mTrackResultTextPaint.setStrokeWidth(2); 523 mTrackResultTextPaint.setTextAlign(Paint.Align.LEFT); 524 mTrackResultTextPaint.setTextSize(sp2px(12)); 525 mTrackResultTextPaint.setTypeface(Typeface.SANS_SERIF); 526 mTrackResultTextPaint.setFakeBoldText(false); 527 528 529 mPorterDuffXfermodeClear = new PorterDuffXfermode(PorterDuff.Mode.CLEAR); 530 mPorterDuffXfermodeSRC = new PorterDuffXfermode(PorterDuff.Mode.SRC); 531 } 532 533 // clear canvas 534 mTrackResultPaint.setXfermode(mPorterDuffXfermodeClear); 535 mTrackResultCanvas.drawPaint(mTrackResultPaint); 536 mTrackResultPaint.setXfermode(mPorterDuffXfermodeSRC); 537 538 //detect result 539 ArrayList<InferenceResult.Recognition> recognitions = mInferenceResult.getResult(mInferenceWrapper); 540 for (int i=0; i<recognitions.size(); ++i) { 541 InferenceResult.Recognition rego = recognitions.get(i); 542 RectF detection = rego.getLocation(); 543 544 detection.left *= width; 545 detection.right *= width; 546 detection.top *= height; 547 detection.bottom *= height; 548 549 // Log.d(TAG, rego.toString()); 550 // Log.d(TAG, detection.toString()); 551 552 mTrackResultCanvas.drawRect(detection, mTrackResultPaint); 553 mTrackResultCanvas.drawText(rego.getTrackId() + " - " + mInferenceResult.mPostProcess.getLabelTitle(rego.getId()), 554 detection.left+5, detection.bottom-5, mTrackResultTextPaint); 555 } 556 557 mTrackResultView.setScaleType(ImageView.ScaleType.FIT_XY); 558 mTrackResultView.setImageBitmap(mTrackResultBitmap); 559 } 560 getPlatform()561 private String getPlatform()//取平台版本 562 { 563 String platform = null; 564 try { 565 Class<?> classType = Class.forName("android.os.SystemProperties"); 566 Method getMethod = classType.getDeclaredMethod("get", new Class<?>[]{String.class}); 567 platform = (String) getMethod.invoke(classType, new Object[]{"ro.board.platform"}); 568 } catch (Exception e) { 569 e.printStackTrace(); 570 } 571 return platform; 572 } 573 } 574