如何使用onPreviewFrame()中的byte[]数据在人像中使用OpenCV人脸检测

Question

我正在尝试使用OpenCV人脸检测，使用从Camera.PreviewCallback 的onPreviewFrame((方法获得的byte[]数据

我设法使用以下代码将数据转换为灰度图像。

    Mat matNew = new Mat(pHeight, pWidth, CvType.CV_8U);
    matNew.put(0, 0, data);
    Mat matrgb = new Mat();
    Imgproc.cvtColor(matNew, matrgb, Imgproc.COLOR_YUV420sp2RGB, 4);
    Mat matgray = new Mat();
    Imgproc.cvtColor(matrgb, matgray, Imgproc.COLOR_RGB2GRAY, 0);

我已经在AndroidManifest文件中将android:screenOrientation设置为"肖像"。

我正在使用OpenCV JavaDetector

mJavaDetector.detectMultiScale(matgray，faceDetected，1.1，3，0，new org.opencv.core.Size(0,0(，new org.openccv.core.Size(matgray.width((，matgray.height(((；并在使用该检测到的人脸上绘制一个矩形

 for (Rect rect : faceDetected.toArray()){
  Core.rectangle(matgray, new Point(rect.x, rect.y), 
new Point(rect.x + rect.width, rect.y + rect.height),
new Scalar(0, 255, 0));
                    }

然而，在产生的灰度垫中，只有当我将Android手机放在横向位置时，才会进行人脸检测。它在人像位置不起作用。

这篇文章中建议的转置和翻转如何通过人像模式检测人脸？似乎不起作用。

有什么办法可以克服这个问题吗？我使用了Android FaceDetectionListener，在人像模式下检测人脸似乎没有问题。但是，与OpenCV相比，FaceDetectionListener的功能是有限的。

如有任何帮助，我们将不胜感激。Thx。

Answer 1

以下内容适用于AndroidManifest.xml 中的android:screenOrientation="portrait"

public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
    mRgba = inputFrame.rgba();
    mGray = inputFrame.gray();
    MatOfRect faces = new MatOfRect();
    Core.flip(mRgba.t(), mRgba, -1);
    Core.flip(mGray.t(), mGray, -1);
    if (mNativeDetector != null)
        mNativeDetector.detect(mGray, faces);
    Rect[] facesArray = faces.toArray();
    for (int i = 0; i < facesArray.length; i++)
        Core.rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(), FACE_RECT_COLOR, 2);
    return mRgba;
}

Answer 2

顺时针翻转彩色和灰色图像(Mat(，以便人脸/特征检测在人像模式下工作。在特征检测逻辑结束时，逆时针翻转彩色图像(mRgba-Mat(。如图所示。

 public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
    Core.flip(inputFrame.gray().t(),mGray,1); //rotate clockwise 
    Core.flip(inputFrame.rgba().t(),mRgba,1);
    mRgba=Feature_DetectionNATIVE(mRgba,mGray);
    Core.flip(mRgba.t(),mRgba,0);             //rotate counter clockwise
//this is a solution for  allowing face detection in portrait view if it isn't working at all.
    return mRgba;
 }
public Mat Feature_DetectionNATIVE(Mat mRgba2, final Mat Gray)
{
if (mAbsoluteFaceSize == 0) 
{
    int height = Gray.rows();
    if (Math.round(height * mRelativeFaceSize) > 0)
     {
       mAbsoluteFaceSize = Math.round(height * mRelativeFaceSize);
     }
 mNativeDetector.setMinFaceSize(mAbsoluteFaceSize);
}
MatOfRect faces = new MatOfRect();
if (mDetectorType == JAVA_DETECTOR) 
 {
if (mJavaDetector != null)
  mJavaDetector.detectMultiScale(Gray, faces, 1.1, 2, 2, 
  new Size(mAbsoluteFaceSize, mAbsoluteFaceSize), new Size());
 }
else if (mDetectorType == NATIVE_DETECTOR)
{
if (mNativeDetector != null)
  mNativeDetector.detect(Gray, faces);
 }


Rect[] facesArray = faces.toArray();
for (int i = 0; i < facesArray.length; i++)
{
  Core.rectangle(mRgba2, facesArray[i].tl(), facesArray[i].br(),     FACE_RECT_COLOR, 3);
}
    return mRgba2;
}

之后，相机将在横向方向显示人脸检测，以解决此问题，您可以在opencv的CameraBridgeViewBase主类中将画布顺时针旋转90，或者破解它。(注意，这会降低FPS，但人脸检测仍然很快(

 protected void deliverAndDrawFrame(CvCameraViewFrame frame) {
 Mat modified;
 if (mListener != null) {
 modified = mListener.onCameraFrame(frame);
 } else {
  modified = frame.rgba();
 }
 boolean bmpValid = true;
 if (modified != null) {
 try {
  Utils.matToBitmap(modified, mCacheBitmap);
  } catch(Exception e) {
 Log.e(TAG, "Mat type: " + modified);
 Log.e(TAG, "Bitmap type: " + mCacheBitmap.getWidth() + "*" +   mCacheBitmap.getHeight());
 Log.e(TAG, "Utils.matToBitmap() throws an exception: " + e.getMessage());
 bmpValid = false;
  }
 }
 if (bmpValid && mCacheBitmap != null) {
 Canvas canvas = getHolder().lockCanvas();
  if (canvas != null) {
 canvas.drawColor(0, android.graphics.PorterDuff.Mode.CLEAR);
 Log.d(TAG, "mStretch value: " + mScale);

 canvas=rotateCanvas(canvas,mCacheBitmap);
 getHolder().unlockCanvasAndPost(canvas);
 }
  }
 }
 protected Canvas rotateCanvas(final Canvas canvas, final Bitmap mCacheBitmap)
 { 
 final CountDownLatch latch =new CountDownLatch(1);
 final Mat[] mRgba=new Mat[1];
 new Thread(new Runnable() {
 @Override
 public void run() {
 try {
 Bitmap bitmap = Bitmap.createScaledBitmap(mCacheBitmap, canvas.getHeight(),  canvas.getWidth(), true);
canvas.rotate(90,0,0);
mScale = canvas.getWidth() / (float)bitmap.getHeight();
float scale2 = canvas.getHeight() / (float)bitmap.getWidth();
if(scale2 > mScale){
    mScale = scale2;
 }
if (mScale != 0) {
 canvas.scale(mScale, mScale,0,0);
 }
canvas.drawBitmap(bitmap, 0, -bitmap.getHeight(), null);
}
catch (CvException e) {           e.printStackTrace();}
latch.countDown();//setting //release await() in this thread
}
}).start();
try {  latch.await(); //waits for countDown in the Thread inorder to obtain a value from the thread
} catch (InterruptedException e) {   e.printStackTrace();}
return canvas;

}

此解决方案在实现Opencv的CameraBridgeViewBase类时有效(在OpenCV 2.4.9上测试(