使用计算机视觉实战项目精通 OpenCV：1~5

int cameraNumber = 0;
if (argc > 1)
  cameraNumber = atoi(argv[1]);
// Get access to the camera.
cv::VideoCapture camera;
camera.open(cameraNumber);
if (!camera.isOpened()) {
  std::cerr << "ERROR: Could not access the camera or video!" <<
  std::endl;
  exit(1);
}
// Try to set the camera resolution.
camera.set(cv::CV_CAP_PROP_FRAME_WIDTH, 640);
camera.set(cv::CV_CAP_PROP_FRAME_HEIGHT, 480);

while (true) {
  // Grab the next camera frame.
  cv::Mat cameraFrame;
 camera >> cameraFrame;
  if (cameraFrame.empty()) {
    std::cerr << "ERROR: Couldn't grab a camera frame." <<
    std::endl;
    exit(1);
  }
  // Create a blank output image, that we will draw onto.
  cv::Mat displayedFrame(cameraFrame.size(), cv::CV_8UC3);
  // Run the cartoonifier filter on the camera frame.
 cartoonifyImage(cameraFrame, displayedFrame);
  // Display the processed image onto the screen.
  imshow("Cartoonifier", displayedFrame);
  // IMPORTANT: Wait for at least 20 milliseconds,
  // so that the image can be displayed on the screen!
  // Also checks if a key was pressed in the GUI window.
  // Note that it should be a "char" to support Linux.
 char keypress = cv::waitKey(20);  // Need this to see anything!
  if (keypress == 27) {   // Escape Key
  // Quit the program!
  break;
  }
}//end while

// Include OpenCV's C++ Interface
#include "opencv2/opencv.hpp"
using namespace cv;
using namespace std;

Mat gray;
cvtColor(srcColor, gray, CV_BGR2GRAY);
const int MEDIAN_BLUR_FILTER_SIZE = 7;
medianBlur(gray, gray, MEDIAN_BLUR_FILTER_SIZE);
Mat edges;
const int LAPLACIAN_FILTER_SIZE = 5;
Laplacian(gray, edges, CV_8U, LAPLACIAN_FILTER_SIZE);

Mat mask;
const int EDGES_THRESHOLD = 80;
threshold(edges, mask, EDGES_THRESHOLD, 255, THRESH_BINARY_INV);

Size size = srcColor.size();
Size smallSize;
smallSize.width = size.width/2;
smallSize.height = size.height/2;
Mat smallImg = Mat(smallSize, CV_8UC3);
resize(srcColor, smallImg, smallSize, 0,0, INTER_LINEAR);

Mat tmp = Mat(smallSize, CV_8UC3);
int repetitions = 7;  // Repetitions for strong cartoon effect.
for (int i=0; i<repetitions; i++) {
  int ksize = 9;     // Filter size. Has a large effect on speed.
  double sigmaColor = 9;    // Filter color strength.
  double sigmaSpace = 7;    // Spatial strength. Affects speed.
 bilateralFilter(smallImg, tmp, ksize, sigmaColor, sigmaSpace);
 bilateralFilter(tmp, smallImg, ksize, sigmaColor, sigmaSpace);
}

Mat bigImg;
resize(smallImg, bigImg, size, 0,0, INTER_LINEAR);
dst.setTo(0);
bigImg.copyTo(dst, mask);

Mat gray;
cvtColor(srcColor, gray, CV_BGR2GRAY);
const int MEDIAN_BLUR_FILTER_SIZE = 7;
medianBlur(gray, gray, MEDIAN_BLUR_FILTER_SIZE);
Mat edges, edges2;
Scharr(srcGray, edges, CV_8U, 1, 0);
Scharr(srcGray, edges2, CV_8U, 1, 0, -1);
edges += edges2;     // Combine the x & y edges together.
const int EVIL_EDGE_THRESHOLD = 12;
threshold(edges, mask, EVIL_EDGE_THRESHOLD, 255, THRESH_BINARY_INV);
medianBlur(mask, mask, 3);

// Draw the color face onto a black background.
Mat faceOutline = Mat::zeros(size, CV_8UC3);
Scalar color = CV_RGB(255,255,0);    // Yellow.
int thickness = 4;
// Use 70% of the screen height as the face height.
int sw = size.width;
int sh = size.height;
int faceH = sh/2 * 70/100;  // "faceH" is the radius of the ellipse.
// Scale the width to be the same shape for any screen width. int faceW = faceH * 72/100;
// Draw the face outline.
ellipse(faceOutline, Point(sw/2, sh/2), Size(faceW, faceH),
 0, 0, 360, color, thickness, CV_AA);

// Draw the eye outlines, as 2 arcs per eye.
int eyeW = faceW * 23/100;
int eyeH = faceH * 11/100;
int eyeX = faceW * 48/100;
int eyeY = faceH * 13/100;
Size eyeSize = Size(eyeW, eyeH);
// Set the angle and shift for the eye half ellipses.
int eyeA = 15; // angle in degrees.
int eyeYshift = 11;
// Draw the top of the right eye.
ellipse(faceOutline, Point(sw/2 - eyeX, sh/2 – eyeY),
 eyeSize, 0, 180+eyeA, 360-eyeA, color, thickness, CV_AA);
// Draw the bottom of the right eye.
ellipse(faceOutline, Point(sw/2 - eyeX, sh/2 - eyeY – eyeYshift),
 eyeSize, 0, 0+eyeA, 180-eyeA, color, thickness, CV_AA);
// Draw the top of the left eye.
ellipse(faceOutline, Point(sw/2 + eyeX, sh/2 - eyeY),
 eyeSize, 0, 180+eyeA, 360-eyeA, color, thickness, CV_AA);
// Draw the bottom of the left eye.
ellipse(faceOutline, Point(sw/2 + eyeX, sh/2 - eyeY – eyeYshift),
 eyeSize, 0, 0+eyeA, 180-eyeA, color, thickness, CV_AA);

// Draw the bottom lip of the mouth.
int mouthY = faceH * 48/100;
int mouthW = faceW * 45/100;
int mouthH = faceH * 6/100;
ellipse(faceOutline, Point(sw/2, sh/2 + mouthY), Size(mouthW,
 mouthH), 0, 0, 180, color, thickness, CV_AA);

// Draw anti-aliased text.
int fontFace = FONT_HERSHEY_COMPLEX;
float fontScale = 1.0f;
int fontThickness = 2;
char *szMsg = "Put your face here";
putText(faceOutline, szMsg, Point(sw * 23/100, sh * 10/100),
 fontFace, fontScale, color, fontThickness, CV_AA);

addWeighted(dst, 1.0, faceOutline, 0.7, 0, dst, CV_8UC3);

Mat yuv = Mat(smallSize, CV_8UC3);
cvtColor(smallImg, yuv, CV_BGR2YCrCb);

int sw = smallSize.width;
int sh = smallSize.height;
Mat mask, maskPlusBorder;
maskPlusBorder = Mat::zeros(sh+2, sw+2, CV_8UC1);
mask = maskPlusBorder(Rect(1,1,sw,sh)); // mask is in maskPlusBorder.
resize(edge, mask, smallSize);         // Put edges in both of them.

const int EDGES_THRESHOLD = 80;
threshold(mask, mask, EDGES_THRESHOLD, 255, THRESH_BINARY);
dilate(mask, mask, Mat());
erode(mask, mask, Mat());

int const NUM_SKIN_POINTS = 6;
Point skinPts[NUM_SKIN_POINTS];
skinPts[0] = Point(sw/2,          sh/2 - sh/6);
skinPts[1] = Point(sw/2 - sw/11,  sh/2 - sh/6);
skinPts[2] = Point(sw/2 + sw/11,  sh/2 - sh/6);
skinPts[3] = Point(sw/2,          sh/2 + sh/16);
skinPts[4] = Point(sw/2 - sw/9,   sh/2 + sh/16);
skinPts[5] = Point(sw/2 + sw/9,   sh/2 + sh/16);

const int LOWER_Y = 60;
const int UPPER_Y = 80;
const int LOWER_Cr = 25;
const int UPPER_Cr = 15;
const int LOWER_Cb = 20;
const int UPPER_Cb = 15;
Scalar lowerDiff = Scalar(LOWER_Y, LOWER_Cr, LOWER_Cb);
Scalar upperDiff = Scalar(UPPER_Y, UPPER_Cr, UPPER_Cb);

const int CONNECTED_COMPONENTS = 4;  // To fill diagonally, use 8\. const int flags = CONNECTED_COMPONENTS | FLOODFILL_FIXED_RANGE \
     | FLOODFILL_MASK_ONLY;
Mat edgeMask = mask.clone();    // Keep a copy of the edge mask.
// "maskPlusBorder" is initialized with edges to block floodFill().
for (int i=0; i< NUM_SKIN_POINTS; i++) {
 floodFill(yuv, maskPlusBorder, skinPts[i], Scalar(), NULL,
 lowerDiff, upperDiff, flags);
}

mask -= edgeMask;

int Red = 0;
int Green = 70;
int Blue = 0;
add(smallImgBGR, CV_RGB(Red, Green, Blue), smallImgBGR, mask);

JNIEXPORT <Return> JNICALL Java_<Package>_<Class>_<Function>(JNIEnv* env, jobject, <Args>)

// Just show the plain camera image without modifying it.
JNIEXPORT void JNICALL Java_com_Cartoonifier_CartoonifierView_ShowPreview(
  JNIEnv* env, jobject,
  jint width, jint height, jbyteArray yuv, jintArray bgra)
{
  jbyte* _yuv  = env->GetByteArrayElements(yuv, 0);
  jint*  _bgra = env->GetIntArrayElements(bgra, 0);
  Mat myuv = Mat(height + height/2, width, CV_8UC1, (uchar *)_yuv);
  Mat mbgra = Mat(height, width, CV_8UC4, (uchar *)_bgra);
  // Convert the color format from the camera's
  // NV21 "YUV420sp" format to an Android BGRA color image.
 cvtColor(myuv, mbgra, CV_YUV420sp2BGRA);
  // OpenCV can now access/modify the BGRA image "mbgra" ...
  env->ReleaseIntArrayElements(bgra, _bgra, 0);
  env->ReleaseByteArrayElements(yuv, _yuv, 0);
}

cvtColor(myuv, mbgra, CV_YUV420sp2BGR, 4);

cvtColor(myuv, mbgra, CV_YUV420sp2BGRA);

public native void ShowPreview(int width, int height,
byte[] yuv, int[] rgba);

ShowPreview(getFrameWidth(), getFrameHeight(), data, rgba);

LOCAL_SRC_FILES += ../../Cartoonifier_Desktop/cartoon.cpp
LOCAL_SRC_FILES += ../../Cartoonifier_Desktop/ImageUtils_0.7.cpp

LOCAL_C_INCLUDES += $(LOCAL_PATH)/../../Cartoonifier_Desktop

#include "cartoon.h"    // Cartoonifier.
#include "ImageUtils.h"    // (Optional) OpenCV debugging // functions.

// Modify the camera image using the Cartoonifier filter.
JNIEXPORT void JNICALL Java_com_Cartoonifier_CartoonifierView_CartoonifyImage(
    JNIEnv* env, jobject,
   jint width, jint height, jbyteArray yuv, jintArray bgra)
   {
    // Get native access to the given Java arrays.
    jbyte* _yuv  = env->GetByteArrayElements(yuv, 0);
    jint* _bgra = env->GetIntArrayElements(bgra, 0);
    // Create OpenCV wrappers around the input & output data.
    Mat myuv(height + height/2, width, CV_8UC1, (uchar *)_yuv);
    Mat mbgra(height, width, CV_8UC4, (uchar *)_bgra);
    // Convert the color format from the camera's YUV420sp // semi-planar
    // format to OpenCV's default BGR color image.
    Mat mbgr(height, width, CV_8UC3);  // Allocate a new image buffer.
    cvtColor(myuv, mbgr, CV_YUV420sp2BGR);
    // OpenCV can now access/modify the BGR image "mbgr", and should
    // store the output as the BGR image "displayedFrame".
    Mat displayedFrame(mbgr.size(), CV_8UC3);
    // TEMPORARY: Just show the camera image without modifying it.
    displayedFrame = mbgr;
      // Convert the output from OpenCV's BGR to Android's BGRA //format.
 cvtColor(displayedFrame, mbgra, CV_BGR2BGRA);
  // Release the native lock we placed on the Java arrays.
  env->ReleaseIntArrayElements(bgra, _bgra, 0);
  env->ReleaseByteArrayElements(yuv, _yuv, 0);
}

cartoonifyImage(mbgr, displayedFrame);

import android.view.View;
import android.view.View.OnTouchListener;
import android.view.MotionEvent;

public class CartoonifierApp
extends Activity implements OnTouchListener {

// Call our "onTouch()" callback function whenever the user // touches the screen.
mView.setOnTouchListener(this);

public boolean onTouch(View v, MotionEvent m) {
    // Ignore finger movement event, we just care about when the 
    // finger first touches the screen.
    if (m.getAction() != MotionEvent.ACTION_DOWN) {
        return false; // We didn't use this touch movement event.
    }
    Log.i(TAG, "onTouch down event");
    // Signal that we should cartoonify the next camera frame and save 
    // it, instead of just showing the preview.
    mView.nextFrameShouldBeSaved(getBaseContext());
    return true;
}

// Cartoonify the next camera frame & save it instead of preview.
protected void nextFrameShouldBeSaved(Context context) {
    bSaveThisFrame = true;
}

private boolean bSaveThisFrame = false;
private boolean bFreezeOutput = false;
private static final int FREEZE_OUTPUT_MSECS = 3000;

@Override
protected Bitmap processFrame(byte[] data) {
    // Store the output image to the RGBA member variable.
    int[] rgba = mRGBA;
    // Only process the camera or update the screen if we aren't 
    // supposed to just show the cartoon image.
    if (bFreezeOutputbFreezeOutput) {
        // Only needs to be triggered here once.
        bFreezeOutput = false;
        // Wait for several seconds, doing nothing!
        try {
            wait(FREEZE_OUTPUT_MSECS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return null;
    }
    if (!bSaveThisFrame) {
        ShowPreview(getFrameWidth(), getFrameHeight(), data, rgba);
    }
    else {
        // Just do it once, then go back to preview mode.
        bSaveThisFrame = false;
        // Don't update the screen for a while, so the user can // see the cartoonifier output.
bFreezeOutput = true;
CartoonifyImage(getFrameWidth(), getFrameHeight(), data,
        rgba, m_sketchMode, m_alienMode, m_evilMode,
        m_debugMode);
}
// Put the processed image into the Bitmap object that will be // returned for display on the screen.
Bitmap bmp = mBitmap;
bmp.setPixels(rgba, 0, getFrameWidth(), 0, 0, getFrameWidth(),
        getFrameHeight());
return bmp;
}

if (bFreezeOutput) {
// Get the current date & time
SimpleDateFormat s = new SimpleDateFormat("yyyy-MM-dd,HH-mm-ss");
String timestamp = s.format(new Date());
String baseFilename = "Cartoon" + timestamp + ".png";
// Save the processed image as a PNG file on the SD card and show // it in the Android Gallery.
savePNGImageToGallery(bmp, mContext, baseFilename);
}

// For saving Bitmaps to file and the Android picture gallery.
import android.graphics.Bitmap.CompressFormat;
import android.net.Uri;
import android.os.Environment;
import android.provider.MediaStore;
import android.provider.MediaStore.Images;
import android.text.format.DateFormat;
import android.util.Log;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;

private static final String TAG = "CartoonifierView";
private Context mContext;  // So we can access the Android // Gallery.

mContext = context;  // Save the Android context, for GUI // access.

// Save the processed image as a PNG file on the SD card
// and shown in the Android Gallery.  
protected void savePNGImageToGallery(Bitmap bmp, Context context,
        String baseFilename)
{
    try {
    // Get the file path to the SD card.
    String baseFolder = \
    Environment.getExternalStoragePublicDirectory( \
    Environment.DIRECTORY_PICTURES).getAbsolutePath() \
    + "/";
    File file = new File(baseFolder + baseFilename);
    Log.i(TAG, "Saving the processed image to file [" + \
    file.getAbsolutePath() + "]");
    // Open the file.
    OutputStream out = new BufferedOutputStream(
    new FileOutputStream(file));
    // Save the image file as PNG.
    bmp.compress(CompressFormat.PNG, 100, out);
    // Make sure it is saved to file soon, because we are about
    // to add it to the Gallery.
    out.flush();
    out.close();
    // Add the PNG file to the Android Gallery.
    ContentValues image = new ContentValues();
    image.put(Images.Media.TITLE, baseFilename);
    image.put(Images.Media.DISPLAY_NAME, baseFilename);
    image.put(Images.Media.DESCRIPTION,
    "Processed by the Cartoonifier App");
    image.put(Images.Media.DATE_TAKEN,
    System.currentTimeMillis()); // msecs since 1970 UTC.
    image.put(Images.Media.MIME_TYPE, "https://gitcode.net/apachecn/apachecn-cv-zh/-/raw/master/docs/master-opencv-prac-cv-proj/img/png");
    image.put(Images.Media.ORIENTATION, 0);
    image.put(Images.Media.DATA, file.getAbsolutePath());
    Uri result = context.getContentResolver().insert(
    MediaStore.Images.Media.EXTERNAL_CONTENT_URI,image);
    }
catch (Exception e) {
    e.printStackTrace();
    }
}

<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>

// For showing a Notification message when saving a file.
import android.app.Notification;
import android.app.NotificationManager;
import android.app.PendingIntent;
import android.content.ContentValues;
import android.content.Intent;

private int mNotificationID = 0;
// To show just 1 notification.

showNotificationMessage(mContext, baseFilename);

// Show a notification message, saying we've saved another image.
protected void showNotificationMessage(Context context,
    String filename)
{
// Popup a notification message in the Android status   
// bar. To make sure a notification is shown for each 
// image but only 1 is kept in the status bar at a time, // use a different ID each time
// but delete previous messages before creating it.
final NotificationManager mgr = (NotificationManager) \
context.getSystemService(Context.NOTIFICATION_SERVICE);
// Close the previous popup message, so we only have 1 //at a time, but it still shows a popup message for each //one.
if (mNotificationID > 0)
  mgr.cancel(mNotificationID);
mNotificationID++;
Notification notification = new Notification(R.drawable.icon,
"Saving to gallery (image " + mNotificationID + ") ...",
System.currentTimeMillis());
Intent intent = new Intent(context, CartoonifierView.class);
// Close it if the user clicks on it.
notification.flags |= Notification.FLAG_AUTO_CANCEL;
PendingIntent pendingIntent = PendingIntent.getActivity(context,
0, intent, 0);
notification.setLatestEventInfo(context, "Cartoonifier saved " +
mNotificationID + " images to Gallery", "Saved as '" +
filename + "'", pendingIntent);
mgr.notify(mNotificationID, notification);
}

import android.view.Menu;
import android.view.MenuItem;

// Items for the Android menu bar.
private MenuItem mMenuAlien;
private MenuItem mMenuEvil;
private MenuItem mMenuSketch;
private MenuItem mMenuDebug;

/** Called when the menu bar is being created by Android. */
public boolean onCreateOptionsMenu(Menu menu) {
Log.i(TAG, "onCreateOptionsMenu");
mMenuSketch = menu.add("Sketch or Painting");
mMenuAlien = menu.add("Alien or Human");
mMenuEvil = menu.add("Evil or Good");
mMenuDebug = menu.add("[Debug mode]");
return true;
}
/** Called whenever the user pressed a menu item in the menu bar. */
public boolean onOptionsItemSelected(MenuItem item) {
Log.i(TAG, "Menu Item selected: " + item);
if (item == mMenuSketch)
mView.toggleSketchMode();
else if (item == mMenuAlien)
mView.toggleAlienMode();
else if (item == mMenuEvil)
mView.toggleEvilMode();
else if (item == mMenuDebug)
mView.toggleDebugMode();
return true;
}

private boolean m_sketchMode = false;
private boolean m_alienMode = false;
private boolean m_evilMode = false;
private boolean m_debugMode = false;

protected void toggleSketchMode() {
m_sketchMode = !m_sketchMode;
}
protected void toggleAlienMode() {
m_alienMode = !m_alienMode;
}
protected void toggleEvilMode() {
m_evilMode = !m_evilMode;
}
protected void toggleDebugMode() {
m_debugMode = !m_debugMode;
}

public native void CartoonifyImage(int width, int height,byte[] yuv,
int[] rgba, boolean sketchMode, boolean alienMode,
boolean evilMode, boolean debugMode);

CartoonifyImage(getFrameWidth(), getFrameHeight(), data,rgba,
m_sketchMode, m_alienMode, m_evilMode, m_debugMode);

JNIEXPORT void JNICALL Java_com_Cartoonifier_CartoonifierView_CartoonifyImage(
  JNIEnv* env, jobject, jint width, jint height,
  jbyteArray yuv, jintArray bgra, jboolean sketchMode,
  jboolean alienMode, jboolean evilMode, jboolean debugMode)

int debugType = 0;
if (debugMode)
  debugType = 1;
cartoonifyImage(mbgr, displayedFrame, sketchMode, alienMode, evilMode, debugType);

```cpp
void cartoonifyImage(Mat srcColor, Mat dst, bool sketchMode,
bool alienMode, bool evilMode, int debugType)
{
```

```cpp
void cartoonifyImage(Mat srcColor, Mat dst, bool sketchMode,
bool alienMode, bool evilMode, int debugType);
```

void removePepperNoise(Mat &mask)
{
for (int y=2; y<mask.rows-2; y++) {
  // Get access to each of the 5 rows near this pixel.
  uchar *pUp2 = mask.ptr(y-2);
  uchar *pUp1 = mask.ptr(y-1);
  uchar *pThis = mask.ptr(y);
  uchar *pDown1 = mask.ptr(y+1);
  uchar *pDown2 = mask.ptr(y+2);
  // Skip the first (and last) 2 pixels on each row.
  pThis += 2;
  pUp1 += 2;
  pUp2 += 2;
  pDown1 += 2;
  pDown2 += 2;
  for (int x=2; x<mask.cols-2; x++) {
    uchar value = *pThis;  // Get this pixel value (0 or 255).
    // Check if this is a black pixel that is surrounded by
    // white pixels (ie: whether it is an "island" of black).
 if (value == 0) {
    bool above, left, below, right, surroundings;
    above = *(pUp2 - 2) && *(pUp2 - 1) && *(pUp2) &&
    *(pUp2 + 1) && *(pUp2 + 2);
    left = *(pUp1 - 2) && *(pThis - 2) && *(pDown1 - 2);
    below = *(pDown2 - 2) && *(pDown2 - 1) && *(pDown2) &&
    *(pDown2 + 1) && *(pDown2 + 2);
    right = *(pUp1 + 2) && *(pThis + 2) && *(pDown1 + 2);
    surroundings = above && left && below && right;
 if (surroundings == true) {
      // Fill the whole 5x5 block as white. Since we know
      // the 5x5 borders are already white, we just need to
      // fill the 3x3 inner region.
      *(pUp1 - 1) = 255;
      *(pUp1 + 0) = 255;
      *(pUp1 + 1) = 255;
      *(pThis - 1) = 255;
      *(pThis + 0) = 255;
      *(pThis + 1) = 255;
      *(pDown1 - 1) = 255;
      *(pDown1 + 0) = 255;
      *(pDown1 + 1) = 255;
      // Since we just covered the whole 5x5 block with
      // white, we know the next 2 pixels won't be black,
     // so skip the next 2 pixels on the right.
      pThis += 2;
      pUp1 += 2;
      pUp2 += 2;
      pDown1 += 2;
      pDown2 += 2;
      }
    }
    // Move to the next pixel on the right.
    pThis++;
    pUp1++;
    pUp2++;
    pDown1++;
    pDown2++;
    }
  }
}

private FpsMeter  mFps;

mFps = new FpsMeter();
mFps.init();

mFps.measure();

mFps.draw(canvas, (canvas.getWidth() - bmp.getWidth()) /2, 0);

public void surfaceChanged(SurfaceHolder _holder, int format,
  int width, int height) {
  Log.i(TAG, "Screen size: " + width + "x" + height);
  // Use a camera resolution of roughly half the screen height.
 setupCamera(width/2, height/2);
}

DECLARE_TIMING(myFilter);
void myImageFunction(Mat img) {
  printMatInfo(img, "input");
  START_TIMING(myFilter);
  bilateralFilter(img, …);
  STOP_TIMING(myFilter);
  SHOW_TIMING(myFilter, "My Filter");
}

input: 800w600h 3ch 8bpp, range[19,255][17,243][47,251]
My Filter: time:  213ms   (ave=215ms min=197ms max=312ms, across 57 runs).

//
// Prefix header for all source files of the 'Example_MarkerBasedAR'
//
#import <Availability.h>
#ifndef __IPHONE_5_0
#warning "This project uses features only available in iOS SDK 5.0 and later."
#endif
#ifdef __cplusplus
#include <opencv2/opencv.hpp>
#endif
#ifdef __OBJC__
  #import <UIKit/UIKit.h>
  #import <Foundation/Foundation.h>
#endif

@protocol VideoSourceDelegate<NSObject>
-(void)frameReady:(BGRAVideoFrame) frame;
@end
@interface VideoSource : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate>
{
}
@property (nonatomic, retain) AVCaptureSession *captureSession;
@property (nonatomic, retain) AVCaptureDeviceInput *deviceInput;
@property (nonatomic, retain) id<VideoSourceDelegate> delegate;
- (bool) startWithDevicePosition:(AVCaptureDevicePosition)devicePosition;
- (CameraCalibration) getCalibration;
- (CGSize) getFrameSize;
@end

@protocol VideoSourceDelegate<NSObject>
-(void)frameReady:(BGRAVideoFrame) frame;
@end

- (id)init
{
  if ((self = [super init]))
  {
    AVCaptureSession * capSession = [[AVCaptureSession alloc] init];
    if ([capSession canSetSessionPreset:AVCaptureSessionPreset640x480])
    {
      [capSession setSessionPreset:AVCaptureSessionPreset640x480];
      NSLog(@"Set capture session preset AVCaptureSessionPreset640x480");
    }
    else if ([capSession canSetSessionPreset:AVCaptureSessionPresetLow])
    {
      [capSession setSessionPreset:AVCaptureSessionPresetLow];
      NSLog(@"Set capture session preset AVCaptureSessionPresetLow");
    }
    self.captureSession = capSession;
  }
  return self;
}

- (bool) startWithDevicePosition:(AVCaptureDevicePosition)devicePosition
{
  AVCaptureDevice *videoDevice = [self cameraWithPosition:devicePosition];
  if (!videoDevice)
    return FALSE;
  {
    NSError *error;
    AVCaptureDeviceInput *videoIn = [AVCaptureDeviceInput 
    deviceInputWithDevice:videoDevice error:&error];
    self.deviceInput = videoIn;
    if (!error)
    {
      if ([[self captureSession] canAddInput:videoIn])
      {
        [[self captureSession] addInput:videoIn];
      }
      else
      {
        NSLog(@"Couldn't add video input");
        return FALSE;
      }
    }
    else
    {
      NSLog(@"Couldn't create video input");
      return FALSE;
    }
  }
  [self addRawViewOutput];
  [captureSession startRunning];
  return TRUE;
}

- (void) addRawViewOutput
{
  /*We setupt the output*/
  AVCaptureVideoDataOutput *captureOutput = [[AVCaptureVideoDataOutput alloc] init];
  /*While a frame is processes in -captureOutput:didOutputSampleBuffer:fromConnection: delegate methods no other frames are added in the queue.
    If you don't want this behaviour set the property to NO */
  captureOutput.alwaysDiscardsLateVideoFrames = YES; 
  /*We create a serial queue to handle the processing of our frames*/
  dispatch_queue_t queue;
  queue = dispatch_queue_create("com.Example_MarkerBasedAR.cameraQueue", 
  NULL);
  [captureOutput setSampleBufferDelegate:self queue:queue];
  dispatch_release(queue);
  // Set the video output to store frame in BGRA (It is supposed to be faster)
  NSString* key = (NSString*)kCVPixelBufferPixelFormatTypeKey;
  NSNumber* value = [NSNumber 
  numberWithUnsignedInt:kCVPixelFormatType_32BGRA];
  NSDictionary* videoSettings = [NSDictionary dictionaryWithObject:value 
  forKey:key];
  [captureOutput setVideoSettings:videoSettings];
  // Register an output
  [self.captureSession addOutput:captureOutput];
}

- (void)captureOutput:(AVCaptureOutput *)captureOutput 
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer 
       fromConnection:(AVCaptureConnection *)connection 
{ 
  // Get a image buffer holding video frame
  CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
  // Lock the image buffer
  CVPixelBufferLockBaseAddress(imageBuffer,0); 
  // Get information about the image
  uint8_t *baseAddress = (uint8_t *)CVPixelBufferGetBaseAddress(imageBuffer); 
  size_t width = CVPixelBufferGetWidth(imageBuffer); 
  size_t height = CVPixelBufferGetHeight(imageBuffer);  
  size_t stride = CVPixelBufferGetBytesPerRow(imageBuffer);
  BGRAVideoFrame frame = {width, height, stride, baseAddress};
  [delegate frameReady:frame];
  /*We unlock the  image buffer*/
  CVPixelBufferUnlockBaseAddress(imageBuffer,0);
}

/**
 * A top-level class that encapsulate marker detector algorithm
 */
class MarkerDetector
{
public:
  /**
   * Initialize a new instance of marker detector object
   * @calibration[in] - Camera calibration necessary for pose estimation.
   */
  MarkerDetector(CameraCalibration calibration);
  void processFrame(const BGRAVideoFrame& frame);
  const std::vector<Transformation>& getTransformations() const;
  protected:
  bool findMarkers(const BGRAVideoFrame& frame, std::vector<Marker>& 
  detectedMarkers);
  void prepareImage(const cv::Mat& bgraMat,
                    cv::Mat& grayscale);
  void performThreshold(const cv::Mat& grayscale,
                        cv::Mat& thresholdImg);
  void findContours(const cv::Mat& thresholdImg, 
                    std::vector<std::vector<cv::Point> >& contours,
                    int minContourPointsAllowed);
  void findMarkerCandidates(const std::vector<std::vector<cv::Point> >& 
  contours, std::vector<Marker>& detectedMarkers);
  void detectMarkers(const cv::Mat& grayscale,
                     std::vector<Marker>& detectedMarkers);
  void estimatePosition(std::vector<Marker>& detectedMarkers);
private:
};

void MarkerDetector::prepareImage(const cv::Mat& bgraMat, cv::Mat& grayscale)
{
  // Convert to grayscale
  cv::cvtColor(bgraMat, grayscale, CV_BGRA2GRAY);
}

void MarkerDetector::performThreshold(const cv::Mat& grayscale, cv::Mat& thresholdImg)
{
  cv::adaptiveThreshold(grayscale,   // Input image
                        thresholdImg,// Result binary image
                        255,         //
                        cv::ADAPTIVE_THRESH_GAUSSIAN_C, //
                        cv::THRESH_BINARY_INV, //
                        7, //
                        7  //
                        );
}

void MarkerDetector::findContours(const cv::Mat& thresholdImg,
                                  std::vector<std::vector<cv::Point> >& contours,
int minContourPointsAllowed)
{
    std::vector< std::vector<cv::Point> > allContours;
    cv::findContours(thresholdImg, allContours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE);
    contours.clear();
    for (size_t i=0; i<allContours.size(); i++)
    {
        int contourSize = allContours[i].size();
        if (contourSize > minContourPointsAllowed)
        {
            contours.push_back(allContours[i]);
        }
    }
}

void MarkerDetector::findCandidates
(
    const ContoursVector& contours,
    std::vector<Marker>& detectedMarkers
)
{
    std::vector<cv::Point>  approxCurve;
    std::vector<Marker>     possibleMarkers;
    // For each contour, analyze if it is a parallelepiped likely to be the 
    marker
    for (size_t i=0; i<contours.size(); i++)
    {
        // Approximate to a polygon
        double eps = contours[i].size() * 0.05;
        cv::approxPolyDP(contours[i], approxCurve, eps, true);
        // We interested only in polygons that contains only four points
        if (approxCurve.size() != 4)
            continue;
        // And they have to be convex
        if (!cv::isContourConvex(approxCurve))
            continue;
        // Ensure that the distance between consecutive points is large enough
        float minDist = std::numeric_limits<float>::max();
        for (int i = 0; i < 4; i++)
        {
            cv::Point side = approxCurve[i] - approxCurve[(i+1)%4];
            float squaredSideLength = side.dot(side);
            minDist = std::min(minDist, squaredSideLength);
        }
        // Check that distance is not very small
        if (minDist < m_minContourLengthAllowed)
            continue;
        // All tests are passed. Save marker candidate:
        Marker m;
        for (int i = 0; i<4; i++)
            m.points.push_back( cv::Point2f(approxCurve[i].x,approxCurve[i].y) );
        // Sort the points in anti-clockwise order
        // Trace a line between the first and second point.
        // If the third point is at the right side, then the points are anti-
        clockwise
        cv::Point v1 = m.points[1] - m.points[0];
        cv::Point v2 = m.points[2] - m.points[0];
        double o = (v1.x * v2.y) - (v1.y * v2.x);
        if (o < 0.0)         //if the third point is in the left side, then 
        sort in anti-clockwise order
            std::swap(m.points[1], m.points[3]);
        possibleMarkers.push_back(m);
    }
    // Remove these elements which corners are too close to each other.
    // First detect candidates for removal:
    std::vector< std::pair<int,int> > tooNearCandidates;
    for (size_t i=0;i<possibleMarkers.size();i++)
    { 
        const Marker& m1 = possibleMarkers[i];
        //calculate the average distance of each corner to the nearest corner 
        of the other marker candidate
        for (size_t j=i+1;j<possibleMarkers.size();j++)
        {
            const Marker& m2 = possibleMarkers[j];
            float distSquared = 0;
            for (int c = 0; c < 4; c++)
            {
                cv::Point v = m1.points[c] - m2.points[c];
                distSquared += v.dot(v);
            }
            distSquared /= 4;
            if (distSquared < 100)
            {
                tooNearCandidates.push_back(std::pair<int,int>(i,j));
            }
        }                
    }
    // Mark for removal the element of the pair with smaller perimeter
    std::vector<bool> removalMask (possibleMarkers.size(), false);
    for (size_t i=0; i<tooNearCandidates.size(); i++)
    {
        float p1 = perimeter(possibleMarkers[tooNearCandidates[i].first 
        ].points);
        float p2 = 
        perimeter(possibleMarkers[tooNearCandidates[i].second].points);
        size_t removalIndex;
        if (p1 > p2)
            removalIndex = tooNearCandidates[i].second;
        else
            removalIndex = tooNearCandidates[i].first;
        removalMask[removalIndex] = true;
    }
    // Return candidates
    detectedMarkers.clear();
    for (size_t i=0;i<possibleMarkers.size();i++)
    {
        if (!removalMask[i])
            detectedMarkers.push_back(possibleMarkers[i]);
    }
}

cv::Mat canonicalMarker;
Marker& marker = detectedMarkers[i];
// Find the perspective transfomation that brings current marker to rectangular form
cv::Mat M = cv::getPerspectiveTransform(marker.points, m_markerCorners2d);
// Transform image to get a canonical marker image
cv::warpPerspective(grayscale, canonicalMarker,  M, markerSize);

//threshold image
cv::threshold(markerImage, markerImage, 125, 255, cv::THRESH_BINARY | cv::THRESH_OTSU);

cv::Mat src(400,400,CV_8UC1);
cv::Rect r(10,10,50,50);
cv::Mat subView = src(r);

cv::Mat bitMatrix = cv::Mat::zeros(5,5,CV_8UC1);
  //get information(for each inner square, determine if it is  black or white)
  for (int y=0;y<5;y++)
  {
    for (int x=0;x<5;x++)
    {
      int cellX = (x+1)*cellSize;
      int cellY = (y+1)*cellSize;
      cv::Mat cell = grey(cv::Rect(cellX,cellY,cellSize,cellSize));
      int nZ = cv::countNonZero(cell);
      if (nZ> (cellSize*cellSize) /2)
        bitMatrix.at<uchar>(y,x) = 1;
    }
  }

//check all possible rotations
  cv::Mat rotations[4];
  int distances[4];
  rotations[0] = bitMatrix;  
  distances[0] = hammDistMarker(rotations[0]);
  std::pair<int,int> minDist(distances[0],0);
  for (int i=1; i<4; i++)
  {
    //get the hamming distance to the nearest possible word
    rotations[i] = rotate(rotations[i-1]);
    distances[i] = hammDistMarker(rotations[i]);
    if (distances[i] < minDist.first)
    {
      minDist.first  = distances[i];
      minDist.second = i;
    }
  }

//sort the points so that they are always in the same order
// no matter the camera orientation
std::rotate(marker.points.begin(), marker.points.begin() + 4 - nRotations,
marker.points.end());

std::vector<cv::Point2f> preciseCorners(4 * goodMarkers.size());
for (size_t i=0; i<goodMarkers.size(); i++)
{
  Marker& marker = goodMarkers[i];
  for (int c=0;c<4;c++)
  {
    preciseCorners[i*4+c] = marker.points[c];
  }
}
cv::cornerSubPix(grayscale, preciseCorners, cvSize(5,5), cvSize(-1,-1), cvTermCriteria(CV_TERMCRIT_ITER,30,0.1));
//copy back
for (size_t i=0;i<goodMarkers.size();i++)
{
  Marker&marker = goodMarkers[i];
  for (int c=0;c<4;c++)
  {
    marker.points[c] = preciseCorners[i*4+c];
  }
}

P = A * [R | T] * M

/**
 * A camera calibration class that stores intrinsic matrix and distorsion coefficients.
 */
class CameraCalibration
{
  public:
  CameraCalibration();
  CameraCalibration(float fx, float fy, float cx, float cy);
  CameraCalibration(float fx, float fy, float cx, float cy, float 
  distorsionCoeff[4]);
  void getMatrix34(float cparam[3][4]) const;
  const Matrix33& getIntrinsic() const;
  const Vector4&  getDistorsion() const;
  private:
  Matrix33 m_intrinsic;
  Vector4  m_distorsion;
};

void solvePnP(const Mat& objectPoints, const Mat& imagePoints, const Mat& 
cameraMatrix, const Mat& distCoeffs, Mat& rvec, Mat& tvec, bool 
useExtrinsicGuess=false);

T(v) = Rv + t

It(R) = 1

void MarkerDetector::estimatePosition(std::vector<Marker>& detectedMarkers)
{
  for (size_t i=0; i<detectedMarkers.size(); i++)
  { 
    Marker& m = detectedMarkers[i];
    cv::Mat Rvec;
    cv::Mat_<float> Tvec;
    cv::Mat raux,taux;
    cv::solvePnP(m_markerCorners3d, m.points, camMatrix, distCoeff,raux,taux);
    raux.convertTo(Rvec,CV_32F);
    taux.convertTo(Tvec ,CV_32F);
    cv::Mat_<float> rotMat(3,3); 
    cv::Rodrigues(Rvec, rotMat);
    // Copy to transformation matrix
    m.transformation = Transformation();
    for (int col=0; col<3; col++)
    {
      for (int row=0; row<3; row++)
      {
        m.transformation.r().mat[row][col] = rotMat(row,col); // Copy rotation 
        component
      }
      m.transformation.t().data[col] = Tvec(col); // Copy translation 
      component
    }
    // Since solvePnP finds camera location, w.r.t to marker pose, to get 
    marker pose w.r.t to the camera we invert it.
    m.transformation = m.transformation.getInverted();
  }

@class EAGLContext;
// This class wraps the CAEAGLLayer from CoreAnimation into a convenient UIView subclass.
// The view content is basically an EAGL surface you render your OpenGL scene into.
// Note that setting the view non-opaque will only work if the EAGL surface has an alpha channel.
@interface EAGLView : UIView
{
@private
    // The OpenGL ES names for the framebuffer and renderbuffer used to render 
    to this view.
    GLuint defaultFramebuffer, colorRenderbuffer;
}
@property (nonatomic, retain) EAGLContext *context;
// The pixel dimensions of the CAEAGLLayer.
@property (readonly) GLint framebufferWidth;
@property (readonly) GLint framebufferHeight;
- (void)setFramebuffer;
- (BOOL)presentFramebuffer;
- (void)initContext;
@end

//The EAGL view is stored in the nib file. When it's unarchived it's sent -initWithCoder:.
- (id)initWithCoder:(NSCoder*)coder
{
  self = [super initWithCoder:coder];
  if (self) {
    CAEAGLLayer *eaglLayer = (CAEAGLLayer *)self.layer;
    eaglLayer.opaque = TRUE;
    eaglLayer.drawableProperties = [NSDictionary dictionaryWithObjectsAndKeys:
                                   [NSNumber numberWithBool:FALSE], 
                                   kEAGLDrawablePropertyRetainedBacking,
                                   kEAGLColorFormatRGBA8, 
                                   kEAGLDrawablePropertyColorFormat,
                                   nil];
    [self initContext];
  }
  return self;
}
- (void)createFramebuffer
{
  if (context && !defaultFramebuffer) {
    [EAGLContext setCurrentContext:context];
    // Create default framebuffer object.
    glGenFramebuffers(1, &defaultFramebuffer);
    glBindFramebuffer(GL_FRAMEBUFFER, defaultFramebuffer);
    // Create color render buffer and allocate backing store.
    glGenRenderbuffers(1, &colorRenderbuffer);
    glBindRenderbuffer(GL_RENDERBUFFER, colorRenderbuffer);
    [context renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer *)self.layer];
    glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &framebufferWidth);
    glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &framebufferHeight);
    glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, 
    GL_RENDERBUFFER, colorRenderbuffer);
    if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
    NSLog(@"Failed to make complete framebuffer object %x", 
    glCheckFramebufferStatus(GL_FRAMEBUFFER));
    //glClearColor(0, 0, 0, 0);
    NSLog(@"Framebuffer created");
  }
}

@interface SimpleVisualizationController : NSObject<VisualizationController>
{
  EAGLView * m_glview;
  GLuint m_backgroundTextureId;
  std::vector<Transformation> m_transformations;
  CameraCalibration m_calibration;
  CGSize m_frameSize;
}
-(id) initWithGLView:(EAGLView*)view calibration:(CameraCalibration) calibration frameSize:(CGSize) size;
-(void) drawFrame;
-(void) updateBackground:(BGRAVideoFrame) frame;
-(void) setTransformationList:(const std::vector<Transformation>&) transformations;

- (void)drawFrame
{
  // Set the active framebuffer
  [m_glview setFramebuffer];  
  // Draw a video on the background
  [self drawBackground];
  // Draw 3D objects on the position of the detected markers
  [self drawAR];
  // Present framebuffer
  bool ok = [m_glview presentFramebuffer];
  int glErCode = glGetError();
  if (!ok || glErCode != GL_NO_ERROR)
  {
    std::cerr << "GL error detected. Error code:" << glErCode << std::endl;
  }
}

- (void) drawBackground
{
  GLfloat w = m_glview.bounds.size.width;
  GLfloat h = m_glview.bounds.size.height;
  const GLfloat squareVertices[] =
  {
    0, 0,
    w, 0,
    0, h,
    w, h
  };
  static const GLfloat textureVertices[] =
  {
    1, 0,
    1, 1,
    0, 0,
    0, 1
  };
  static const GLfloat proj[] =
  {
    0, -2.f/w, 0, 0,
    -2.f/h, 0, 0, 0,
    0, 0, 1, 0,
    1, 1, 0, 1
  };
  glMatrixMode(GL_PROJECTION);
  glLoadMatrixf(proj);
  glMatrixMode(GL_MODELVIEW);
  glLoadIdentity();
  glDisable(GL_COLOR_MATERIAL);
  glEnable(GL_TEXTURE_2D);
  glBindTexture(GL_TEXTURE_2D, m_backgroundTextureId);
  // Update attribute values.
  glVertexPointer(2, GL_FLOAT, 0, squareVertices);
  glEnableClientState(GL_VERTEX_ARRAY);
  glTexCoordPointer(2, GL_FLOAT, 0, textureVertices);
  glEnableClientState(GL_TEXTURE_COORD_ARRAY);
  glColor4f(1,1,1,1);
  glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
  glDisableClientState(GL_VERTEX_ARRAY);
  glDisableClientState(GL_TEXTURE_COORD_ARRAY);
  glDisable(GL_TEXTURE_2D);
}

- (void)buildProjectionMatrix:(Matrix33)cameraMatrix: (int)screen_width: (int)screen_height: (Matrix44&) projectionMatrix
{    
  float near = 0.01;  // Near clipping distance
  float far = 100;  // Far clipping distance
  // Camera parameters
  float f_x = cameraMatrix.data[0]; // Focal length in x axis
  float f_y = cameraMatrix.data[4]; // Focal length in y axis (usually the 
  same?)
  float c_x = cameraMatrix.data[2]; // Camera primary point x
  float c_y = cameraMatrix.data[5]; // Camera primary point y
  projectionMatrix.data[0] =  - 2.0 * f_x / screen_width;
  projectionMatrix.data[1] = 0.0;
  projectionMatrix.data[2] = 0.0;
  projectionMatrix.data[3] = 0.0;
  projectionMatrix.data[4] = 0.0;
  projectionMatrix.data[5] = 2.0 * f_y / screen_height;
  projectionMatrix.data[6] = 0.0;
  projectionMatrix.data[7] = 0.0;
  projectionMatrix.data[8] = 2.0 * c_x / screen_width - 1.0;
  projectionMatrix.data[9] = 2.0 * c_y / screen_height - 1.0;   
  projectionMatrix.data[10] = -( far+near ) / ( far - near );
  projectionMatrix.data[11] = -1.0;
  projectionMatrix.data[12] = 0.0;
  projectionMatrix.data[13] = 0.0;
  projectionMatrix.data[14] = -2.0 * far * near / ( far - near );
  projectionMatrix.data[15] = 0.0;
}

- (void) drawAR
{
  Matrix44 projectionMatrix;
  [self buildProjectionMatrix:m_calibration.getIntrinsic():m_frameSize.width
  :m_frameSize.height :projectionMatrix];
  glMatrixMode(GL_PROJECTION);
  glLoadMatrixf(projectionMatrix.data);
  glMatrixMode(GL_MODELVIEW);
  glLoadIdentity();
  glEnableClientState(GL_VERTEX_ARRAY);
  glEnableClientState(GL_NORMAL_ARRAY);
  glPushMatrix();
  glLineWidth(3.0f);
  float lineX[] = {0,0,0,1,0,0};
  float lineY[] = {0,0,0,0,1,0};
  float lineZ[] = {0,0,0,0,0,1};
  const GLfloat squareVertices[] = {
    -0.5f, -0.5f,
    0.5f,  -0.5f,
    -0.5f,  0.5f,
    0.5f,   0.5f,
  };
  const GLubyte squareColors[] = {
    255, 255,   0, 255,
    0,   255, 255, 255,
    0,     0,   0,   0,
    255,   0, 255, 255,
  };
  for (size_t transformationIndex=0; 
  transformationIndex<m_transformations.size(); transformationIndex++)
  {
    const Transformation& transformation = 
    m_transformations[transformationIndex];
    Matrix44 glMatrix = transformation.getInverted().getMat44();
    glLoadMatrixf(reinterpret_cast<const GLfloat*>(&glMatrix.data[0]));
    // draw data
    glVertexPointer(2, GL_FLOAT, 0, squareVertices);
    glEnableClientState(GL_VERTEX_ARRAY);
    glColorPointer(4, GL_UNSIGNED_BYTE, 0, squareColors);
    glEnableClientState(GL_COLOR_ARRAY);
    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
    glDisableClientState(GL_COLOR_ARRAY);
    float scale = 0.5;
    glScalef(scale, scale, scale);
    glColor4f(1.0f, 0.0f, 0.0f, 1.0f);
    glVertexPointer(3, GL_FLOAT, 0, lineX);
    glDrawArrays(GL_LINES, 0, 2);
    glColor4f(0.0f, 1.0f, 0.0f, 1.0f);
    glVertexPointer(3, GL_FLOAT, 0, lineY);
    glDrawArrays(GL_LINES, 0, 2);
    glColor4f(0.0f, 0.0f, 1.0f, 1.0f);
    glVertexPointer(3, GL_FLOAT, 0, lineZ);
    glDrawArrays(GL_LINES, 0, 2);
  }
  glPopMatrix();
  glDisableClientState(GL_VERTEX_ARRAY);
}

cv::Ptr<cv::FeatureDetector> detector = cv::Ptr<cv::FeatureDetector>(new cv::SurfFeatureDetector());

cv::Ptr<cv::FeatureDetector> detector = cv::FeatureDetector::create("SURF");

std::vector<cv::KeyPoint> keypoints;
detector->detect(image, keypoints);

/**
 * Store the image data and computed descriptors of target pattern
 */
struct Pattern
{
  cv::Size                  size;
  cv::Mat                   data;
  std::vector<cv::KeyPoint> keypoints;
  cv::Mat                   descriptors;
  std::vector<cv::Point2f>  points2d;
  std::vector<cv::Point3f>  points3d;
};

void PatternDetector::train(const Pattern& pattern)
{
    // Store the pattern object
    m_pattern = pattern;
    // API of cv::DescriptorMatcher is somewhat tricky
    // First we clear old train data:
    m_matcher->clear();
    // That we add vector of descriptors 
    // (each descriptors matrix describe one image). 
    // This allows us to perform search across multiple images:
    std::vector<cv::Mat> descriptors(1);
    descriptors[0] = pattern.descriptors.clone(); 
    m_matcher->add(descriptors);
    // After adding train data perform actual train:
    m_matcher->train();
}

void match(const Mat& queryDescriptors, vector<DMatch>& matches,
           const vector<Mat>& masks=vector<Mat>() );

void knnMatch(const Mat& queryDescriptors, vector<vector<DMatch> >& matches, int k, const vector<Mat>& masks=vector<Mat>(),bool compactResult=false );

void radiusMatch(const Mat& queryDescriptors, vector<vector<DMatch> >& matches, maxDistance, const vector<Mat>& masks=vector<Mat>(), bool compactResult=false );

cv::Ptr<cv::DescriptorMatcher> matcher(new cv::BFMatcher(cv::NORM_HAMMING, true));

void PatternDetector::getMatches(const cv::Mat& queryDescriptors, std::vector<cv::DMatch>& matches)
{
    matches.clear();
    if (enableRatioTest)
    {
        // To avoid NaNs when best match has 
        // zero distance we will use inverse ratio. 
        const float minRatio = 1.f / 1.5f;
        // KNN match will return 2 nearest 
        // matches for each query descriptor
        m_matcher->knnMatch(queryDescriptors, m_knnMatches, 2);
        for (size_t i=0; i<m_knnMatches.size(); i++)
        {
            const cv::DMatch& bestMatch   = m_knnMatches[i][0];
            const cv::DMatch& betterMatch = m_knnMatches[i][1];
            float distanceRatio = bestMatch.distance / betterMatch.distance;
            // Pass only matches where distance ratio between 
            // nearest matches is greater than 1.5 
            // (distinct criteria)
            if (distanceRatio < minRatio)
            {
                matches.push_back(bestMatch);
            }
        }
    }
    else
    {
        // Perform regular match
        m_matcher->match(queryDescriptors, matches);
    }
}

bool PatternDetector::refineMatchesWithHomography
    (
        const std::vector<cv::KeyPoint>& queryKeypoints,
        const std::vector<cv::KeyPoint>& trainKeypoints, 
        float reprojectionThreshold,
        std::vector<cv::DMatch>& matches,
        cv::Mat& homography
    )
{
    const int minNumberMatchesAllowed = 8;
    if (matches.size() < minNumberMatchesAllowed)
        return false;
    // Prepare data for cv::findHomography
    std::vector<cv::Point2f> srcPoints(matches.size());
    std::vector<cv::Point2f> dstPoints(matches.size());
    for (size_t i = 0; i < matches.size(); i++)
    {
        srcPoints[i] = trainKeypoints[matches[i].trainIdx].pt;
        dstPoints[i] = queryKeypoints[matches[i].queryIdx].pt;
    }
    // Find homography matrix and get inliers mask
    std::vector<unsigned char> inliersMask(srcPoints.size());
    homography = cv::findHomography(srcPoints, 
                                    dstPoints, 
                                    CV_FM_RANSAC, 
                                    reprojectionThreshold, 
                                    inliersMask);
    std::vector<cv::DMatch> inliers;
    for (size_t i=0; i<inliersMask.size(); i++)
    {
        if (inliersMask[i])
            inliers.push_back(matches[i]);
    }
    matches.swap(inliers);
    return matches.size() > minNumberMatchesAllowed;
}

bool PatternDetector::findPattern(const cv::Mat& image, PatternTrackingInfo& info)
{
    // Convert input image to gray
    getGray(image, m_grayImg);
    // Extract feature points from input gray image
    extractFeatures(m_grayImg, m_queryKeypoints, m_queryDescriptors);
    // Get matches with current pattern
    getMatches(m_queryDescriptors, m_matches);
    // Find homography transformation and detect good matches
    bool homographyFound = refineMatchesWithHomography(
        m_queryKeypoints, 
        m_pattern.keypoints, 
        homographyReprojectionThreshold, 
        m_matches, 
        m_roughHomography);
    if (homographyFound)
    {
        // If homography refinement enabled 
        // improve found transformation
        if (enableHomographyRefinement)
        {
            // Warp image using found homography
            cv::warpPerspective(m_grayImg, m_warpedImg, m_roughHomography, m_pattern.size, cv::WARP_INVERSE_MAP | cv::INTER_CUBIC);
            // Get refined matches:
            std::vector<cv::KeyPoint> warpedKeypoints;
            std::vector<cv::DMatch> refinedMatches;
            // Detect features on warped image
            extractFeatures(m_warpedImg, warpedKeypoints, m_queryDescriptors);
            // Match with pattern
            getMatches(m_queryDescriptors, refinedMatches);
            // Estimate new refinement homography
            homographyFound = refineMatchesWithHomography(
                warpedKeypoints, 
                m_pattern.keypoints, 
                homographyReprojectionThreshold, 
                refinedMatches, 
                m_refinedHomography);
            // Get a result homography as result of matrix product 
            // of refined and rough homographies:
            info.homography = m_roughHomography * m_refinedHomography;
            // Transform contour with precise homography
            cv::perspectiveTransform(m_pattern.points2d, info.points2d, info.homography);
        }
        else
        {
            info.homography = m_roughHomography;
            // Transform contour with rough homography
            cv::perspectiveTransform(m_pattern.points2d, info.points2d, m_roughHomography);
        }
    }
    return homographyFound;
}

void PatternDetector::computePatternFromImage(const cv::Mat& image, Pattern& pattern);

void PatternDetector::train(const Pattern& pattern)

void PatternDetector::buildPatternFromImage(const cv::Mat& image, Pattern& pattern) const
{
    int numImages = 4;
    float step = sqrtf(2.0f);
    // Store original image in pattern structure
    pattern.size = cv::Size(image.cols, image.rows);
    pattern.frame = image.clone();
    getGray(image, pattern.grayImg);
    // Build 2d and 3d contours (3d contour lie in XY plane since // it's planar)
    pattern.points2d.resize(4);
    pattern.points3d.resize(4);
    // Image dimensions
    const float w = image.cols;
    const float h = image.rows;
    // Normalized dimensions:
    const float maxSize = std::max(w,h);
    const float unitW = w / maxSize;
    const float unitH = h / maxSize;
    pattern.points2d[0] = cv::Point2f(0,0);
    pattern.points2d[1] = cv::Point2f(w,0);
    pattern.points2d[2] = cv::Point2f(w,h);
    pattern.points2d[3] = cv::Point2f(0,h);
    pattern.points3d[0] = cv::Point3f(-unitW, -unitH, 0);
    pattern.points3d[1] = cv::Point3f( unitW, -unitH, 0);
    pattern.points3d[2] = cv::Point3f( unitW,  unitH, 0);
    pattern.points3d[3] = cv::Point3f(-unitW,  unitH, 0);
    extractFeatures(pattern.grayImg, pattern.keypoints, pattern.descriptors);
}

imagelist_creator imagelist.yaml *.png
calibration -w 9 -h 6 -o camera_intrinsic.yaml imagelist.yaml

%YAML:1.0
calibration_time: "06/12/12 11:17:56"
image_width: 640
image_height: 480
board_width: 9
board_height: 6
square_size: 1.
flags: 0
camera_matrix: !!opencv-matrix
 rows: 3
 cols: 3
 dt: d
 data: [ 5.2658037684199849e+002, 0., 3.1841744018680112e+002, 0.,
 5.2465577209994706e+002, 2.0296659047014398e+002, 0., 0., 1\. ]
distortion_coefficients: !!opencv-matrix
 rows: 5
 cols: 1
 dt: d
 data: [ 7.3253671786835686e-002, -8.6143199924308911e-002,
 -2.0800255026966759e-002, -6.8004894417795971e-004,
 -1.7750733073535208e-001 ]
avg_reprojection_error: 3.6539552933501085e-001

CameraCalibration calibration(526.58037684199849e, 524.65577209994706e, 318.41744018680112, 202.96659047014398)

void PatternTrackingInfo::computePose(const Pattern& pattern, const CameraCalibration& calibration)
{
  cv::Mat camMatrix, distCoeff;
  cv::Mat(3,3, CV_32F, const_cast<float*>(&calibration.getIntrinsic().data[0])).copyTo(camMatrix);
  cv::Mat(4,1, CV_32F, const_cast<float*>(&calibration.getDistorsion().data[0])).copyTo(distCoeff);
  cv::Mat Rvec;
  cv::Mat_<float> Tvec;
  cv::Mat raux,taux;
  cv::solvePnP(pattern.points3d, points2d, camMatrix, distCoeff,raux,taux);
  raux.convertTo(Rvec,CV_32F);
  taux.convertTo(Tvec ,CV_32F);
  cv::Mat_<float> rotMat(3,3); 
  cv::Rodrigues(Rvec, rotMat);
  // Copy to transformation matrix
  pose3d = Transformation();
  for (int col=0; col<3; col++)
  {
    for (int row=0; row<3; row++)
    {        
     pose3d.r().mat[row][col] = rotMat(row,col); 
     // Copy rotation component
    }
    pose3d.t().data[col] = Tvec(col); 
    // Copy translation component
  }
  // Since solvePnP finds camera location, w.r.t to marker pose, 
  // to get marker pose w.r.t to the camera we invert it.
  pose3d = pose3d.getInverted();
}

class ARPipeline
{
public:
  ARPipeline(const cv::Mat& patternImage, const CameraCalibration& calibration);
  bool processFrame(const cv::Mat& inputFrame);
  const Transformation& getPatternLocation() const;
private:
  CameraCalibration   m_calibration;
  Pattern             m_pattern;
  PatternTrackingInfo m_patternInfo;
  PatternDetector     m_patternDetector;
};

ARPipeline::ARPipeline(const cv::Mat& patternImage, const CameraCalibration& calibration)
  : m_calibration(calibration)
{
  m_patternDetector.buildPatternFromImage (patternImage, m_pattern);
  m_patternDetector.train(m_pattern);
}
bool ARPipeline::processFrame(const cv::Mat& inputFrame)
{
  bool patternFound = m_patternDetector.findPattern(inputFrame, m_patternInfo);
  if (patternFound)
  {
    m_patternInfo.computePose(m_pattern, m_calibration);
  }
  return patternFound;
}
const Transformation& ARPipeline::getPatternLocation() const
{
  return m_patternInfo.pose3d;
}

cmake -D ENABLE_OPENGL=YES <path to the OpenCV source directory>

cv::namedWindow(ARWindowName, cv::WINDOW_OPENGL);

cv::resizeWindow(ARWindowName, 640, 480);

cv::setOpenGlContext(ARWindowName);

cv::setOpenGlDrawCallback(ARWindowName, drawAR, NULL);

void drawAR(void* param)
{
  // Draw something using OpenGL here
}

cv::updateWindow(ARWindowName);

class ARDrawingContext
{
public:
  ARDrawingContext(const CameraCalibration& c);
  bool                patternPresent;
  Transformation      patternPose;
  //! Request the redraw of the OpenGl window
  void draw();
  //! Set the new frame for the background
  void updateBackground(const cv::Mat& frame);
private:
  //! Draws the background with video
  void drawCameraFrame ();
  //! Draws the AR
  void drawAugmentedScene();
  //! Builds the right projection matrix 
  //! from the camera calibration for AR
  void buildProjectionMatrix(const Matrix33& calibration, int w, int h, Matrix44& result);
  //! Draws the coordinate axis 
  void drawCoordinateAxis();
  //! Draw the cube model
  void drawCubeModel();
private:
  bool               m_textureInitialized;
  unsigned int       m_backgroundTextureId;
  CameraCalibration  m_calibration;
  cv::Mat            m_backgroundImage;
};

ARDrawingContext::ARDrawingContext(std::string windowName, cv::Size frameSize, const CameraCalibration& c)
  : m_isTextureInitialized(false)
  , m_calibration(c)
  , m_windowName(windowName)
{
  // Create window with OpenGL support
  cv::namedWindow(windowName, cv::WINDOW_OPENGL);
  // Resize it exactly to video size
  cv::resizeWindow(windowName, frameSize.width, frameSize.height);
  // Initialize OpenGL draw callback:
  cv::setOpenGlContext(windowName);
  cv::setOpenGlDrawCallback(windowName, ARDrawingContextDrawCallback, this);
}

void ARDrawingContextDrawCallback(void* param)
{
  ARDrawingContext * ctx = static_cast<ARDrawingContext*>(param);
  if (ctx)
  {
    ctx->draw();
  }
}

void ARDrawingContext::draw()
{
  // Clear entire screen
  glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT); 
  // Render background
  drawCameraFrame();                                   
  // Draw AR
  drawAugmentedScene();                               
}

void ARDrawingContext::drawAugmentedScene()
{
  // Init augmentation projection
  Matrix44 projectionMatrix;
  int w = m_backgroundImage.cols;
  int h = m_backgroundImage.rows;
  buildProjectionMatrix(m_calibration, w, h, projectionMatrix);
  glMatrixMode(GL_PROJECTION);
  glLoadMatrixf(projectionMatrix.data);
  glMatrixMode(GL_MODELVIEW);
  glLoadIdentity();
  if (isPatternPresent)
  {
    // Set the pattern transformation
    Matrix44 glMatrix = patternPose.getMat44();
    glLoadMatrixf(reinterpret_cast<const GLfloat*>(&glMatrix.data[0]));
    // Render model
    drawCoordinateAxis();
    drawCubeModel();
  }
}

void processVideo(const cv::Mat& patternImage, CameraCalibration& calibration, cv::VideoCapture& capture);
void processSingleImage(const cv::Mat& patternImage, CameraCalibration& calibration, const cv::Mat& image);

/**
 * Performs full detection routine on camera frame 
.* and draws the scene using drawing context. 
 * In addition, this function draw overlay with debug information 
.* on top of the AR window. Returns true 
.* if processing loop should be stopped; otherwise - false.
 */
bool processFrame(const cv::Mat& cameraFrame, ARPipeline& pipeline, ARDrawingContext& drawingCtx)
{
    // Clone image used for background (we will 
    // draw overlay on it)
    cv::Mat img = cameraFrame.clone();
    // Draw information:
    if (pipeline.m_patternDetector.enableHomographyRefinement)
        cv::putText(img, "Pose refinement: On   ('h' to switch off)", cv::Point(10,15), CV_FONT_HERSHEY_PLAIN, 1, CV_RGB(0,200,0));
    else
        cv::putText(img, "Pose refinement: Off  ('h' to switch 
          on)",  cv::Point(10,15), CV_FONT_HERSHEY_PLAIN, 1, CV_RGB(0,200,0));
    cv::putText(img, "RANSAC threshold: " + ToString(pipeline.m_patternDetector.homographyReprojectionThreshold) + "( Use'-'/'+' to adjust)", cv::Point(10, 30), CV_FONT_HERSHEY_PLAIN, 1, CV_RGB(0,200,0));
    // Set a new camera frame:
    drawingCtx.updateBackground(img);
    // Find a pattern and update its detection status:
    drawingCtx.isPatternPresent = pipeline.processFrame(cameraFrame);
    // Update a pattern pose:
    drawingCtx.patternPose = pipeline.getPatternLocation();
    // Request redraw of the window:
    drawingCtx.updateWindow();
    // Read the keyboard input:
    int keyCode = cv::waitKey(5); 
    bool shouldQuit = false;
    if (keyCode == '+' || keyCode == '=')
    {
        pipeline.m_patternDetector.homographyReprojectionThreshold += 0.2f;
        pipeline.m_patternDetector.homographyReprojectionThreshold = std::min(10.0f, pipeline.m_patternDetector.homographyReprojectionThreshold);
    }
    else if (keyCode == '-')
    {
        pipeline.m_patternDetector.homographyReprojectionThreshold -= 0.2f;
        pipeline.m_patternDetector.homographyReprojectionThreshold = std::max(0.0f, pipeline.m_patternDetector.homographyReprojectionThreshold);
    }
    else if (keyCode == 'h')
    {
        pipeline.m_patternDetector.enableHomographyRefinement = !pipeline.m_patternDetector.enableHomographyRefinement;
    }
    else if (keyCode == 27 || keyCode == 'q')
    {
        shouldQuit = true;
    }
    return shouldQuit;
}

void processSingleImage(const cv::Mat& patternImage, CameraCalibration& calibration, const cv::Mat& image)
{
    cv::Size frameSize(image.cols, image.rows);
    ARPipeline pipeline(patternImage, calibration);
    ARDrawingContext drawingCtx("Markerless AR", frameSize, calibration);
    bool shouldQuit = false;
    do
    {
        shouldQuit = processFrame(image, pipeline, drawingCtx);
    } while (!shouldQuit);
}

markerless_ar_demo pattern.png test_image.png

markerless_ar_demo pattern.png test_video.avi