web端实现AR人脸特效

前言
直播、短视频、在线会议等应用越来越多地进入人们的生活，随之诞生的是丰富的各类创意玩法与新鲜体验，其中大量应用了以AI检测和图形渲染为基础的AR技术。

而随着Web技术的不断成熟，AR技术在Web上的实现成为了一种可能。今天就总结了在Web端实现此功能的几个技术要点，跟大家一起探讨一下。

架构和概念
抽象整体的实现思路如下

调取Camera获得相机画面使用tensorflow加载人脸识别模型生成FaceMesh根据FaceMesh生成三角网格并进行UV贴图
FaceMesh
MediaPipe Face Mesh是一种脸部几何解决方案，即使在移动设备上，也可以实时估计468个3D脸部界标。它采用 机器学习 （ML）来推断3D表面几何形状，只需要单个摄像机输入，而无需专用的深度传感器。该解决方案利用轻量级的模型架构以及整个管线中的GPU加速，可提供对实时体验至关重要的实时性能。

UVMap
UV是二维纹理坐标，U代表水平方向，V代表垂直方向。UV Map用来描述三维物体表面与图像纹理(Texture) 的映射关系，有了UV Map，我们就可以将二维的图像纹理粘贴到三维的物体表面。

image.png

矩形贴图和球面的映射图
技术实现
调取Camera获得相机画面
通过navigator.mediaDevices.getUserMedia获取stream，放到video查看。

async function setupWebcam() {

return new Promise( ( resolve, reject ) => {
    const webcamElement = document.getElementById( "webcam" );
    const navigatorAny = navigator;
    navigator.getUserMedia = navigator.getUserMedia ||
    navigatorAny.webkitGetUserMedia || navigatorAny.mozGetUserMedia ||
    navigatorAny.msGetUserMedia;
    if( navigator.getUserMedia ) {
        navigator.getUserMedia( { video: true },
            stream => {
                webcamElement.srcObject = stream;
                webcamElement.addEventListener( "loadeddata", resolve, false );
            },
        error => reject());
    }
    else {
        reject();
    }
});

}
复制代码
人脸识别
//创建模型
createModel() {

return new Promise(async resolve => {
    await tf.setBackend('webgl')
    const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
    const detectorConfig = {
        maxFaces: 1, //检测到的最大面部数量
        refineLandmarks: true, //可以完善眼睛和嘴唇周围的地标坐标，并在虹膜周围输出其他地标
        runtime: 'mediapipe',
        solutionPath: 'https://unpkg.com/@mediapipe/face_mesh', //WASM二进制文件和模型文件所在的路径
    };
    this.model = await faceLandmarksDetection.createDetector(model, detectorConfig);
    resolve(this.model);
})

},
//识别
async recognition() {

try {
    const video = this.$refs.video;
    const faces = await this.model.estimateFaces(video, {
        flipHorizontal: false, //镜像
    });
    if (faces.length > 0) {
        const keypoints = faces[0].keypoints;
        this.render3D({
            scaledMesh:keypoints.reduce((acc, pos) =>{
                acc.push([pos.x,pos.y,pos.z])
                return acc
            }, [])
        });
    }else{
        this.render3D({scaledMesh:[]})
    }
} catch (error) {
    console.log(error);
}

}
复制代码
3D场景贴图
TRIANGULATION
UV_COORDS

//3D场景
const scene = new THREE.Scene();

//添加一些光照
scene.add( new THREE.AmbientLight( 0xcccccc, 0.4 ) );
camera.add( new THREE.PointLight( 0xffffff, 0.8 ) );

//正交相机
scene camera = new THREE.PerspectiveCamera( 45, 1, 0.1, 2000 );
camera.position.x = videoWidth / 2;
camera.position.y = -videoHeight / 2;
camera.position.z = -( videoHeight / 2 ) / Math.tan( 45 / 2 )
scene.add( camera ); 

//渲染器
const renderer = new THREE.WebGLRenderer({
    canvas: document.getElementById( "overlay" ),
    alpha: true
});

//创建geometry，将468个人脸特征点按照一定的顺序(TRIANGULATION)组成三角网格，并加载UV_COORDS
const geometry = new THREE.BufferGeometry()
geometry.setIndex(TRIANGULATION)
geometry.setAttribute('uv', new THREE.Float32BufferAttribute(UV_COORDS.map((item, index) => index % 2 ? item : 1 - item), 2))
geometry.computeVertexNormals()
            
//创建material
const textureLoader = new THREE.TextureLoader();
const meshImg = this.meshList[meshIndex].src;//材质图片地址
textureLoader.load(meshImg,texture=>{
    texture.encoding = THREE.sRGBEncoding
    texture.anisotropy = 16
    const material = new THREE.MeshBasicMaterial({
        map: texture,
        transparent: true,
        color: new THREE.Color(0xffffff),
        reflectivity: 0.5
    });
    const mesh = new THREE.Mesh(geometry, material)
    scene.add(mesh)
})
// 根据face mesh实时更新geometry
updateGeometry(prediction){
    let w = canvasWidth;
    let h = canvasWidth;
    const faceMesh = resolveMesh(prediction.scaledMesh, w, h)
    const positionBuffer = faceMesh.reduce((acc, pos) => acc.concat(pos), [])
    geometry.setAttribute('position', new THREE.Float32BufferAttribute(positionBuffer, 3))
    geometry.attributes.position.needsUpdate = true
}

resolveMesh(faceMesh, vw, vh){

   return faceMesh.map(p => [p[0] - vw / 2, vh / 2 - p[1], -p[2]])

}

//渲染
render3D(prediction){

    if (prediction) {
        updateGeometry(prediction)
    }
    renderer.render(scene, threeCamera)
}

复制代码
加载3D模型
//加载3D模型
const loader = new GLTFLoader();
const Object3D = new THREE.Object3D();
loader.load(modelUrl, (gltf) => {

const object = gltf.scene
const box = new THREE.Box3().setFromObject(object)
const size = box.getSize(new THREE.Vector3()).length()
const center = box.getCenter(new THREE.Vector3())
object.position.x += (object.position.x - center.x);
object.position.y += (object.position.y - center.y + 1);
object.position.z += (object.position.z - center.z - 15);
Object3D.add(object)
this.scene.add(Object3D)

})

//计算Matrix
const position = prediction.midwayBetweenEyes[0]
const scale = this.getScale(prediction.scaledMesh, 234, 454)
const rotation = this.getRotation(prediction.scaledMesh, 10, 50, 280)
object.position.set(...position)
object.scale.setScalar(scale / 20)
object.scale.x *= -1
object.rotation.setFromRotationMatrix(rotation)
object.rotation.y = -object.rotation.y
object.rotateZ(Math.PI)
object.rotateX(-Math.PI * .05)
if (this.morphTarget) {

// flipped
this.morphTarget['leftEye'] && this.morphTarget['leftEye'](1 - prediction.faceRig.eye.r)
this.morphTarget['rightEye'] && this.morphTarget['rightEye'](1 - prediction.faceRig.eye.l)
this.morphTarget['mouth'] && this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)

}