一文读懂Web Codecs API：浏览器背后的媒体魔术师

// 使用MP4 Box 解封装
import Mp4box from 'mp4box'
 
// 非worker模式
 
export default class VideoPlayer {
 // options 暂时只有一个renderCanvas
 constructor(options) {
   // 解码器
   this.$codecs = new Codecs(this.output.bind(this))
   // 渲染器
   this.$render = new Render(options.renderCanvas)
 }
 // 输出
 output(type, data) {
   // 输出
   if(type === 'frame') {
     this.$render.render(data)
   } else {
     this.$render.playAudio(data)
   }
   // 销毁帧
   data.close()
 }
 // 加载文件
 loadFile(file) {
   return new Promise((resolve, reject) => {
     file.arrayBuffer().then((buffer) => {
       buffer.fileStart = 0
       this.$codecs.codecFile(buffer)
       resolve(true)
     }).catch((e) => {
       reject(e)
     })
   })
 }
}

// 渲染类
class Render {
 constructor(canvas) {
   this.canvas = canvas
   this.ctx = canvas.getContext('2d')
   this.audioCtx = new AudioContext()
   this.audioOutput = this.audioCtx.destination
 
   this.interleavingBuffers = []
   this.trackGenerator = new MediaStreamTrackGenerator({ kind: "audio" });
   this.writer = this.trackGenerator.writable.getWriter()
 
   const audio = document.createElement('audio')
   audio.setAttribute('controls', 'controls')
   document.body.appendChild(audio)
   const mediaStream = new MediaStream([this.trackGenerator]);
   audio.srcObject = mediaStream;
   
 }
 render(frame) {
   const { canvas, ctx } = this
   canvas.width = frame.displayWidth
   canvas.height = frame.displayHeight
   ctx.fillStyle = '#f00'
   ctx.clearRect(0, 0, frame.displayWidth, frame.displayHeight)
   ctx.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight)
 }
 playAudio(audioData) {
   this.bufferAudioData(audioData)
 }
 bufferAudioData(audioData) {
   const data = structuredClone(audioData)
   this.writer.write(audioData)
 }
}

class Codecs {
  // 初始化MP4
  public $M: any = Mp4box.createFile();
  constructor(output = (type, frame) => {}) {
    // 初始化
    this.$M.onReady = this.onReady.bind(this);
    this.$M.onSamples = this.onSamples.bind(this);
    // 视频解码器
    this.$videoCodec = new VideoDecoder({
      output(frame: any) {
        output('frame', frame);
      },
      error(e: any) {
        console.error(e);
      },
    });
    // 音频解码器
    this.$audioCodec = new AudioDecoder({
      output: (audioData: any) => {
        output('audioData', audioData);
      },
      error: (e: any) => {
        console.error(e);
      },
    });
  }
  // 当准备好了之后
  onReady(info: any) {
    const videoTrack = info.videoTracks[0];
    const videoConfig = {
      codec: videoTrack.codec,
      codedHeight: videoTrack.video.height,
      codedWidth: videoTrack.video.width,
      description: this.description(videoTrack),
    };
    const audioTrack = info.audioTracks[0];
    const audioConfig = {
      codec: audioTrack.codec,
      sampleRate: 48000,
      numberOfChannels: 2,
      description: this.description(audioTrack),
    };
    // 是否支持编码标准
    VideoDecoder.isConfigSupported(videoConfig).then((res) => {
      this.$videoCodec.configure(videoConfig);
      this.$M.setExtractionOptions(videoTrack.id);
      AudioDecoder.isConfigSupported(audioConfig).then((res) => {
        this.$audioCodec.configure(audioConfig);
        this.$M.setExtractionOptions(audioTrack.id);
        this.$M.start();
      });
    });
  }
  // 获取解码器的描述，才能完整解码
  description(track) {
    const trak = this.$M.getTrackById(track.id);
    console.log('trak', trak);
    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
      if (entry.avcC || entry.hvcC) {
        const stream = new Mp4box.DataStream(
          undefined,
          0,
          Mp4box.DataStream.BIG_ENDIAN
        );
        if (entry.avcC) {
          entry.avcC.write(stream);
        } else {
          entry.hvcC.write(stream);
        }
        return new Uint8Array(stream.buffer, 8); // Remove the box header.
      }
      if (entry.esds) {
        return entry.esds.esd.descs[0].descs[0].data;
      }
    }
    throw 'avcC or hvcC not found';
  }
  // MP4Box 传出的样本
  onSamples(track_id: any, ref: any, samples: any) {
    if (track_id === 2) {
      this.decodeAudio(samples);
    } else {
      this.decodeVideo(samples);
    }
  }
  // 解码视频
  decodeVideo(samples) {
    for (const sample of samples) {
      const data = new EncodedVideoChunk({
        type: sample.is_sync ? 'key' : 'delta',
        timestamp: (1e6 * sample.cts) / sample.timescale,
        duration: (1e6 * sample.duration) / sample.timescale,
        data: sample.data,
      });
      this.$videoCodec.decode(data);
    }
  }
  // 解码音频
  decodeAudio(samples) {
    for (const sample of samples) {
      const data = new EncodedAudioChunk({
        type: sample.is_sync ? 'key' : 'delta',
        timestamp: (1e6 * sample.cts) / sample.timescale,
        duration: (1e6 * sample.duration) / sample.timescale,
        data: sample.data,
      });
      this.$audioCodec.decode(data);
    }
  }
  // 开始解码
  codecFile(file: File | ArrayBuffer) {
    if (file instanceof File) {
      file.arrayBuffer().then((buffer) => {
        this.$M.appendBuffer(buffer);
        this.$M.flush();
      });
    } else if (file instanceof ArrayBuffer) {
      this.$M.appendBuffer(file);
      this.$M.flush();
    }
  }
}