IanWizard IanWizard - 11 months ago 78
Javascript Question

JavaScript: Extract video frames reliably

I'm working on a client-side project which lets a user supply a video file and apply basic manipulations to it. I'm trying to extract the frames from the video reliably. At the moment I have a

which I'm loading selected video into, and then pulling out each frame as follows:

  1. Seek to the beginning

  2. Pause the video

  3. Draw
    to a

  4. Capture the frame from the canvas with

  5. Seek forward by 1 / 30 seconds (1 frame).

  6. Rinse and repeat

This is a rather inefficient process, and more specifically, is proving unreliable as I'm often getting stuck frames. This seems to be from it not updating the actual
element before it draws to the canvas.

I'd rather not have to upload the original video to the server just to split the frames, and then download them back to the client.

Any suggestions for a better way to do this are greatly appreciated. The only caveat is that I need it to work with any format the browser supports (decoding in JS isn't a great option).

Answer Source

Mostly taken from this great answer by GameAlchemist :

Since browsers doesn't respect videos' framerates, but instead "use of some tricks to make a match between the frame-rate of the movie and the refresh-rate of the screen", your assumption that every 30th of a second, a new frame will be painted is quite inaccurate.
However, the timeupdate event should fire when the currentTime has changed, and we can assume that a new frame was painted.

So, I would do it like so :

document.querySelector('input').addEventListener('change', extractFrames, false);

function extractFrames() {
  var video = document.createElement('video');
  var array = [];
  var canvas = document.createElement('canvas');
  var ctx = canvas.getContext('2d');
  var pro = document.querySelector('#progress');

  function initCanvas(e) {
    canvas.width = this.videoWidth;
    canvas.height = this.videoHeight;

  function drawFrame(e) {
    ctx.drawImage(this, 0, 0);
    this will save as a Blob, less memory consumptive than toDataURL
    a polyfill can be found at
    canvas.toBlob(saveFrame, 'image/jpeg');
    pro.innerHTML = ((this.currentTime / this.duration) * 100).toFixed(2) + ' %';
    if (this.currentTime < this.duration) {

  function saveFrame(blob) {

  function revokeURL(e) {
  function onend(e) {
    var img;
    // do whatever with the frames
    for (var i = 0; i < array.length; i++) {
      img = new Image();
      img.onload = revokeURL;
      img.src = URL.createObjectURL(array[i]);
    // we don't need the video's objectURL anymore
  video.autoplay = true;
  video.muted = true;

  video.addEventListener('loadedmetadata', initCanvas, false);
  video.addEventListener('timeupdate', drawFrame, false);
  video.addEventListener('ended', onend, false);

  video.src = URL.createObjectURL(this.files[0]);
<input type="file" accept="video/*" />
<p id="progress"></p>