user1475412 user1475412 - 3 months ago 16
Javascript Question

Synchronize video and audio (preferably without JavaScript)

If I have HTML5

video
and
audio
elements, is there a clean way to keep them in sync? They should act like a video file that contains an audio track, so advancing one track manually should bring the other one along with it. Let's say the two tracks have the same duration.

I'd like a solution that works across browsers, but I don't particularly care if it works on older browsers. It would also be nice to avoid the use of JavaScript if possible. Otherwise, a dead-simple JavaScript library would be best -- something that only asks for which tracks to synchronize and takes care of the rest.

I've looked into mediagroup... but it looks like it only works in Safari. I've looked into audioTracks... but the user has to enable the feature in Firefox.

I've looked into Popcorn.js, which is a JavaScript framework that seems designed for this task... but it looks like there hasn't been any activity in over a year. Besides that, the only demonstrations I can find are of synchronizing things like text or slides to video, not audio to video.

Answer

You can use Promise.all(), fetch() to retrieve media resource as a Blob, URL.createObjectURL() to create a Blob URL of resource; canplaythrough event and Array.prototype.every() to check if each resource can play; call .play() on each resource when both resources can play

I didn't make it clear in the question. I meant that the two tracks should stay in sync as though playing a regular video file with audio.

One approach could be to create a timestamp variable, utilize seeked event to update .currentTime of elements when set variable is greater than a minimum delay to prevent .currentTime being called recursively.

<!DOCTYPE html>
<html>
<head>
</head>
<body>
  <button>load media</button>
  <br>
  <div id="loading" style="display:none;">loading media...</div>
  <script>
    var mediaBlobs = [];
    var mediaTypes = [];
    var mediaLoaded = [];
    var button = document.querySelector("button");
    var loading = document.getElementById("loading");
    var curr = void 0;
    var loadMedia = () => {
      loading.style.display = "block";
      button.setAttribute("disabled", "disabled");
      return Promise.all([
        // `RETURN` by smiling cynic are licensed under a Creative Commons Attribution 3.0 Unported License
        fetch("https://ia600305.us.archive.org/30/items/return_201605/return.mp3")
      , fetch("http://nickdesaulniers.github.io/netfix/demo/frag_bunny.mp4")
        ])
        .then(responses => responses.map(media => ({
          [media.headers.get("Content-Type").split("/")[0]]: media.blob()
        })))
        .then(data => {
          for (var currentMedia = 0; currentMedia < data.length; currentMedia++) {
            (function(i) {
              var type = Object.keys(data[i])[0];
              mediaTypes.push(type);
              console.log(data[i]);
              var mediaElement = document.createElement(type);
              mediaElement.id = type;
              var label = document.createElement("label");
              mediaElement.setAttribute("controls", "controls");
              mediaElement.oncanplaythrough = () => {
                mediaLoaded.push(true);
                if (mediaLoaded.length === data.length 
                   && mediaLoaded.every(Boolean)) {
                    loading.style.display = "none";
                    for (var track = 0; track < mediaTypes.length; track++) {
                      document.getElementById(mediaTypes[track]).play();
                      console.log(document.getElementById(mediaTypes[track]));
                    }
                }
              }
              var seek = (e) => {
                if (!curr || new Date().getTime() - curr > 20) {
                  document.getElementById(
                    mediaTypes.filter(id => id !== e.target.id)[0]
                  ).currentTime = e.target.currentTime;
                  curr = new Date().getTime();
                }
              }
              mediaElement.onseeked = seek;
              mediaElement.onended = () => {
                for (var track = 0; track < mediaTypes.length; track++) {
                  document.getElementById(mediaTypes[track]).pause()
                }
              }
              mediaElement.ontimeupdate = (e) => {
                e.target.previousElementSibling
                .innerHTML = `${mediaTypes[i]} currentTime: ${Math.round(e.target.currentTime)}<br>`;
              }
              data[i][type].then(blob => {
                mediaBlobs.push(URL.createObjectURL(blob));
                mediaElement.src = mediaBlobs[mediaBlobs.length - 1];
                document.body.appendChild(mediaElement);
                document.body.insertBefore(label, mediaElement);
              })
            }(currentMedia));
          }
        })
    };
    button.addEventListener("click", loadMedia);
  </script>
</body>
</html>

plnkr http://plnkr.co/edit/r51oh7KsZv8DEYhpRJlL?p=preview