fix: LSDV-4801: Better file type playable detection for video (#1259)

* fix: LSDV-4801: Better file type playable detection for video

* adding unit test for VirtualVideo filetype detection

* fixing cache invalidation of urls, to not operate on signed urls as it will invalidate the signature

* fix preview loading

* fix audio resolving in tests

Author: bmartel

e2e/tests/helpers.js

@@ -176,7 +176,7 @@ const waitForAudio = async () => {
 
   await Promise.all(
     [...audios].map(audio => {
-      if (audio.readyState === 4) return true;
+      if (audio.readyState === 4) return Promise.resolve(true);
       return new Promise(resolve => {
         audio.addEventListener('durationchange', () => {
           resolve(true);

jest.config.js

@@ -5,6 +5,7 @@ module.exports = {
     '<rootDir>/src',
   ],
   'preset': 'ts-jest',
+  'setupFilesAfterEnv': ['./jest.setup.js'],
   'testEnvironment': 'jsdom',
   'verbose': false,
   'collectCoverageFrom': [

jest.setup.js

@@ -0,0 +1,74 @@
+require('jest-fetch-mock').enableMocks();
+
+// Mock HTMLMediaElement data and methods not implemented by jsdom.
+window.HTMLMediaElement.prototype._mock = {
+  paused: true,
+  duration: NaN,
+  _loaded: false,
+  // Emulates the media file loading
+  _load: function mediaInit(media) {
+    media.dispatchEvent(new Event('loadedmetadata'));
+    media.dispatchEvent(new Event('loadeddata'));
+    media.dispatchEvent(new Event('canplaythrough'));
+  },
+  // Reset to the initial state
+  _resetMock: function resetMock(media) {
+    media._mock = Object.assign(
+      {},
+      window.HTMLMediaElement.prototype._mock,
+    );
+  },
+  _supportsTypes: [
+    'video/mp4', 'video/webm', 'video/ogg',
+    'audio/mp3', 'audio/webm', 'audio/ogg', 'audio/wav',
+  ],
+};
+
+// Get "paused" value, it is automatically set to true / false when we play / pause the media.
+Object.defineProperty(window.HTMLMediaElement.prototype, 'paused', {
+  get() {
+    return this._mock.paused;
+  },
+});
+
+// Get and set media duration
+Object.defineProperty(window.HTMLMediaElement.prototype, 'duration', {
+  get() {
+    return this._mock.duration;
+  },
+  set(value) {
+    // Reset the mock state to initial (paused) when we set the duration.
+    this._mock._resetMock(this);
+    this._mock.duration = value;
+  },
+});
+
+// Load the media file
+window.HTMLMediaElement.prototype.load = function loadMock() {
+  if (!this._mock._loaded) {
+    // emulate the media file load and metadata initialization
+    this._mock._load(this);
+  }
+  this.dispatchEvent(new Event('load'));
+};
+
+// Start the playback.
+window.HTMLMediaElement.prototype.play = function playMock() {
+  if (!this._mock._loaded) {
+    // emulate the media file load and metadata initialization
+    this._mock._load(this);
+  }
+  this._mock.paused = false;
+  this.dispatchEvent(new Event('play'));
+};
+
+// Pause the playback
+window.HTMLMediaElement.prototype.pause = function pauseMock() {
+  this._mock.paused = true;
+  this.dispatchEvent(new Event('pause'));
+};
+
+// Can play the media file
+window.HTMLMediaElement.prototype.canPlayType = function canPlayTypeMock(type) {
+  return this._mock._supportsTypes.includes(type) ? 'maybe' : '';
+};

package.json

@@ -112,6 +112,7 @@
     "@babel/runtime": "7.18.6",
     "@heartexlabs/eslint-plugin-frontend": "https://github.com/heartexlabs/eslint-plugin-frontend.git",
     "@svgr/webpack": "^5.5.0",
+    "@testing-library/react": "12.1.2",
     "@types/chroma-js": "^2.1.3",
     "@types/enzyme": "^3.10.12",
     "@types/jest": "^29.2.3",
@@ -149,6 +150,7 @@
     "insert-after": "^0.1.4",
     "jest": "^29.3.1",
     "jest-environment-jsdom": "^29.3.1",
+    "jest-fetch-mock": "^3.0.3",
     "jsdoc-to-markdown": "^8",
     "keymaster": "*",
     "konva": "^8.1.3",

src/components/VideoCanvas/VirtualVideo.tsx

@@ -8,18 +8,58 @@ type VirtualVideoProps = DetailedHTMLProps<VideoHTMLAttributes<HTMLVideoElement>
 
 const DEBUG_MODE = false;
 
-const canPlayUrl = async (url: string) => {
+// Just a mapping of file types to mime types, so we can check if the browser can play the file
+// before having to fall back to using a fetch request.
+const mimeTypeMapping = {
+  // Supported
+  'mp4': 'video/mp4',
+  'mp4v': 'video/mp4',
+  'mpg4': 'video/mp4',
+
+  'ogg': 'video/ogg',
+  'ogv': 'video/ogg',
+  'ogm': 'video/ogg',
+  'ogx': 'video/ogg',
+
+  // Partially supported
+  'webm': 'video/webm',
+
+  // Unsupported
+  'avi': 'video/avi',
+  'mov': 'video/quicktime',
+  'qt': 'video/quicktime',
+};
+
+const isBinary = (mimeType: string|null|undefined) => {
+  if (!mimeType) {
+    return false;
+  }
+
+  return mimeType.includes('octet-stream');
+};
+
+export const canPlayUrl = async (url: string) => {
   const video = document.createElement('video');
 
-  const fileMeta = await fetch(url, {
-    method: 'GET',
-    headers: {
-      'Range': 'bytes=0-0',
-    },
-  });
+  const pathName = new URL(url, /^https?/.exec(url) ? undefined : window.location.href).pathname;
 
-  const fileType = fileMeta.headers.get('content-type');
-  const supported = !!fileType && video.canPlayType(fileType) !== '';
+  const fileType = (pathName.split('.').pop() ?? '') as keyof typeof mimeTypeMapping;
+
+  let fileMimeType: string|null|undefined = mimeTypeMapping[fileType];
+
+  if (!fileMimeType) {
+    const fileMeta = await fetch(url, {
+      method: 'GET',
+      headers: {
+        'Range': 'bytes=0-0',
+      },
+    });
+
+    fileMimeType = fileMeta.headers.get('content-type');
+  }
+
+  // If the file is binary, we can't check if the browser can play it, so we just assume it can.
+  const supported = isBinary(fileMimeType) || (!!fileMimeType && video.canPlayType(fileMimeType) !== '');
   const modalExists = document.querySelector('.ant-modal');
 
   if (!supported && !modalExists) InfoModal.error('There has been an error rendering your video, please check the format is supported');
@@ -140,11 +180,14 @@ export const VirtualVideo = forwardRef<HTMLVideoElement, VirtualVideoProps>((pro
   useEffect(() => {
     createVideoElement();
     attachEventListeners();
-    canPlayType(props.src ?? '');
-    attachSource();
-    attachRef(video.current);
+    canPlayType(props.src ?? '').then((canPlay) => {
+      if (canPlay) {
+        attachSource();
+        attachRef(video.current);
 
-    document.body.append(video.current!);
+        document.body.append(video.current!);
+      }
+    });
   }, []);
 
   // Handle video cleanup

src/components/VideoCanvas/__tests__/VirtualVideo.test.tsx

@@ -0,0 +1,101 @@
+
+import { render } from '@testing-library/react';
+import fetchMock from 'jest-fetch-mock';
+import * as VirtualVideo from '../VirtualVideo';
+
+
+describe('VirtualVideo', () => {
+  it('should call canPlayUrl and return false if no url specified', async () => {
+    const canPlayType = jest.fn();
+
+    render(<VirtualVideo.VirtualVideo canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(false);
+  });
+
+  it('should call canPlayUrl and return true if valid url specified', async () => {
+    const canPlayType = jest.fn();
+
+    render(<VirtualVideo.VirtualVideo src="https://app.heartex.ai/static/samples/opossum_snow.mp4" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(true);
+  });
+
+  it('should call canPlayUrl and return true if valid relative url specified', async () => {
+    const canPlayType = jest.fn();
+
+    render(<VirtualVideo.VirtualVideo src="/files/opossum_intro.webm" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(true);
+  });
+
+  it('should call canPlayUrl and return true if valid url specified, even if content-type is binary/octet-stream', async () => {
+    const canPlayType = jest.fn();
+
+    // return binary/octet-stream for all requests, mimicking the situation where
+    // the server doesn't set the content-type header and defaults to binary/octet-stream
+    fetchMock.mockResponseOnce('', {
+      headers: {
+        'content-type': 'binary/octet-stream',
+      },
+    });
+
+    render(<VirtualVideo.VirtualVideo src="https://app.heartex.ai/static/samples/opossum_snow.mp4" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(true);
+  });
+
+  it('should call canPlayUrl and return true if valid file is specified, and content-type is binary/octet-stream but no file extension', async () => {
+    const canPlayType = jest.fn();
+
+    // return binary/octet-stream for all requests, mimicking the situation where
+    // the server doesn't set the content-type header and defaults to binary/octet-stream
+    fetchMock.mockResponseOnce('', {
+      headers: {
+        'content-type': 'binary/octet-stream',
+      },
+    });
+
+    render(<VirtualVideo.VirtualVideo src="https://app.heartex.ai/static/samples/opossum_snow" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(true);
+  });
+
+  it('should call canPlayUrl and return false if invalid url specified', async () => {
+    const canPlayType = jest.fn();
+
+    render(<VirtualVideo.VirtualVideo src="https://app.heartex.ai/static/samples/opossum_snow.avi" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(false);
+  });
+
+  it('should call canPlayUrl and return false if invalid url specified, even if content-type is binary/octet-stream', async () => {
+    const canPlayType = jest.fn();
+
+    // return binary/octet-stream for all requests, mimicking the situation where
+    // the server doesn't set the content-type header and defaults to binary/octet-stream
+    fetchMock.mockResponseOnce('', {
+      headers: {
+        'content-type': 'binary/octet-stream',
+      },
+    });
+
+    render(<VirtualVideo.VirtualVideo src="https://app.heartex.ai/static/samples/opossum_snow.avi" canPlayType={canPlayType} />);
+
+    await new Promise(resolve => setTimeout(resolve, 10));
+
+    expect(canPlayType).toBeCalledWith(false);
+  });
+});

src/lib/AudioUltra/Media/MediaLoader.ts

@@ -188,10 +188,20 @@ export class MediaLoader extends Destructable {
       // Handle relative urls, by converting them to absolute so any query params can be preserved
       const newUrl = new URL(url, /^https?/.exec(url) ? undefined : window.location.href);
 
-      // Arbitrary setting of query param to stop caching from reusing any media requests which may have less headers
-      // cached than this request. This is to prevent a CORS error when the headers are different between partial
-      // content and full content requests.
-      newUrl.searchParams.set('lsref', '1');
+      const signedUrlParams = [
+        'X-Goog-Signature', // Google Cloud Storage
+        'X-Amz-Signature', // S3|Minio|DigitalOcean|Backblaze
+        'sig', // Azure
+      ];
+
+      // If the url is signed, we need to preserve the query params otherwise the signature will be invalid
+      if (!signedUrlParams.some(p => newUrl.searchParams.has(p))) {
+        // Arbitrary setting of query param to stop caching from reusing any media requests which may have less headers
+        // cached than this request. This is to prevent a CORS error when the headers are different between partial
+        // content and full content requests.
+        newUrl.searchParams.set('lsref', '1');
+      }
+
 
       xhr.open('GET', newUrl.toString(), true);
       xhr.send();