Keep tiles in a stable order (#2670)

* Keep tiles in a stable order

This introduces a new layer of abstraction on top of MediaViewModel: TileViewModel, which gives us a place to store data relating to tiles rather than their media, and also generally makes it easier to reason about tiles as they move about the call layout. I have created a class called TileStore to keep track of these tiles.

This allows us to swap out the media shown on a tile as the spotlight speaker changes, and avoid moving tiles around unless they really need to jump between the visible/invisible regions of the layout.

* Don't throttle spotlight updates

Since we now assume that the spotlight and grid will be in sync (i.e. an active speaker in one will behave as an active speaker in the other), we don't want the spotlight to ever lag behind due to throttling. If this causes usability issues we should maybe look into making LiveKit's 'speaking' indicators less erratic first.

* Make layout shifts due to a change in speaker less surprising

Although we try now to avoid layout shifts due to the spotlight speaker changing wherever possible, a spotlight speaker coming from off screen can still trigger one. Let's shift the layout a bit more gracefully in this case.

* Improve the tile ordering tests

* Maximize the spotlight tile in portrait layout

* Tell tiles whether they're actually visible in a more timely manner

* Fix test

* Fix speaking indicators logic

* Improve readability of marbles

* Fix test case

---------

Co-authored-by: Hugh Nimmo-Smith <[email protected]>

Author: robintown

src/grid/CallLayout.ts

@@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details.
 import { BehaviorSubject, Observable } from "rxjs";
 import { ComponentType } from "react";
 
-import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel";
 import { LayoutProps } from "./Grid";
+import { TileViewModel } from "../state/TileViewModel";
 
 export interface Bounds {
   width: number;
@@ -42,19 +42,6 @@ export interface CallLayoutInputs {
   pipAlignment: BehaviorSubject<Alignment>;
 }
 
-export interface GridTileModel {
-  type: "grid";
-  vm: UserMediaViewModel;
-}
-
-export interface SpotlightTileModel {
-  type: "spotlight";
-  vms: MediaViewModel[];
-  maximised: boolean;
-}
-
-export type TileModel = GridTileModel | SpotlightTileModel;
-
 export interface CallLayoutOutputs<Model> {
   /**
    * Whether the scrolling layer of the layout should appear on top.
@@ -63,11 +50,11 @@ export interface CallLayoutOutputs<Model> {
   /**
    * The visually fixed (non-scrolling) layer of the layout.
    */
-  fixed: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
+  fixed: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
   /**
    * The layer of the layout that can overflow and be scrolled.
    */
-  scrolling: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
+  scrolling: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
 }
 
 /**

src/grid/Grid.tsx

@@ -24,6 +24,7 @@ import {
   createContext,
   forwardRef,
   memo,
+  useCallback,
   useContext,
   useEffect,
   useMemo,
@@ -33,6 +34,8 @@ import {
 import useMeasure from "react-use-measure";
 import classNames from "classnames";
 import { logger } from "matrix-js-sdk/src/logger";
+import { useObservableEagerState } from "observable-hooks";
+import { fromEvent, map, startWith } from "rxjs";
 
 import styles from "./Grid.module.css";
 import { useMergedRefs } from "../useMergedRefs";
@@ -51,6 +54,7 @@ interface Tile<Model> {
   id: string;
   model: Model;
   onDrag: DragCallback | undefined;
+  setVisible: (visible: boolean) => void;
 }
 
 type PlacedTile<Model> = Tile<Model> & Rect;
@@ -84,6 +88,7 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
   id: string;
   model: Model;
   onDrag?: DragCallback;
+  onVisibilityChange?: (visible: boolean) => void;
   style?: CSSProperties;
   className?: string;
 }
@@ -131,6 +136,11 @@ export function useUpdateLayout(): void {
   );
 }
 
+const windowHeightObservable = fromEvent(window, "resize").pipe(
+  startWith(null),
+  map(() => window.innerHeight),
+);
+
 export interface LayoutProps<LayoutModel, TileModel, R extends HTMLElement> {
   ref: LegacyRef<R>;
   model: LayoutModel;
@@ -232,19 +242,42 @@ export function Grid<
   const [gridRoot, gridRef2] = useState<HTMLElement | null>(null);
   const gridRef = useMergedRefs<HTMLElement>(gridRef1, gridRef2);
 
+  const windowHeight = useObservableEagerState(windowHeightObservable);
   const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
   const [generation, setGeneration] = useState<number | null>(null);
   const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
   const prefersReducedMotion = usePrefersReducedMotion();
 
   const Slot: FC<SlotProps<TileModel>> = useMemo(
     () =>
-      function Slot({ id, model, onDrag, style, className, ...props }) {
+      function Slot({
+        id,
+        model,
+        onDrag,
+        onVisibilityChange,
+        style,
+        className,
+        ...props
+      }) {
         const ref = useRef<HTMLDivElement | null>(null);
+        const prevVisible = useRef<boolean | null>(null);
+        const setVisible = useCallback(
+          (visible: boolean) => {
+            if (
+              onVisibilityChange !== undefined &&
+              visible !== prevVisible.current
+            ) {
+              onVisibilityChange(visible);
+              prevVisible.current = visible;
+            }
+          },
+          [onVisibilityChange],
+        );
+
         useEffect(() => {
-          tiles.set(id, { id, model, onDrag });
+          tiles.set(id, { id, model, onDrag, setVisible });
           return (): void => void tiles.delete(id);
-        }, [id, model, onDrag]);
+        }, [id, model, onDrag, setVisible]);
 
         return (
           <div
@@ -302,6 +335,17 @@ export function Grid<
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [gridRoot, layoutRoot, tiles, gridBounds, generation]);
 
+  // The height of the portion of the grid visible at any given time
+  const visibleHeight = useMemo(
+    () => Math.min(gridBounds.bottom, windowHeight) - gridBounds.top,
+    [gridBounds, windowHeight],
+  );
+
+  useEffect(() => {
+    for (const tile of placedTiles)
+      tile.setVisible(tile.y + tile.height <= visibleHeight);
+  }, [placedTiles, visibleHeight]);
+
   // Drag state is stored in a ref rather than component state, because we use
   // react-spring's imperative API during gestures to improve responsiveness
   const dragState = useRef<DragState | null>(null);

src/grid/GridLayout.tsx

@@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks";
 import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
 import styles from "./GridLayout.module.css";
 import { useInitial } from "../useInitial";
-import {
-  CallLayout,
-  GridTileModel,
-  TileModel,
-  arrangeTiles,
-} from "./CallLayout";
+import { CallLayout, arrangeTiles } from "./CallLayout";
 import { DragCallback, useUpdateLayout } from "./Grid";
 
 interface GridCSSProperties extends CSSProperties {
@@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
         ),
       ),
     );
-    const tileModel: TileModel | undefined = useMemo(
-      () =>
-        model.spotlight && {
-          type: "spotlight",
-          vms: model.spotlight,
-          maximised: false,
-        },
-      [model.spotlight],
-    );
 
     const onDragSpotlight: DragCallback = useCallback(
       ({ xRatio, yRatio }) =>
@@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
 
     return (
       <div ref={ref} className={styles.fixed}>
-        {tileModel && (
+        {model.spotlight && (
           <Slot
             className={styles.slot}
             id="spotlight"
-            model={tileModel}
+            model={model.spotlight}
             onDrag={onDragSpotlight}
             data-block-alignment={alignment.block}
             data-inline-alignment={alignment.inline}
@@ -93,11 +79,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
       [width, minHeight, model.grid.length],
     );
 
-    const tileModels: GridTileModel[] = useMemo(
-      () => model.grid.map((vm) => ({ type: "grid", vm })),
-      [model.grid],
-    );
-
     return (
       <div
         ref={ref}
@@ -111,8 +92,14 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
           } as GridCSSProperties
         }
       >
-        {tileModels.map((m) => (
-          <Slot key={m.vm.id} className={styles.slot} id={m.vm.id} model={m} />
+        {model.grid.map((m) => (
+          <Slot
+            key={m.id}
+            className={styles.slot}
+            id={m.id}
+            model={m}
+            onVisibilityChange={m.setVisible}
+          />
         ))}
       </div>
     );

src/grid/OneOnOneLayout.tsx

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
 import classNames from "classnames";
 
 import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
-import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout";
+import { CallLayout, arrangeTiles } from "./CallLayout";
 import styles from "./OneOnOneLayout.module.css";
 import { DragCallback, useUpdateLayout } from "./Grid";
 
@@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
       [width, height],
     );
 
-    const remoteTileModel: GridTileModel = useMemo(
-      () => ({ type: "grid", vm: model.remote }),
-      [model.remote],
-    );
-    const localTileModel: GridTileModel = useMemo(
-      () => ({ type: "grid", vm: model.local }),
-      [model.local],
-    );
-
     const onDragLocalTile: DragCallback = useCallback(
       ({ xRatio, yRatio }) =>
         pipAlignment.next({
@@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
     return (
       <div ref={ref} className={styles.layer}>
         <Slot
-          id={remoteTileModel.vm.id}
-          model={remoteTileModel}
+          id={model.remote.id}
+          model={model.remote}
+          onVisibilityChange={model.remote.setVisible}
           className={styles.container}
           style={{ width: tileWidth, height: tileHeight }}
         >
           <Slot
             className={classNames(styles.slot, styles.local)}
-            id={localTileModel.vm.id}
-            model={localTileModel}
+            id={model.local.id}
+            model={model.local}
             onDrag={onDragLocalTile}
+            onVisibilityChange={model.local.setVisible}
             data-block-alignment={pipAlignmentValue.block}
             data-inline-alignment={pipAlignmentValue.inline}
           />

src/grid/SpotlightExpandedLayout.tsx

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { forwardRef, useCallback, useMemo } from "react";
+import { forwardRef, useCallback } from "react";
 import { useObservableEagerState } from "observable-hooks";
 
 import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
-import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout";
+import { CallLayout } from "./CallLayout";
 import { DragCallback, useUpdateLayout } from "./Grid";
 import styles from "./SpotlightExpandedLayout.module.css";
 
@@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
     ref,
   ) {
     useUpdateLayout();
-    const spotlightTileModel: SpotlightTileModel = useMemo(
-      () => ({ type: "spotlight", vms: model.spotlight, maximised: true }),
-      [model.spotlight],
-    );
 
     return (
       <div ref={ref} className={styles.layer}>
         <Slot
           className={styles.spotlight}
           id="spotlight"
-          model={spotlightTileModel}
+          model={model.spotlight}
         />
       </div>
     );
@@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
     useUpdateLayout();
     const pipAlignmentValue = useObservableEagerState(pipAlignment);
 
-    const pipTileModel: GridTileModel | undefined = useMemo(
-      () => model.pip && { type: "grid", vm: model.pip },
-      [model.pip],
-    );
-
     const onDragPip: DragCallback = useCallback(
       ({ xRatio, yRatio }) =>
         pipAlignment.next({
@@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
 
     return (
       <div ref={ref} className={styles.layer}>
-        {pipTileModel && (
+        {model.pip && (
           <Slot
             className={styles.pip}
-            id="pip"
-            model={pipTileModel}
+            id={model.pip.id}
+            model={model.pip}
             onDrag={onDragPip}
+            onVisibilityChange={model.pip.setVisible}
             data-block-alignment={pipAlignmentValue.block}
             data-inline-alignment={pipAlignmentValue.inline}
           />