Squash

Author: mnemitz

examples/nextjs/.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "command": "npm run dev",
+      "name": "Run npm run dev",
+      "request": "launch",
+      "type": "node-terminal",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "command": "npm run test",
+      "name": "Run npm run test",
+      "request": "launch",
+      "type": "node-terminal",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Attach by Process ID",
+      "processId": "${command:PickProcess}",
+      "request": "attach",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "node"
+    }
+  ]
+}

examples/nextjs/package.json

@@ -15,6 +15,7 @@
     "@speechmatics/browser-audio-input": "workspace:*",
     "@speechmatics/browser-audio-input-react": "workspace:*",
     "@speechmatics/flow-client-react": "workspace:*",
+    "@speechmatics/real-time-client-react": "workspace:*",
     "next": "15.0.1",
     "react": "19.0.0-rc-69d4b800-20241021",
     "react-dom": "19.0.0-rc-69d4b800-20241021",

examples/nextjs/src/app/real-time/Component.tsx

@@ -0,0 +1,39 @@
+'use client';
+
+import { useRealtimeTranscription } from '@speechmatics/real-time-client-react';
+import { Controls } from './Controls';
+import { useEffect } from 'react';
+import {
+  usePcmAudioListener,
+  usePcmAudioRecorder,
+} from '@speechmatics/browser-audio-input-react';
+import { Output } from './Output';
+import { Status } from './Status';
+
+export default function Component() {
+  const { stopTranscription, sendAudio } = useRealtimeTranscription();
+  const { stopRecording } = usePcmAudioRecorder();
+
+  usePcmAudioListener(sendAudio);
+
+  // Cleanup
+  useEffect(() => {
+    return () => {
+      stopTranscription();
+      stopRecording();
+    };
+  }, [stopTranscription, stopRecording]);
+
+  return (
+    <section>
+      <h3>Real-time Example</h3>
+      <section className="grid">
+        <Controls />
+        <Status />
+      </section>
+      <section>
+        <Output />
+      </section>
+    </section>
+  );
+}

examples/nextjs/src/app/real-time/Controls.tsx

@@ -0,0 +1,139 @@
+import { type ChangeEvent, type FormEvent, useCallback, useState } from 'react';
+import {
+  useAudioDevices,
+  usePcmAudioRecorder,
+} from '@speechmatics/browser-audio-input-react';
+import {
+  type RealtimeTranscriptionConfig,
+  useRealtimeTranscription,
+} from '@speechmatics/real-time-client-react';
+import { getJWT } from '../actions';
+import { configFromFormData } from '@/lib/real-time/config-from-form-data';
+import { RECORDING_SAMPLE_RATE } from '@/lib/constants';
+
+export function Controls() {
+  const { startTranscription } = useRealtimeTranscription();
+  const { startRecording } = usePcmAudioRecorder();
+
+  const [deviceId, setDeviceId] = useState<string>();
+
+  const startSession = useCallback(
+    async (config: RealtimeTranscriptionConfig) => {
+      const jwt = await getJWT('rt');
+      await startTranscription(jwt, config);
+      await startRecording({ deviceId, sampleRate: RECORDING_SAMPLE_RATE });
+    },
+    [startTranscription, startRecording, deviceId],
+  );
+
+  const handleSubmit = useCallback(
+    (e: FormEvent<HTMLFormElement>) => {
+      e.preventDefault();
+      const formData = new FormData(e.currentTarget);
+      const config = configFromFormData(formData);
+      config.audio_format = {
+        type: 'raw',
+        encoding: 'pcm_f32le',
+        sample_rate: RECORDING_SAMPLE_RATE,
+      };
+      startSession(config);
+    },
+    [startSession],
+  );
+
+  return (
+    <article>
+      <form onSubmit={handleSubmit}>
+        <div className="grid">
+          <MicrophoneSelect setDeviceId={setDeviceId} />
+          <label>
+            Select language
+            <select name="language">
+              <option value="en" label="English" defaultChecked />
+              <option value="es" label="Spanish" />
+              <option value="ar" label="Arabic" />
+            </select>
+          </label>
+        </div>
+        <div className="grid">
+          <StartStopButton />
+        </div>
+      </form>
+    </article>
+  );
+}
+
+function StartStopButton() {
+  const { stopRecording } = usePcmAudioRecorder();
+  const { stopTranscription } = useRealtimeTranscription();
+
+  const stopSession = useCallback(() => {
+    stopTranscription();
+    stopRecording();
+  }, [stopRecording, stopTranscription]);
+
+  const connected = useRealtimeTranscription().socketState === 'open';
+
+  if (connected) {
+    return (
+      <button type="button" onClick={stopSession}>
+        Stop transcription
+      </button>
+    );
+  }
+
+  return <button type="submit">Transcribe audio</button>;
+}
+
+function MicrophoneSelect({
+  setDeviceId,
+}: { setDeviceId: (deviceId: string) => void }) {
+  const devices = useAudioDevices();
+
+  switch (devices.permissionState) {
+    case 'prompt':
+      return (
+        <label>
+          Enable mic permissions
+          <select
+            onClick={devices.promptPermissions}
+            onKeyDown={devices.promptPermissions}
+          />
+        </label>
+      );
+    case 'prompting':
+      return (
+        <label>
+          Enable mic permissions
+          <select aria-busy="true" />
+        </label>
+      );
+    case 'granted': {
+      const onChange = (e: ChangeEvent<HTMLSelectElement>) => {
+        setDeviceId(e.target.value);
+      };
+      return (
+        <label>
+          Select audio device
+          <select onChange={onChange}>
+            {devices.deviceList.map((d) => (
+              <option key={d.deviceId} value={d.deviceId}>
+                {d.label}
+              </option>
+            ))}
+          </select>
+        </label>
+      );
+    }
+    case 'denied':
+      return (
+        <label>
+          Microphone permission disabled
+          <select disabled />
+        </label>
+      );
+    default:
+      devices satisfies never;
+      return null;
+  }
+}

examples/nextjs/src/app/real-time/Output.tsx

@@ -0,0 +1,26 @@
+import { useReducer } from 'react';
+import {
+  type RealtimeServerMessage,
+  useRealtimeEventListener,
+} from '@speechmatics/real-time-client-react';
+
+export function Output() {
+  const [transcription, dispatch] = useReducer(transcriptReducer, '');
+
+  useRealtimeEventListener('receiveMessage', (e) => dispatch(e.data));
+
+  return (
+    <article>
+      <header>Output</header>
+      <p>{transcription}</p>
+    </article>
+  );
+}
+
+function transcriptReducer(acc: string, event: RealtimeServerMessage) {
+  if (event.message === 'AddTranscript') {
+    return `${acc} ${event.results.map((result) => result.alternatives?.[0].content).join(' ')}`;
+  }
+
+  return acc;
+}

examples/nextjs/src/app/real-time/Status.tsx

@@ -0,0 +1,21 @@
+import { usePcmAudioRecorder } from '@speechmatics/browser-audio-input-react';
+import { useRealtimeTranscription } from '@speechmatics/real-time-client-react';
+
+export function Status() {
+  const { socketState, sessionId } = useRealtimeTranscription();
+  const { isRecording } = usePcmAudioRecorder();
+
+  return (
+    <article>
+      <header>Status</header>
+      <dl>
+        <dt>🔌 Socket is</dt>
+        <dd>{socketState ?? '(uninitialized)'}</dd>
+        <dt>💬 Session ID</dt>
+        <dd>{sessionId ?? '(none)'}</dd>
+        <dt>🎤 Microphone is</dt>
+        <dd>{isRecording ? 'recording' : 'not recording'}</dd>
+      </dl>
+    </article>
+  );
+}

examples/nextjs/src/app/real-time/page.tsx

@@ -1,3 +1,13 @@
+import { RealtimeTranscriptionProvider } from '@speechmatics/real-time-client-react';
+import Component from './Component';
+import { PcmAudioRecorderProvider } from '@speechmatics/browser-audio-input-react';
+
 export default function Page() {
-  return <h1>Coming soon!</h1>;
+  return (
+    <PcmAudioRecorderProvider workletScriptURL="/js/pcm-audio-worklet.min.js">
+      <RealtimeTranscriptionProvider appId="nextjs-rt-example">
+        <Component />
+      </RealtimeTranscriptionProvider>
+    </PcmAudioRecorderProvider>
+  );
 }

examples/nextjs/src/lib/real-time/config-from-form-data.ts

@@ -0,0 +1,19 @@
+import type { RealtimeTranscriptionConfig } from '@speechmatics/real-time-client-react';
+
+// TODO could have zod schemas here
+export function configFromFormData(
+  formData: FormData,
+): RealtimeTranscriptionConfig {
+  const language = formData.get('language')?.toString();
+
+  if (!language) {
+    throw new Error('Language is required');
+  }
+
+  return {
+    transcription_config: {
+      language,
+      max_delay: 1,
+    },
+  };
+}

packages/real-time-client-react/README.md

@@ -0,0 +1,89 @@
+# Speechmatics Flow client (React) 🤖 ⚛
+
+React hooks for interacting with the [Speechmatics Flow API](https://docs.speechmatics.com/flow/getting-started).
+
+This package wraps the `@speechmatics/flow-client` package for use in React projects.
+
+## Installlation
+
+```
+npm i @speechmatics/flow-client-react
+```
+
+> [!WARNING]  
+> For React Native, make sure to install the [`event-target-polyfill`](https://www.npmjs.com/package/event-target-polyfill) package, or any other polyfill for the [`EventTarget` class](https://developer.mozilla.org/en-US/docs/Web/API/EventTarget)
+
+## Usage
+
+
+1. Use the `FlowProvider` at a high level of your component tree:
+
+    ```JSX
+    import { FlowProvider } from "@speechmatics/flow-client-react";
+
+    function RootLayout({children}) {
+      <FlowProvider appId="your-app-id">
+        {children}
+      </FlowProvider>
+    }
+    ```
+    Note that`FlowProvider` is a [client component](https://nextjs.org/docs/app/building-your-application/rendering/client-components), like any other context provider. In NextJS, it's best to put this either in a root layout, or inside another client component. For frameworks like Remix which don't use React Server Components, it should work anywhere.
+
+    _Note for React Native_: Pass `websocketBinaryType="arraybuffer"` to the `FlowProvider` as it is more reliable than the Blob implementation.
+
+1. Inside a component below the `FlowProvider`:
+    ```JSX
+    function MyComponent() {
+      const { startConversation, endConversation, sendAudio } = useFlow()
+    }
+    ```
+
+
+1. Connect and start conversation
+  
+    `startConversation` is a function which requires a JWT to open a websocket and begin a session.
+  
+    See our documentation about generating JWTs (temporary keys): https://docs.speechmatics.com/introduction/authentication#temporary-keys
+
+    An example credentials fetching flow can be found in the [NextJS example](/examples/nextjs/src/lib/fetch-credentials.ts).
+  
+    ```typescript
+      await startConversation(jwt, {
+        config: {
+          template_id: personaId,
+          template_variables: {},
+        },
+        // `audioFormat` is optional. The value below is the default:
+        audioFormat: {
+          type: 'raw',
+          encoding: 'pcm_s16le', // this can also be set to 'pcm_f32le' for 32-bit Float
+          sample_rate: 16000,
+        },
+      });
+    ```
+
+
+1. Sending audio
+
+    The `sendAudio` function above accepts any `ArrayBufferLike`. You should send a buffer with the audio encoded as you requested when calling `startConversation` (either 32-bit float or 16-bit signed integer PCM).
+
+
+1. Listen for audio and messages
+
+    Incoming data from the Flow service can be subscribed to using the `useFlowEventListener` hook:
+
+    ```TSX
+    // Handling Messages
+    useFlowEventListener("message", ({ data }) => {
+      if (data.message === "AddTranscript") {
+        // handle transcript message
+      }
+    });
+
+    // Handling audio
+    useFlowEventListener("agentAudio", (audio) => {
+      // Incoming audio data is always 16-bit signed int PCM.
+      // How you handle this depends on your environment.
+      myPlayAudioFunction(audio.data);
+    })
+    ```