Update makima FE to add initial listening system

3 files changed, 544 insertions, 0 deletions

diff --git a/makima/frontend/src/hooks/useMicrophone.ts b/makima/frontend/src/hooks/useMicrophone.ts
new file mode 100644
index 0000000..307904b
--- /dev/null
+++ b/makima/frontend/src/hooks/useMicrophone.ts
@@ -0,0 +1,248 @@
+import { useState, useCallback, useRef, useEffect } from "react";
+
+export type MicrophoneStatus =
+  | "idle"
+  | "requesting"
+  | "ready"
+  | "recording"
+  | "denied"
+  | "error";
+
+export interface MicrophoneState {
+  status: MicrophoneStatus;
+  error: string | null;
+  sampleRate: number;
+  channels: number;
+  volume: number;
+}
+
+interface UseMicrophoneOptions {
+  sampleRate?: number;
+  onAudioData?: (samples: Float32Array) => void;
+}
+
+function getErrorMessage(err: unknown): { message: string; status: MicrophoneStatus } {
+  if (err instanceof DOMException) {
+    switch (err.name) {
+      case "NotAllowedError":
+      case "PermissionDeniedError":
+        return { message: "Microphone permission denied", status: "denied" };
+      case "NotFoundError":
+        return { message: "No microphone found", status: "error" };
+      case "NotReadableError":
+      case "TrackStartError":
+        return { message: "Microphone is in use by another application", status: "error" };
+      case "OverconstrainedError":
+        return { message: "Microphone does not support requested settings", status: "error" };
+      case "AbortError":
+        return { message: "Microphone access was aborted", status: "error" };
+      case "SecurityError":
+        return { message: "Microphone access blocked (requires HTTPS)", status: "error" };
+      default:
+        return { message: `Microphone error: ${err.name} - ${err.message}`, status: "error" };
+    }
+  }
+
+  if (err instanceof Error) {
+    return { message: err.message, status: "error" };
+  }
+
+  return { message: "Failed to access microphone", status: "error" };
+}
+
+export function useMicrophone(options: UseMicrophoneOptions = {}) {
+  const { onAudioData } = options;
+
+  const [state, setState] = useState<MicrophoneState>({
+    status: "idle",
+    error: null,
+    sampleRate: 48000,
+    channels: 1,
+    volume: 0,
+  });
+
+  const streamRef = useRef<MediaStream | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const processorRef = useRef<ScriptProcessorNode | null>(null);
+  const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const onAudioDataRef = useRef(onAudioData);
+
+  // Keep callback ref updated
+  useEffect(() => {
+    onAudioDataRef.current = onAudioData;
+  }, [onAudioData]);
+
+  // Check if microphone permission is already granted
+  const checkPermission = useCallback(async (): Promise<boolean> => {
+    try {
+      const result = await navigator.permissions.query({
+        name: "microphone" as PermissionName,
+      });
+      return result.state === "granted";
+    } catch {
+      return false;
+    }
+  }, []);
+
+  // Request microphone permission without starting recording
+  const requestPermission = useCallback(async (): Promise<boolean> => {
+    setState((s) => ({ ...s, status: "requesting", error: null }));
+
+    // Check for secure context
+    if (typeof window !== "undefined" && !window.isSecureContext) {
+      setState((s) => ({ ...s, status: "error", error: "Microphone requires HTTPS (or localhost)" }));
+      return false;
+    }
+
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      // Permission granted - stop the stream immediately
+      stream.getTracks().forEach((track) => track.stop());
+      setState((s) => ({ ...s, status: "ready", error: null }));
+      return true;
+    } catch (err) {
+      const { message, status } = getErrorMessage(err);
+      setState((s) => ({ ...s, status, error: message }));
+      return false;
+    }
+  }, []);
+
+  const start = useCallback(async (): Promise<boolean> => {
+    if (state.status === "recording") return true;
+
+    setState((s) => ({ ...s, status: "requesting", error: null }));
+
+    // Check for secure context
+    if (typeof window !== "undefined" && !window.isSecureContext) {
+      setState((s) => ({ ...s, status: "error", error: "Microphone requires HTTPS (or localhost)" }));
+      return false;
+    }
+
+    let stream: MediaStream;
+
+    try {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch (err) {
+      const { message, status } = getErrorMessage(err);
+      setState((s) => ({ ...s, status, error: message }));
+      return false;
+    }
+
+    try {
+      streamRef.current = stream;
+
+      // Create audio context
+      const AudioContextClass = window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext;
+      const audioContext = new AudioContextClass();
+      audioContextRef.current = audioContext;
+
+      // Resume audio context if it's suspended
+      if (audioContext.state === "suspended") {
+        await audioContext.resume();
+      }
+
+      // Create source from microphone
+      const source = audioContext.createMediaStreamSource(stream);
+      sourceRef.current = source;
+
+      // Use ScriptProcessor for audio processing
+      const bufferSize = 4096;
+      const processor = audioContext.createScriptProcessor(bufferSize, 1, 1);
+      processorRef.current = processor;
+
+      processor.onaudioprocess = (event) => {
+        const inputData = event.inputBuffer.getChannelData(0);
+        const samples = new Float32Array(inputData.length);
+        samples.set(inputData);
+
+        // Calculate RMS volume (0-1 range)
+        let sum = 0;
+        for (let i = 0; i < samples.length; i++) {
+          sum += samples[i] * samples[i];
+        }
+        const rms = Math.sqrt(sum / samples.length);
+        // Normalize and clamp to 0-1 range (typical speech is around 0.1-0.3 RMS)
+        const normalizedVolume = Math.min(1, rms * 3);
+        setState((s) => ({ ...s, volume: normalizedVolume }));
+
+        if (onAudioDataRef.current) {
+          onAudioDataRef.current(samples);
+        }
+      };
+
+      source.connect(processor);
+      processor.connect(audioContext.destination);
+
+      setState((s) => ({
+        ...s,
+        status: "recording",
+        sampleRate: audioContext.sampleRate,
+        error: null,
+      }));
+
+      return true;
+    } catch (err) {
+      stream.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+
+      const { message, status } = getErrorMessage(err);
+      setState((s) => ({ ...s, status, error: message }));
+      return false;
+    }
+  }, [state.status]);
+
+  const stop = useCallback(() => {
+    if (processorRef.current && sourceRef.current) {
+      try {
+        sourceRef.current.disconnect(processorRef.current);
+        processorRef.current.disconnect();
+      } catch {
+        // Already disconnected
+      }
+      processorRef.current = null;
+      sourceRef.current = null;
+    }
+
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+    }
+
+    if (audioContextRef.current) {
+      audioContextRef.current.close();
+      audioContextRef.current = null;
+    }
+
+    setState((s) => ({ ...s, status: "idle", error: null, volume: 0 }));
+  }, []);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      if (processorRef.current && sourceRef.current) {
+        try {
+          sourceRef.current.disconnect(processorRef.current);
+          processorRef.current.disconnect();
+        } catch {
+          // Already disconnected
+        }
+      }
+      if (streamRef.current) {
+        streamRef.current.getTracks().forEach((track) => track.stop());
+      }
+      if (audioContextRef.current) {
+        audioContextRef.current.close();
+      }
+    };
+  }, []);
+
+  return {
+    ...state,
+    start,
+    stop,
+    checkPermission,
+    requestPermission,
+    isRecording: state.status === "recording",
+    isDenied: state.status === "denied",
+  };
+}
diff --git a/makima/frontend/src/hooks/useTextScramble.ts b/makima/frontend/src/hooks/useTextScramble.ts
new file mode 100644
index 0000000..bb3f365
--- /dev/null
+++ b/makima/frontend/src/hooks/useTextScramble.ts
@@ -0,0 +1,52 @@
+import { useState, useCallback, useRef } from "react";
+
+const GLYPHS = "▒▓░█#@*+:-/[]{}<>_";
+
+export function useTextScramble(originalText: string) {
+  const [displayText, setDisplayText] = useState(originalText);
+  const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const iterationRef = useRef(0);
+
+  const scramble = useCallback(() => {
+    // Clear any existing animation
+    if (timerRef.current) {
+      clearInterval(timerRef.current);
+    }
+
+    iterationRef.current = 0;
+
+    timerRef.current = setInterval(() => {
+      const text = originalText;
+      const iteration = iterationRef.current;
+
+      const display = text
+        .split("")
+        .map((char, index) => {
+          if (index < iteration) return char;
+          return GLYPHS.charAt(Math.floor(Math.random() * GLYPHS.length));
+        })
+        .join("");
+
+      setDisplayText(display);
+      iterationRef.current += 1;
+
+      if (iteration > text.length + 2) {
+        if (timerRef.current) {
+          clearInterval(timerRef.current);
+          timerRef.current = null;
+        }
+        setDisplayText(originalText);
+      }
+    }, 26);
+  }, [originalText]);
+
+  const reset = useCallback(() => {
+    if (timerRef.current) {
+      clearInterval(timerRef.current);
+      timerRef.current = null;
+    }
+    setDisplayText(originalText);
+  }, [originalText]);
+
+  return { displayText, scramble, reset };
+}
diff --git a/makima/frontend/src/hooks/useWebSocket.ts b/makima/frontend/src/hooks/useWebSocket.ts
new file mode 100644
index 0000000..de6c1a6
--- /dev/null
+++ b/makima/frontend/src/hooks/useWebSocket.ts
@@ -0,0 +1,244 @@
+import { useState, useCallback, useRef, useEffect } from "react";
+import { LISTEN_ENDPOINT } from "../lib/api";
+import type {
+  ClientMessage,
+  ServerMessage,
+  TranscriptEntry,
+} from "../types/messages";
+
+export type ConnectionStatus =
+  | "disconnected"
+  | "connecting"
+  | "connected"
+  | "error";
+
+export interface WebSocketState {
+  status: ConnectionStatus;
+  sessionId: string | null;
+  error: string | null;
+  transcripts: TranscriptEntry[];
+}
+
+interface UseWebSocketOptions {
+  onReady?: (sessionId: string) => void;
+  onTranscript?: (transcript: TranscriptEntry) => void;
+  onError?: (code: string, message: string) => void;
+  onStopped?: (reason: string) => void;
+}
+
+export function useWebSocket(options: UseWebSocketOptions = {}) {
+  const { onReady, onTranscript, onError, onStopped } = options;
+
+  const [state, setState] = useState<WebSocketState>({
+    status: "disconnected",
+    sessionId: null,
+    error: null,
+    transcripts: [],
+  });
+
+  const wsRef = useRef<WebSocket | null>(null);
+  const transcriptIdRef = useRef(0);
+
+  // Store callbacks in refs to avoid recreating handlers
+  const callbacksRef = useRef({ onReady, onTranscript, onError, onStopped });
+  useEffect(() => {
+    callbacksRef.current = { onReady, onTranscript, onError, onStopped };
+  }, [onReady, onTranscript, onError, onStopped]);
+
+  const connect = useCallback((): Promise<boolean> => {
+    return new Promise((resolve) => {
+      if (wsRef.current?.readyState === WebSocket.OPEN) {
+        resolve(true);
+        return;
+      }
+
+      // Close any existing connection
+      if (wsRef.current) {
+        wsRef.current.close();
+        wsRef.current = null;
+      }
+
+      setState((s) => ({ ...s, status: "connecting", error: null }));
+
+      try {
+        const ws = new WebSocket(LISTEN_ENDPOINT);
+        wsRef.current = ws;
+
+        ws.onopen = () => {
+          setState((s) => ({ ...s, status: "connected", error: null }));
+          resolve(true);
+        };
+
+        ws.onmessage = (event) => {
+          try {
+            const message: ServerMessage = JSON.parse(event.data);
+
+            switch (message.type) {
+              case "ready":
+                setState((s) => ({ ...s, sessionId: message.sessionId }));
+                callbacksRef.current.onReady?.(message.sessionId);
+                break;
+
+              case "transcript": {
+                const entry: TranscriptEntry = {
+                  id: `t-${++transcriptIdRef.current}`,
+                  speaker: message.speaker,
+                  start: message.start,
+                  end: message.end,
+                  text: message.text,
+                  isFinal: message.isFinal,
+                };
+
+                setState((s) => {
+                  if (message.isFinal) {
+                    // Final transcript replaces all previous transcripts from this speaker
+                    const filtered = s.transcripts.filter(
+                      (t) => t.speaker !== message.speaker
+                    );
+                    return { ...s, transcripts: [...filtered, entry] };
+                  } else {
+                    // Non-final: replace if same speaker and overlapping time, otherwise append
+                    const existingIdx = s.transcripts.findIndex(
+                      (t) =>
+                        !t.isFinal &&
+                        t.speaker === message.speaker &&
+                        Math.abs(t.start - message.start) < 0.1
+                    );
+                    if (existingIdx >= 0) {
+                      const newTranscripts = [...s.transcripts];
+                      newTranscripts[existingIdx] = entry;
+                      return { ...s, transcripts: newTranscripts };
+                    }
+                    return { ...s, transcripts: [...s.transcripts, entry] };
+                  }
+                });
+
+                callbacksRef.current.onTranscript?.(entry);
+                break;
+              }
+
+              case "error":
+                setState((s) => ({ ...s, error: message.message }));
+                callbacksRef.current.onError?.(message.code, message.message);
+                break;
+
+              case "stopped":
+                setState((s) => ({ ...s, status: "disconnected" }));
+                callbacksRef.current.onStopped?.(message.reason);
+                break;
+            }
+          } catch {
+            console.error("Failed to parse WebSocket message:", event.data);
+          }
+        };
+
+        ws.onerror = () => {
+          setState((s) => ({
+            ...s,
+            status: "error",
+            error: "Failed to connect to server",
+          }));
+          resolve(false);
+        };
+
+        ws.onclose = (event) => {
+          // Check for specific close codes
+          let errorMessage: string | null = null;
+          if (event.code === 1006) {
+            errorMessage = "Connection failed - server may be unavailable";
+          } else if (event.code !== 1000 && event.code !== 1001) {
+            errorMessage = `Connection closed unexpectedly (code: ${event.code})`;
+          }
+
+          setState((s) => ({
+            ...s,
+            status: "disconnected",
+            sessionId: null,
+            error: errorMessage || s.error,
+          }));
+          wsRef.current = null;
+        };
+      } catch (err) {
+        const message = err instanceof Error ? err.message : "Failed to create WebSocket connection";
+        setState((s) => ({
+          ...s,
+          status: "error",
+          error: message,
+        }));
+        resolve(false);
+      }
+    });
+  }, []);
+
+  const disconnect = useCallback(() => {
+    if (wsRef.current) {
+      wsRef.current.close(1000, "User disconnected");
+      wsRef.current = null;
+    }
+    setState((s) => ({ ...s, status: "disconnected", sessionId: null }));
+  }, []);
+
+  const sendMessage = useCallback((message: ClientMessage) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify(message));
+    }
+  }, []);
+
+  const sendAudio = useCallback((samples: Float32Array) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      // Convert Float32Array to bytes (little-endian)
+      const bytes = new Uint8Array(samples.length * 4);
+      const view = new DataView(bytes.buffer);
+      for (let i = 0; i < samples.length; i++) {
+        view.setFloat32(i * 4, samples[i], true);
+      }
+      wsRef.current.send(bytes);
+    }
+  }, []);
+
+  const startSession = useCallback(
+    (sampleRate: number, channels: number = 1) => {
+      sendMessage({
+        type: "start",
+        sampleRate,
+        channels,
+        encoding: "pcm32f",
+      });
+    },
+    [sendMessage]
+  );
+
+  const stopSession = useCallback(
+    (reason?: string) => {
+      sendMessage({
+        type: "stop",
+        reason,
+      });
+    },
+    [sendMessage]
+  );
+
+  const clearTranscripts = useCallback(() => {
+    setState((s) => ({ ...s, transcripts: [], error: null }));
+  }, []);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      if (wsRef.current) {
+        wsRef.current.close();
+      }
+    };
+  }, []);
+
+  return {
+    ...state,
+    connect,
+    disconnect,
+    sendAudio,
+    startSession,
+    stopSession,
+    clearTranscripts,
+    isConnected: state.status === "connected",
+  };
+}