Human chat: combined media captions, composer drop, redaction & scroll

packages/core/src/matrix/rich-reply.ts

@@ -1,6 +1,6 @@
 /** Matrix rich reply plaintext helpers (Client-Server API — rich replies). */
 
-import { MatrixEventEvent, RelationType } from 'matrix-js-sdk';
+import { EventType, MatrixEventEvent, RelationType } from 'matrix-js-sdk';
 import type * as MatrixSdk from 'matrix-js-sdk';
 
 import type { Message, MessageMediaBundleItem } from './types';
@@ -18,7 +18,7 @@ export const HYPHA_SPOILER_FIELD = 'org.hypha.spoiler';
 export const HYPHA_MEDIA_BUNDLE_FIELD = 'org.hypha.media_bundle';
 
 export type HyphaMediaBundleItemWire = {
-  msgtype: 'm.file' | 'm.image';
+  msgtype: 'm.file' | 'm.image' | 'm.audio';
   url: string;
   body?: string;
   filename?: string;
@@ -257,6 +257,18 @@ function parseReplyFallbackFirstLine(body: string): {
 /**
  * Map a timeline `m.room.message` event to Hypha `Message` (reply metadata + display body).
  */
+/**
+ * Redacted `m.room.message` events still exist on the timeline; skip them so the UI
+ * does not render empty rows (avatar + “You” with no body).
+ */
+export function isRedactedRoomMessageEvent(
+  event: MatrixSdk.MatrixEvent,
+): boolean {
+  return (
+    event.getType() === EventType.RoomMessage && event.isRedacted() === true
+  );
+}
+
 export function messageFromRoomMessageEvent(
   client: MatrixSdk.MatrixClient,
   roomId: string,
@@ -279,7 +291,10 @@ export function messageFromRoomMessageEvent(
     [key: string]: unknown;
   };
   const msgtypeRaw = content.msgtype;
-  const isMedia = msgtypeRaw === 'm.file' || msgtypeRaw === 'm.image';
+  const isMedia =
+    msgtypeRaw === 'm.file' ||
+    msgtypeRaw === 'm.image' ||
+    msgtypeRaw === 'm.audio';
   const rawBody = content.body ?? '';
   const replyToId = event.getWireContent()?.['m.relates_to']?.['m.in_reply_to']
     ?.event_id as string | undefined;
@@ -318,13 +333,24 @@ export function messageFromRoomMessageEvent(
           content.formatted_body,
         );
       }
+    } else if (
+      content.format === MATRIX_CUSTOM_HTML_FORMAT &&
+      typeof content.formatted_body === 'string'
+    ) {
+      formattedContentHtml = extractReplyFormattedHtml(content.formatted_body);
     }
   } else if (
     !isMedia &&
     content.format === MATRIX_CUSTOM_HTML_FORMAT &&
     typeof content.formatted_body === 'string'
   ) {
     formattedContentHtml = content.formatted_body;
+  } else if (
+    isMedia &&
+    content.format === MATRIX_CUSTOM_HTML_FORMAT &&
+    typeof content.formatted_body === 'string'
+  ) {
+    formattedContentHtml = content.formatted_body;
   }
 
   const id = event.getId();
@@ -352,13 +378,16 @@ export function messageFromRoomMessageEvent(
         size: typeof o.size === 'number' ? o.size : undefined,
         w: typeof o.w === 'number' ? o.w : undefined,
         h: typeof o.h === 'number' ? o.h : undefined,
+        duration: typeof o.duration === 'number' ? o.duration : undefined,
       };
     };
 
     const parseBundleItem = (wire: unknown): MessageMediaBundleItem => {
       const w = wire as HyphaMediaBundleItemWire;
       const mt =
-        w.msgtype === 'm.file' || w.msgtype === 'm.image'
+        w.msgtype === 'm.file' ||
+        w.msgtype === 'm.image' ||
+        w.msgtype === 'm.audio'
           ? w.msgtype
           : 'm.file';
       const url =
@@ -383,7 +412,7 @@ export function messageFromRoomMessageEvent(
     let mediaBundle: Message['mediaBundle'];
     if (Array.isArray(bundleRaw) && bundleRaw.length > 0) {
       const first = {
-        msgtype: msgtypeRaw as 'm.file' | 'm.image',
+        msgtype: msgtypeRaw as 'm.file' | 'm.image' | 'm.audio',
         mxcUrl,
         filename:
           typeof content.filename === 'string'

packages/core/src/matrix/types.ts

@@ -55,11 +55,13 @@ export type MessageMediaInfo = {
   size?: number;
   w?: number;
   h?: number;
+  /** Matrix `m.audio` / MSC1767: duration in milliseconds */
+  duration?: number;
 };
 
 /** One slot in a multi-attachment `org.hypha.media_bundle` message. */
 export type MessageMediaBundleItem = {
-  msgtype: 'm.file' | 'm.image';
+  msgtype: 'm.file' | 'm.image' | 'm.audio';
   mxcUrl?: string;
   filename?: string;
   mediaInfo?: MessageMediaInfo;
@@ -69,8 +71,10 @@ export type MessageMediaBundleItem = {
 export interface Message {
   id: string;
   sender: string;
+  /** Synthetic row from a redaction handler: remove this timeline id from UI. */
+  redacted?: boolean;
   /** Matrix msgtype for timeline rendering (`m.text` is implicit when omitted). */
-  msgtype?: 'm.text' | 'm.file' | 'm.image';
+  msgtype?: 'm.text' | 'm.file' | 'm.image' | 'm.audio';
   /** Visible message text (reply fallback stripped when applicable). */
   content: string;
   /**

packages/epics/src/common/human-chat-panel/chat-panel-media-types.ts

@@ -2,18 +2,30 @@
  * Matrix timeline attachment slice shared by chat list, bubbles, and HumanRightPanel.
  */
 export type ChatPanelAttachmentMedia = {
-  msgtype: 'm.file' | 'm.image';
+  msgtype: 'm.file' | 'm.image' | 'm.audio';
   mxcUrl?: string;
   filename?: string;
   mediaInfo?: {
     mimetype?: string;
     size?: number;
     w?: number;
     h?: number;
+    duration?: number;
   };
   spoiler?: boolean;
 };
 
+const AUDIO_FILE_EXTENSIONS = new Set([
+  'ogg',
+  'oga',
+  'opus',
+  'mp3',
+  'm4a',
+  'aac',
+  'flac',
+  'wav',
+]);
+
 const VIDEO_FILE_EXTENSIONS = new Set([
   'mov',
   'mp4',
@@ -33,6 +45,19 @@ function extensionFromFileNameHint(name: string): string {
   return base.split('.').pop()?.toLowerCase() ?? '';
 }
 
+/** Local file draft: treat as voice/audio attachment (not video). */
+export function looksLikeAudioMimeOrName(
+  mimetype?: string,
+  filename?: string,
+): boolean {
+  const mt = mimetype?.toLowerCase() ?? '';
+  if (mt.startsWith('audio/')) return true;
+  const ext = extensionFromFileNameHint(filename ?? '');
+  if (!ext) return false;
+  if (ext === 'webm') return false;
+  return AUDIO_FILE_EXTENSIONS.has(ext);
+}
+
 /** Local file or Matrix `info.mimetype` + name (composer drafts, timeline). */
 export function looksLikeVideoMimeOrName(
   mimetype?: string,
@@ -54,3 +79,15 @@ export function isChatPanelVideoFile(
     media.filename ?? '',
   );
 }
+
+/** Matrix `m.audio` or audio-like `m.file` (voice clips often use m.file + audio/*). */
+export function isChatPanelAudioFile(
+  media: Pick<ChatPanelAttachmentMedia, 'msgtype' | 'mediaInfo' | 'filename'>,
+): boolean {
+  if (media.msgtype === 'm.audio') return true;
+  if (media.msgtype !== 'm.file') return false;
+  const mt = media.mediaInfo?.mimetype?.toLowerCase() ?? '';
+  if (mt.startsWith('audio/')) return true;
+  const ext = extensionFromFileNameHint(media.filename ?? '');
+  return AUDIO_FILE_EXTENSIONS.has(ext);
+}