From 30d839e729ec190aa5037a8cfe2c72dc6c59973a Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 09:23:30 -0700 Subject: [PATCH 01/24] Add HEIC canvas fallback for client-side processing When users upload HEIC images, try server processing first. If the server lacks HEIC support, fall back to the browser's native createImageBitmap() + OffscreenCanvas to decode and convert to JPEG. The original HEIC is sideloaded as the original, the JPEG as the scaled version, and sub-sizes are generated from the JPEG via the existing vips resize pipeline. PHP: bypass MIME type check for HEIC/HEIF uploads so files can be stored even when the server can't process them. JS: add HEIC detection in prepareItem(), canvas conversion utility, and HEIC-aware generateThumbnails() fallback. --- ...-gutenberg-rest-attachments-controller.php | 23 +- lib/media/load.php | 1 + packages/upload-media/src/canvas-utils.ts | 46 +++ packages/upload-media/src/store/constants.ts | 12 + .../upload-media/src/store/private-actions.ts | 282 ++++++++++++------ 5 files changed, 272 insertions(+), 92 deletions(-) create mode 100644 packages/upload-media/src/canvas-utils.ts diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 802c424db3edcf..44115a7982a01f 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -84,19 +84,36 @@ public function register_routes(): void { * Checks if a given request has access to create an attachment. * * Skips the server-side image type support check when the client - * will handle image processing (generate_sub_sizes is false). + * will handle image processing (generate_sub_sizes is false), or + * when the file is HEIC/HEIF (client-side canvas fallback handles + * processing when the server's image editor doesn't support them). * * @param WP_REST_Request $request Full details about the request. * @return true|WP_Error True if the request has access to create items, WP_Error object otherwise. */ public function create_item_permissions_check( $request ) { - if ( false === $request['generate_sub_sizes'] ) { + $bypass_mime_check = false === $request['generate_sub_sizes']; + + // Always allow HEIC/HEIF uploads through even if the server's image + // editor doesn't support them. The client-side canvas fallback will + // handle processing using the browser's native HEVC decoder. + if ( ! $bypass_mime_check ) { + $files = $request->get_file_params(); + if ( + ! empty( $files['file']['type'] ) && + in_array( $files['file']['type'], array( 'image/heic', 'image/heif' ), true ) + ) { + $bypass_mime_check = true; + } + } + + if ( $bypass_mime_check ) { add_filter( 'wp_prevent_unsupported_mime_type_uploads', '__return_false' ); } $result = parent::create_item_permissions_check( $request ); - if ( false === $request['generate_sub_sizes'] ) { + if ( $bypass_mime_check ) { remove_filter( 'wp_prevent_unsupported_mime_type_uploads', '__return_false' ); } diff --git a/lib/media/load.php b/lib/media/load.php index d628851a932697..bae118918a3146 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -51,6 +51,7 @@ function gutenberg_get_default_image_output_formats() { 'image/webp', 'image/avif', 'image/heic', + 'image/heif', ); $output_formats = array(); diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts new file mode 100644 index 00000000000000..63b1b600b82155 --- /dev/null +++ b/packages/upload-media/src/canvas-utils.ts @@ -0,0 +1,46 @@ +/** + * Internal dependencies + */ +import { getFileBasename } from './utils'; + +/** + * Converts an image file to JPEG using the browser's native decoder and canvas. + * + * Uses createImageBitmap() for decoding (leverages OS/browser-licensed HEVC codecs) + * and OffscreenCanvas for JPEG conversion. This avoids shipping our own HEVC decoder, + * sidestepping patent/licensing concerns. + * + * @param file Source image file (e.g., HEIC/HEIF). + * @param quality JPEG quality (0-1). Default 0.82. + * @return JPEG File object. + */ +export async function canvasConvertToJpeg( + file: File, + quality = 0.82 +): Promise< File > { + const bitmap = await createImageBitmap( file ); + + try { + const canvas = new OffscreenCanvas( bitmap.width, bitmap.height ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } + + ctx.drawImage( bitmap, 0, 0 ); + + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + const baseName = getFileBasename( file.name ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } finally { + bitmap.close(); + } +} diff --git a/packages/upload-media/src/store/constants.ts b/packages/upload-media/src/store/constants.ts index d0b6f5591387ae..d71b6c1ed62499 100644 --- a/packages/upload-media/src/store/constants.ts +++ b/packages/upload-media/src/store/constants.ts @@ -28,3 +28,15 @@ export const CLIENT_SIDE_SUPPORTED_MIME_TYPES: readonly string[] = [ 'image/webp', 'image/avif', ] as const; + +/** + * HEIC/HEIF MIME types. + * + * These formats use the HEVC codec which has patent/licensing restrictions. + * Instead of shipping our own decoder, the client falls back to the browser's + * native createImageBitmap() which leverages OS/browser-licensed HEVC codecs. + */ +export const HEIC_MIME_TYPES: readonly string[] = [ + 'image/heic', + 'image/heif', +] as const; diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index b06af25f2d6d43..f0e34728ed5c73 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -13,8 +13,14 @@ type WPDataRegistry = ReturnType< typeof createRegistry >; /** * Internal dependencies */ -import { cloneFile, convertBlobToFile, renameFile } from '../utils'; -import { CLIENT_SIDE_SUPPORTED_MIME_TYPES } from './constants'; +import { + cloneFile, + convertBlobToFile, + getFileBasename, + renameFile, +} from '../utils'; +import { canvasConvertToJpeg } from '../canvas-utils'; +import { CLIENT_SIDE_SUPPORTED_MIME_TYPES, HEIC_MIME_TYPES } from './constants'; import { StubFile } from '../stub-file'; import { UploadError } from '../upload-error'; import { @@ -735,6 +741,7 @@ export function prepareItem( id: QueueItemId ) { const isVipsSupported = CLIENT_SIDE_SUPPORTED_MIME_TYPES.includes( file.type ); + const isHeic = HEIC_MIME_TYPES.includes( file.type ); // For images that can be processed by vips, check if we need to scale down based on threshold. if ( isImage && isVipsSupported ) { @@ -754,6 +761,15 @@ export function prepareItem( id: QueueItemId ) { } } + operations.push( + OperationType.Upload, + OperationType.ThumbnailGeneration, + OperationType.Finalize + ); + } else if ( isImage && isHeic ) { + // HEIC/HEIF: upload with server processing first. + // If the server can't generate sub-sizes, the client falls back + // to canvas-based decoding using the browser's native HEVC codec. operations.push( OperationType.Upload, OperationType.ThumbnailGeneration, @@ -1082,46 +1098,129 @@ export function generateThumbnails( id: QueueItemId ) { return; } const attachment = item.attachment; + const settings = select.getSettings(); - // Check if image needs rotation. - // If exif_orientation is not 1, the image needs rotation. - // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. - const needsRotation = - attachment.exif_orientation && - attachment.exif_orientation !== 1 && - ! item.file.name.includes( '-scaled' ); - - // If rotation is needed for a non-scaled image, sideload the rotated version. - // This matches WordPress core's behavior of creating a -rotated version. - if ( needsRotation && attachment.id ) { + // HEIC/HEIF canvas fallback handling. + // When the source is HEIC, sideload the original and convert to JPEG + // using the browser's native decoder for sub-size generation. + const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); + let jpegConversion: File | null = null; + + if ( isHeicSource && attachment.id ) { + // Sideload the original HEIC to ensure it's tracked in metadata + // as original_image. The sideload response includes missing_image_sizes. + dispatch.addSideloadItem( { + file: item.sourceFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } + + if ( + isHeicSource && + attachment.missing_image_sizes && + attachment.missing_image_sizes.length > 0 + ) { + // Convert HEIC to JPEG using browser's native HEVC decoder. + // Uses createImageBitmap() which leverages OS/browser-licensed codecs, + // avoiding patent concerns with shipping our own HEVC decoder. try { - const rotatedFile = await vipsRotateImage( - item.id, + jpegConversion = await canvasConvertToJpeg( item.sourceFile, - attachment.exif_orientation as number, - item.abortController?.signal + settings.imageQuality ?? DEFAULT_OUTPUT_QUALITY ); + } catch { + dispatch.cancelItem( + id, + new UploadError( { + code: 'HEIC_DECODE_ERROR', + message: + 'This browser cannot decode HEIC images and the server does not support them either. Please convert to JPEG before uploading.', + file: item.sourceFile, + } ) + ); + return; + } + + // Sideload the full-size JPEG as the "scaled" version. + // This makes the JPEG the main displayable image while keeping + // the original HEIC stored as original_image in metadata. + if ( attachment.id ) { + const scaledFile = attachment.filename + ? renameFile( + jpegConversion, + getFileBasename( attachment.filename ) + '.jpeg' + ) + : jpegConversion; - // Sideload the rotated file as the "original" to set original_image metadata. - // The server will store this in $metadata['original_image']. dispatch.addSideloadItem( { - file: rotatedFile, + file: scaledFile, + onChange: ( [ updatedAttachment ] ) => { + if ( isBlobURL( updatedAttachment.url ) ) { + return; + } + item.onChange?.( [ updatedAttachment ] ); + }, batchId: uuidv4(), parentId: item.id, additionalData: { post: attachment.id, - image_size: 'original', + image_size: 'scaled', convert_format: false, }, operations: [ OperationType.Upload ], } ); - } catch { - // If rotation fails, continue with thumbnail generation. - // Thumbnails will still be rotated correctly by vips. - // eslint-disable-next-line no-console - console.warn( - 'Failed to rotate image, continuing with thumbnails' - ); + } + } + + // Check if image needs rotation (non-HEIC images only). + // If exif_orientation is not 1, the image needs rotation. + // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. + // HEIC rotation is handled by the browser's native decoder. + if ( ! isHeicSource ) { + const needsRotation = + attachment.exif_orientation && + attachment.exif_orientation !== 1 && + ! item.file.name.includes( '-scaled' ); + + // If rotation is needed for a non-scaled image, sideload the rotated version. + // This matches WordPress core's behavior of creating a -rotated version. + if ( needsRotation && attachment.id ) { + try { + const rotatedFile = await vipsRotateImage( + item.id, + item.sourceFile, + attachment.exif_orientation as number, + item.abortController?.signal + ); + + // Sideload the rotated file as the "original" to set original_image metadata. + // The server will store this in $metadata['original_image']. + dispatch.addSideloadItem( { + file: rotatedFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } catch { + // If rotation fails, continue with thumbnail generation. + // Thumbnails will still be rotated correctly by vips. + // eslint-disable-next-line no-console + console.warn( + 'Failed to rotate image, continuing with thumbnails' + ); + } } } @@ -1131,17 +1230,17 @@ export function generateThumbnails( id: QueueItemId ) { attachment.missing_image_sizes && attachment.missing_image_sizes.length > 0 ) { - const settings = select.getSettings(); const allImageSizes = settings.allImageSizes || {}; const sizesToGenerate: string[] = attachment.missing_image_sizes as string[]; - // Use sourceFile for thumbnail generation to preserve quality. - // WordPress core generates thumbnails from the original (unscaled) image. - // Vips will auto-rotate based on EXIF orientation during thumbnail generation. + // Use the JPEG conversion for HEIC images, or the sourceFile otherwise. + // For HEIC, vips can resize the JPEG efficiently. + // For other formats, vips auto-rotates based on EXIF orientation. + const thumbnailSource = jpegConversion ?? item.sourceFile; const file = attachment.filename - ? renameFile( item.sourceFile, attachment.filename ) - : item.sourceFile; + ? renameFile( thumbnailSource, attachment.filename ) + : thumbnailSource; const batchId = uuidv4(); const { imageOutputFormats } = settings; @@ -1149,7 +1248,7 @@ export function generateThumbnails( id: QueueItemId ) { // Check if thumbnails should be transcoded to a different format. // Uses the same transparency-aware logic as the main image // to avoid converting transparent PNGs to JPEG. - const sourceType = item.sourceFile.type; + const sourceType = thumbnailSource.type; const outputMimeType = imageOutputFormats?.[ sourceType ]; let thumbnailTranscodeOperation: @@ -1161,7 +1260,7 @@ export function generateThumbnails( id: QueueItemId ) { if ( outputMimeType && outputMimeType !== sourceType ) { thumbnailTranscodeOperation = await getTranscodeImageOperation( - item.sourceFile, + thumbnailSource, outputMimeType, settings ); @@ -1217,62 +1316,67 @@ export function generateThumbnails( id: QueueItemId ) { } ); } - // Create and sideload the scaled version. - const { bigImageSizeThreshold } = settings; - if ( bigImageSizeThreshold && attachment.id ) { - // Check if the image actually exceeds the threshold. - // Only create a scaled version for images larger than the threshold, - // matching WordPress core's wp_create_image_subsizes() behavior. - const bitmap = await createImageBitmap( item.sourceFile ); - const needsScaling = - bitmap.width > bigImageSizeThreshold || - bitmap.height > bigImageSizeThreshold; - bitmap.close(); - - if ( needsScaling ) { - // Rename sourceFile to match the server attachment filename. - const sourceForScaled = attachment.filename - ? renameFile( item.sourceFile, attachment.filename ) - : item.sourceFile; - - // Add scaling to queue. - const scaledOperations: Operation[] = [ - [ - OperationType.ResizeCrop, - { - resize: { - width: bigImageSizeThreshold, - height: bigImageSizeThreshold, + // Create and sideload the scaled version (non-HEIC only). + // For HEIC, the scaled JPEG version is already sideloaded above. + if ( ! isHeicSource ) { + const { bigImageSizeThreshold } = settings; + if ( bigImageSizeThreshold && attachment.id ) { + // Check if the image actually exceeds the threshold. + // Only create a scaled version for images larger than the threshold, + // matching WordPress core's wp_create_image_subsizes() behavior. + const bitmap = await createImageBitmap( thumbnailSource ); + const needsScaling = + bitmap.width > bigImageSizeThreshold || + bitmap.height > bigImageSizeThreshold; + bitmap.close(); + + if ( needsScaling ) { + // Rename sourceFile to match the server attachment filename. + const sourceForScaled = attachment.filename + ? renameFile( thumbnailSource, attachment.filename ) + : thumbnailSource; + + // Add scaling to queue. + const scaledOperations: Operation[] = [ + [ + OperationType.ResizeCrop, + { + resize: { + width: bigImageSizeThreshold, + height: bigImageSizeThreshold, + }, + isThresholdResize: true, }, - isThresholdResize: true, - }, - ], - ]; - - // Add transcoding if format conversion is configured. - if ( thumbnailTranscodeOperation ) { - scaledOperations.push( thumbnailTranscodeOperation ); - } + ], + ]; + + // Add transcoding if format conversion is configured. + if ( thumbnailTranscodeOperation ) { + scaledOperations.push( + thumbnailTranscodeOperation + ); + } - scaledOperations.push( OperationType.Upload ); + scaledOperations.push( OperationType.Upload ); - dispatch.addSideloadItem( { - file: sourceForScaled, - onChange: ( [ updatedAttachment ] ) => { - if ( isBlobURL( updatedAttachment.url ) ) { - return; - } - item.onChange?.( [ updatedAttachment ] ); - }, - batchId, - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'scaled', - convert_format: false, - }, - operations: scaledOperations, - } ); + dispatch.addSideloadItem( { + file: sourceForScaled, + onChange: ( [ updatedAttachment ] ) => { + if ( isBlobURL( updatedAttachment.url ) ) { + return; + } + item.onChange?.( [ updatedAttachment ] ); + }, + batchId, + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'scaled', + convert_format: false, + }, + operations: scaledOperations, + } ); + } } } } From 75bb70859402b28205786b94e0ff425bfc00a577 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 10:00:18 -0700 Subject: [PATCH 02/24] Add backport changelog entry for HEIC canvas fallback Links core PR wordpress-develop#11323 to Gutenberg PR #76731. --- backport-changelog/7.1/11323.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 backport-changelog/7.1/11323.md diff --git a/backport-changelog/7.1/11323.md b/backport-changelog/7.1/11323.md new file mode 100644 index 00000000000000..fa30d7359b6685 --- /dev/null +++ b/backport-changelog/7.1/11323.md @@ -0,0 +1,3 @@ +https://github.com/WordPress/wordpress-develop/pull/11323 + +* https://github.com/WordPress/gutenberg/pull/76731 From d329759e4d55d2039e5882f588c26ca868bf1ed6 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 15:37:17 -0700 Subject: [PATCH 03/24] Fix HEIC canvas fallback for Chrome on macOS Three issues prevented the HEIC canvas fallback from working: 1. generate_sub_sizes was set to true for HEIC uploads, causing the server to attempt (and silently fail) sub-size generation instead of letting the client handle it. 2. The server can't read HEIC files, so it always returned an empty missing_image_sizes array. The client now derives missing sizes from registered image sizes in settings. 3. createImageBitmap() doesn't support HEIC in Chrome. Added a fallback using element + HTMLCanvasElement, which works on macOS because Chrome exposes OS-level HEIC decoding through the img rendering pipeline. --- packages/upload-media/src/canvas-utils.ts | 85 ++++++++++++++++--- .../upload-media/src/store/private-actions.ts | 16 +++- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 63b1b600b82155..3243fdcf52694f 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -6,9 +6,13 @@ import { getFileBasename } from './utils'; /** * Converts an image file to JPEG using the browser's native decoder and canvas. * - * Uses createImageBitmap() for decoding (leverages OS/browser-licensed HEVC codecs) - * and OffscreenCanvas for JPEG conversion. This avoids shipping our own HEVC decoder, - * sidestepping patent/licensing concerns. + * Tries two decoding strategies: + * 1. createImageBitmap() + OffscreenCanvas (works in Safari, future Chrome). + * 2. element + HTMLCanvasElement (works in Chrome on macOS, which + * exposes OS-level HEIC decoding through the rendering pipeline + * but not through createImageBitmap). + * + * This avoids shipping our own HEVC decoder, sidestepping patent/licensing concerns. * * @param file Source image file (e.g., HEIC/HEIF). * @param quality JPEG quality (0-1). Default 0.82. @@ -18,29 +22,84 @@ export async function canvasConvertToJpeg( file: File, quality = 0.82 ): Promise< File > { - const bitmap = await createImageBitmap( file ); + const baseName = getFileBasename( file.name ); + // Strategy 1: createImageBitmap + OffscreenCanvas. try { - const canvas = new OffscreenCanvas( bitmap.width, bitmap.height ); - const ctx = canvas.getContext( '2d' ); + const bitmap = await createImageBitmap( file ); + try { + const canvas = new OffscreenCanvas( bitmap.width, bitmap.height ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } + + ctx.drawImage( bitmap, 0, 0 ); + + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } finally { + bitmap.close(); + } + } catch { + // createImageBitmap doesn't support HEIC in this browser. + // Fall through to strategy 2. + } + // Strategy 2: element + HTMLCanvasElement. + // Chrome on macOS can decode HEIC via the rendering pipeline + // using OS-level codecs, even though createImageBitmap cannot. + const blobUrl = URL.createObjectURL( file ); + + try { + const img = await new Promise< HTMLImageElement >( + ( resolve, reject ) => { + const image = new Image(); + image.onload = () => resolve( image ); + image.onerror = () => + reject( + new Error( 'Image element could not decode the file' ) + ); + image.src = blobUrl; + } + ); + + const canvas = document.createElement( 'canvas' ); + canvas.width = img.naturalWidth; + canvas.height = img.naturalHeight; + + const ctx = canvas.getContext( '2d' ); if ( ! ctx ) { throw new Error( 'Could not get canvas 2d context' ); } - ctx.drawImage( bitmap, 0, 0 ); + ctx.drawImage( img, 0, 0 ); - const jpegBlob = await canvas.convertToBlob( { - type: 'image/jpeg', - quality, + const jpegBlob = await new Promise< Blob >( ( resolve, reject ) => { + canvas.toBlob( + ( blob ) => { + if ( blob ) { + resolve( blob ); + } else { + reject( new Error( 'Canvas toBlob returned null' ) ); + } + }, + 'image/jpeg', + quality + ); } ); - const baseName = getFileBasename( file.name ); - return new File( [ jpegBlob ], `${ baseName }.jpeg`, { type: 'image/jpeg', } ); } finally { - bitmap.close(); + URL.revokeObjectURL( blobUrl ); } } diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index f0e34728ed5c73..832571c29b7b9a 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -787,8 +787,10 @@ export function prepareItem( id: QueueItemId ) { // If the file is not processed by vips, tell the server to // generate sub-sizes since they won't be created client-side. + // Exception: HEIC images — the client handles sub-sizes via + // canvas fallback, so tell the server NOT to generate them. const updates = - ! isVipsSupported || ! isImage + ( ! isVipsSupported && ! isHeic ) || ! isImage ? { additionalData: { ...item.additionalData, @@ -1106,6 +1108,18 @@ export function generateThumbnails( id: QueueItemId ) { const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); let jpegConversion: File | null = null; + // The server can't read HEIC files, so missing_image_sizes will be + // empty even though no sub-sizes were generated. Derive the list + // from the registered image sizes in settings instead. + if ( + isHeicSource && + ( ! attachment.missing_image_sizes || + attachment.missing_image_sizes.length === 0 ) + ) { + const allImageSizes = settings.allImageSizes || {}; + attachment.missing_image_sizes = Object.keys( allImageSizes ); + } + if ( isHeicSource && attachment.id ) { // Sideload the original HEIC to ensure it's tracked in metadata // as original_image. The sideload response includes missing_image_sizes. From 60a9d107d122bfb03c02a0888172b95a6e61495a Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 17:39:33 -0700 Subject: [PATCH 04/24] Add HEIC decoding for Chrome on macOS via VideoDecoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chrome doesn't expose macOS's native HEIC decoder through image APIs (createImageBitmap, ImageDecoder, ). However, Chrome 107+ supports HEVC video decoding via the WebCodecs VideoDecoder API using macOS VideoToolbox. This adds a new decoding strategy that bridges the gap: 1. Parse the HEIC/ISOBMFF container in pure JS to extract the HEVC decoder configuration (hvcC) and compressed image data 2. Feed the HEVC frames through VideoDecoder (hardware-accelerated) 3. Composite decoded tiles onto a canvas and convert to JPEG Handles both single-image and grid/tiled HEIC files (the common iPhone format where photos are split into multiple HEVC tiles). No WASM decoder or patented codec is shipped — only the standard container format is parsed, and decoding uses Chrome's already-licensed platform decoder. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...-gutenberg-rest-attachments-controller.php | 18 + packages/upload-media/src/canvas-utils.ts | 191 ++++- packages/upload-media/src/heic-parser.ts | 769 ++++++++++++++++++ .../upload-media/src/store/private-actions.ts | 12 - 4 files changed, 936 insertions(+), 54 deletions(-) create mode 100644 packages/upload-media/src/heic-parser.ts diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 44115a7982a01f..9bda193bd7c0c1 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -241,6 +241,24 @@ public function prepare_item_for_response( $item, $request ): WP_REST_Response { $missing_image_sizes = array_diff( $merged_sizes, array_keys( $metadata['sizes'] ) ); $data['missing_image_sizes'] = $missing_image_sizes; } + + // HEIC/HEIF: the server's image editor cannot read these files, + // so missing_image_sizes is empty even though no sub-sizes were + // generated. Report all registered sizes as missing so the + // client-side canvas fallback can generate them. + if ( in_array( $mime_type, array( 'image/heic', 'image/heif' ), true ) ) { + $metadata = wp_get_attachment_metadata( $item->ID, true ); + + if ( ! is_array( $metadata ) ) { + $metadata = array(); + } + + $metadata['sizes'] = $metadata['sizes'] ?? array(); + + $registered_sizes = wp_get_registered_image_subsizes(); + $missing_image_sizes = array_diff( array_keys( $registered_sizes ), array_keys( $metadata['sizes'] ) ); + $data['missing_image_sizes'] = array_values( $missing_image_sizes ); + } } $context = ! empty( $request['context'] ) ? $request['context'] : 'view'; diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 3243fdcf52694f..232b209003abf1 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -2,15 +2,19 @@ * Internal dependencies */ import { getFileBasename } from './utils'; +import { parseHeic } from './heic-parser'; /** * Converts an image file to JPEG using the browser's native decoder and canvas. * - * Tries two decoding strategies: + * Tries three decoding strategies: * 1. createImageBitmap() + OffscreenCanvas (works in Safari, future Chrome). - * 2. element + HTMLCanvasElement (works in Chrome on macOS, which - * exposes OS-level HEIC decoding through the rendering pipeline - * but not through createImageBitmap). + * 2. WebCodecs ImageDecoder API (uses platform codecs; may work in future + * Chrome if HEIC is added to its image decoder pipeline). + * 3. HEIC container parsing + WebCodecs VideoDecoder (Chrome 107+ on macOS). + * Parses the HEIC/ISOBMFF container to extract the HEVC bitstream, then + * decodes it using Chrome's platform HEVC video decoder (hardware- + * accelerated via macOS VideoToolbox). * * This avoids shipping our own HEVC decoder, sidestepping patent/licensing concerns. * @@ -53,53 +57,156 @@ export async function canvasConvertToJpeg( // Fall through to strategy 2. } - // Strategy 2: element + HTMLCanvasElement. - // Chrome on macOS can decode HEIC via the rendering pipeline - // using OS-level codecs, even though createImageBitmap cannot. - const blobUrl = URL.createObjectURL( file ); + // Strategy 2: WebCodecs ImageDecoder API. + // Uses platform codecs (e.g., macOS HEIC support) that may not be + // exposed through createImageBitmap or elements. + if ( typeof ImageDecoder !== 'undefined' ) { + const supported = await ImageDecoder.isTypeSupported( file.type ); + if ( supported ) { + const decoder = new ImageDecoder( { + type: file.type, + data: file.stream(), + } ); + const { image: videoFrame } = await decoder.decode(); - try { - const img = await new Promise< HTMLImageElement >( - ( resolve, reject ) => { - const image = new Image(); - image.onload = () => resolve( image ); - image.onerror = () => - reject( - new Error( 'Image element could not decode the file' ) - ); - image.src = blobUrl; + const canvas = new OffscreenCanvas( + videoFrame.displayWidth, + videoFrame.displayHeight + ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + videoFrame.close(); + decoder.close(); + throw new Error( 'Could not get canvas 2d context' ); } - ); - const canvas = document.createElement( 'canvas' ); - canvas.width = img.naturalWidth; - canvas.height = img.naturalHeight; + ctx.drawImage( videoFrame, 0, 0 ); + videoFrame.close(); + decoder.close(); - const ctx = canvas.getContext( '2d' ); - if ( ! ctx ) { - throw new Error( 'Could not get canvas 2d context' ); + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); } + } + + // Strategy 3: HEIC container parsing + WebCodecs VideoDecoder. + // Chrome 107+ on macOS supports HEVC *video* decoding via platform codecs + // (macOS VideoToolbox), even though it doesn't support HEIC through image + // APIs. A HEIC file is an ISOBMFF container with HEVC-encoded tiles — + // we parse the container and decode each tile via VideoDecoder. + if ( typeof VideoDecoder !== 'undefined' ) { + try { + const heicData = parseHeic( await file.arrayBuffer() ); + + const support = await VideoDecoder.isConfigSupported( { + codec: heicData.codecString, + } ); + + if ( support.supported ) { + const canvas = new OffscreenCanvas( + heicData.outputWidth, + heicData.outputHeight + ); + const ctx = canvas.getContext( '2d' ); - ctx.drawImage( img, 0, 0 ); + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } - const jpegBlob = await new Promise< Blob >( ( resolve, reject ) => { - canvas.toBlob( - ( blob ) => { - if ( blob ) { - resolve( blob ); - } else { - reject( new Error( 'Canvas toBlob returned null' ) ); + // Decode each tile and draw it at its grid position. + for ( const tile of heicData.tiles ) { + const frame = await decodeHevcFrame( + heicData.codecString, + heicData.description, + heicData.tileWidth, + heicData.tileHeight, + tile.data + ); + try { + ctx.drawImage( frame, tile.x, tile.y ); + } finally { + frame.close(); } - }, - 'image/jpeg', - quality - ); + } + + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } + } catch { + // VideoDecoder HEVC not available or HEIC parsing failed. + // Fall through to error. + } + } + + throw new Error( + 'This browser cannot decode HEIC images. Please use Safari or convert to JPEG before uploading.' + ); +} + +/** + * Decode a single HEVC key frame using the WebCodecs VideoDecoder API. + * + * @param codec HEVC codec string (e.g. 'hvc1.1.6.L93.B0'). + * @param description HEVCDecoderConfigurationRecord bytes. + * @param width Coded width of the frame. + * @param height Coded height of the frame. + * @param data Raw HEVC bitstream (IDR frame). + * @return Decoded VideoFrame. Caller must call frame.close(). + */ +function decodeHevcFrame( + codec: string, + description: Uint8Array, + width: number, + height: number, + data: Uint8Array +): Promise< VideoFrame > { + return new Promise< VideoFrame >( ( resolve, reject ) => { + const decoder = new VideoDecoder( { + output: ( frame ) => { + decoder.close(); + resolve( frame ); + }, + error: ( e ) => { + if ( decoder.state !== 'closed' ) { + decoder.close(); + } + reject( e ); + }, } ); - return new File( [ jpegBlob ], `${ baseName }.jpeg`, { - type: 'image/jpeg', + decoder.configure( { + codec, + codedWidth: width, + codedHeight: height, + description, } ); - } finally { - URL.revokeObjectURL( blobUrl ); - } + + decoder.decode( + new EncodedVideoChunk( { + type: 'key', + timestamp: 0, + data, + } ) + ); + + decoder.flush().catch( ( e ) => { + if ( decoder.state !== 'closed' ) { + decoder.close(); + } + reject( e ); + } ); + } ); } diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts new file mode 100644 index 00000000000000..a4aea7b9aad36e --- /dev/null +++ b/packages/upload-media/src/heic-parser.ts @@ -0,0 +1,769 @@ +/* eslint-disable no-bitwise */ + +/** + * Lightweight HEIC (ISOBMFF) container parser. + * + * Extracts the HEVC decoder configuration and compressed image data from + * HEIC files, including grid/tiled images (the common iPhone format). + * This enables decoding via the WebCodecs VideoDecoder API without + * shipping our own HEVC codec. + * + * Only the container format is parsed here (no patented codec involved). + * Actual HEVC decoding is delegated to the browser's platform decoder. + */ + +/** A single tile to decode and draw. */ +export interface HeicTile { + /** Raw HEVC bitstream for this tile. */ + data: Uint8Array; + /** X position on the output canvas. */ + x: number; + /** Y position on the output canvas. */ + y: number; +} + +export interface HeicImageData { + /** HEVC codec string for VideoDecoder (e.g. 'hvc1.1.6.L93.B0'). */ + codecString: string; + /** Raw HEVCDecoderConfigurationRecord bytes for VideoDecoder description. */ + description: Uint8Array; + /** Tiles to decode (1 for single-image, many for grid). */ + tiles: HeicTile[]; + /** Width of each HEVC tile in pixels. */ + tileWidth: number; + /** Height of each HEVC tile in pixels. */ + tileHeight: number; + /** Final output width in pixels. */ + outputWidth: number; + /** Final output height in pixels. */ + outputHeight: number; +} + +// --------------------------------------------------------------------------- +// Binary reader +// --------------------------------------------------------------------------- + +class Reader { + readonly view: DataView; + readonly buffer: ArrayBuffer; + pos: number; + + constructor( buffer: ArrayBuffer, offset = 0 ) { + this.buffer = buffer; + this.view = new DataView( buffer ); + this.pos = offset; + } + + u8(): number { + const v = this.view.getUint8( this.pos ); + this.pos += 1; + return v; + } + + u16(): number { + const v = this.view.getUint16( this.pos ); + this.pos += 2; + return v; + } + + u32(): number { + const v = this.view.getUint32( this.pos ); + this.pos += 4; + return v; + } + + u64(): number { + const hi = this.view.getUint32( this.pos ); + const lo = this.view.getUint32( this.pos + 4 ); + this.pos += 8; + return hi * 0x100000000 + lo; + } + + /** + * Read a variable-width unsigned integer (0, 4 or 8 bytes). + * + * @param size Byte width to read (0, 4, or 8). + */ + uN( size: number ): number { + if ( size === 0 ) { + return 0; + } + if ( size === 4 ) { + return this.u32(); + } + if ( size === 8 ) { + return this.u64(); + } + throw new Error( `Unsupported uint size: ${ size }` ); + } + + str( len: number ): string { + let s = ''; + for ( let i = 0; i < len; i++ ) { + s += String.fromCharCode( this.view.getUint8( this.pos + i ) ); + } + this.pos += len; + return s; + } + + bytes( len: number ): Uint8Array { + const b = new Uint8Array( this.buffer, this.pos, len ); + this.pos += len; + return new Uint8Array( b ); // copy to avoid detach issues + } +} + +// --------------------------------------------------------------------------- +// ISOBMFF box helpers +// --------------------------------------------------------------------------- + +interface BoxInfo { + type: string; + offset: number; + size: number; + headerSize: number; +} + +function readBox( r: Reader ): BoxInfo | null { + if ( r.pos + 8 > r.view.byteLength ) { + return null; + } + const offset = r.pos; + let size: number = r.u32(); + const type = r.str( 4 ); + let headerSize = 8; + + if ( size === 1 ) { + size = r.u64(); + headerSize = 16; + } else if ( size === 0 ) { + size = r.view.byteLength - offset; + } + + return { type, offset, size, headerSize }; +} + +function findBoxes( r: Reader, start: number, end: number ): BoxInfo[] { + const boxes: BoxInfo[] = []; + r.pos = start; + while ( r.pos < end ) { + const box = readBox( r ); + if ( ! box || box.size < 8 ) { + break; + } + boxes.push( box ); + r.pos = box.offset + box.size; + } + return boxes; +} + +function findBox( + r: Reader, + start: number, + end: number, + type: string +): BoxInfo | undefined { + r.pos = start; + while ( r.pos < end ) { + const box = readBox( r ); + if ( ! box || box.size < 8 ) { + break; + } + if ( box.type === type ) { + return box; + } + r.pos = box.offset + box.size; + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Specific box parsers +// --------------------------------------------------------------------------- + +/** + * Parse Primary Item Box → primary item ID. + * + * @param r Binary reader. + * @param box BoxInfo for the pitm box. + */ +function parsePitm( r: Reader, box: BoxInfo ): number { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + return version === 0 ? r.u16() : r.u32(); +} + +interface ItemExtent { + offset: number; + length: number; +} + +interface ItemLocation { + extents: ItemExtent[]; +} + +/** + * Parse Item Location Box → map of item ID to data extents. + * + * @param r Binary reader. + * @param box BoxInfo for the iloc box. + */ +function parseIloc( r: Reader, box: BoxInfo ): Map< number, ItemLocation > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const byte1 = r.u8(); + const offsetSize = ( byte1 >> 4 ) & 0xf; + const lengthSize = byte1 & 0xf; + + const byte2 = r.u8(); + const baseOffsetSize = ( byte2 >> 4 ) & 0xf; + const indexSize = version >= 1 ? byte2 & 0xf : 0; + + const itemCount = version < 2 ? r.u16() : r.u32(); + const items = new Map< number, ItemLocation >(); + + for ( let i = 0; i < itemCount; i++ ) { + const itemId = version < 2 ? r.u16() : r.u32(); + + if ( version === 1 || version === 2 ) { + r.u16(); // construction_method + reserved + } + + r.u16(); // data_reference_index + const baseOffset = r.uN( baseOffsetSize ); + const extentCount = r.u16(); + const extents: ItemExtent[] = []; + + for ( let j = 0; j < extentCount; j++ ) { + if ( version >= 1 ) { + r.uN( indexSize ); // extent_index — skip + } + const extOffset = r.uN( offsetSize ); + const extLength = r.uN( lengthSize ); + extents.push( { + offset: baseOffset + extOffset, + length: extLength, + } ); + } + + items.set( itemId, { extents } ); + } + + return items; +} + +/** + * Parse Item Property Association Box → map of item ID to 1-based property indices. + * + * @param r Binary reader. + * @param box BoxInfo for the ipma box. + */ +function parseIpma( r: Reader, box: BoxInfo ): Map< number, number[] > { + r.pos = box.offset + box.headerSize; + const vf = r.u32(); // version (8 bits) + flags (24 bits) + const version = vf >>> 24; + const flags = vf & 0xffffff; + const largeIndex = ( flags & 1 ) !== 0; + + const entryCount = r.u32(); + const associations = new Map< number, number[] >(); + + for ( let i = 0; i < entryCount; i++ ) { + const itemId = version < 1 ? r.u16() : r.u32(); + const assocCount = r.u8(); + const indices: number[] = []; + + for ( let j = 0; j < assocCount; j++ ) { + if ( largeIndex ) { + indices.push( r.u16() & 0x7fff ); // strip essential bit + } else { + indices.push( r.u8() & 0x7f ); // strip essential bit + } + } + + associations.set( itemId, indices ); + } + + return associations; +} + +/** + * Parse Image Spatial Extents → width & height. + * + * @param r Binary reader. + * @param box BoxInfo for the ispe box. + */ +function parseIspe( + r: Reader, + box: BoxInfo +): { width: number; height: number } { + // ispe is a FullBox: skip version (1) + flags (3). + r.pos = box.offset + box.headerSize + 4; + return { width: r.u32(), height: r.u32() }; +} + +/** + * Parse Item Info Box → map of item ID to item type (4-char code). + * + * @param r Binary reader. + * @param box BoxInfo for the iinf box. + */ +function parseIinf( r: Reader, box: BoxInfo ): Map< number, string > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const entryCount = version === 0 ? r.u16() : r.u32(); + const itemTypes = new Map< number, string >(); + + // Parse infe (ItemInfoEntry) sub-boxes. + const entriesStart = r.pos; + const boxEnd = box.offset + box.size; + + const infeBoxes = findBoxes( r, entriesStart, boxEnd ); + for ( let i = 0; i < Math.min( entryCount, infeBoxes.length ); i++ ) { + const infe = infeBoxes[ i ]; + if ( infe.type !== 'infe' ) { + continue; + } + r.pos = infe.offset + infe.headerSize; + const infeVersion = r.u8(); + r.pos += 3; // flags + + if ( infeVersion >= 2 ) { + const itemId = infeVersion === 2 ? r.u16() : r.u32(); + r.u16(); // item_protection_index + const itemType = r.str( 4 ); + itemTypes.set( itemId, itemType ); + } + } + + return itemTypes; +} + +/** + * Parse Item Reference Box → map of (fromItemId) to array of referenced item IDs, + * filtered by reference type. + * + * @param r Binary reader. + * @param box BoxInfo for the iref box. + * @param refType Reference type to filter (e.g. 'dimg'). + */ +function parseIref( + r: Reader, + box: BoxInfo, + refType: string +): Map< number, number[] > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const refs = new Map< number, number[] >(); + const boxEnd = box.offset + box.size; + + while ( r.pos < boxEnd ) { + const refBox = readBox( r ); + if ( ! refBox || refBox.size < 8 ) { + break; + } + + r.pos = refBox.offset + refBox.headerSize; + const fromId = version === 0 ? r.u16() : r.u32(); + const refCount = r.u16(); + const toIds: number[] = []; + + for ( let i = 0; i < refCount; i++ ) { + toIds.push( version === 0 ? r.u16() : r.u32() ); + } + + if ( refBox.type === refType ) { + refs.set( fromId, toIds ); + } + + r.pos = refBox.offset + refBox.size; + } + + return refs; +} + +// --------------------------------------------------------------------------- +// HEVC codec string construction +// --------------------------------------------------------------------------- + +/** + * Reverse all 32 bits of a number. + * + * @param n 32-bit unsigned integer. + */ +function reverseBits32( n: number ): number { + n = ( ( n >>> 1 ) & 0x55555555 ) | ( ( n & 0x55555555 ) << 1 ); + n = ( ( n >>> 2 ) & 0x33333333 ) | ( ( n & 0x33333333 ) << 2 ); + n = ( ( n >>> 4 ) & 0x0f0f0f0f ) | ( ( n & 0x0f0f0f0f ) << 4 ); + n = ( ( n >>> 8 ) & 0x00ff00ff ) | ( ( n & 0x00ff00ff ) << 8 ); + n = ( n >>> 16 ) | ( n << 16 ); + return n >>> 0; +} + +/** + * Build an HEVC codec string from an HEVCDecoderConfigurationRecord. + * + * Format: hvc1.{profile}.{compat}.{tier}{level}[.{constraints}] + * See ISO 14496-15 Annex E and W3C WebCodecs HEVC Codec Registration. + * + * @param r Binary reader. + * @param recordOffset Byte offset of the HEVCDecoderConfigurationRecord. + */ +function buildCodecString( r: Reader, recordOffset: number ): string { + r.pos = recordOffset; + r.u8(); // configurationVersion + + const byte1 = r.u8(); + const profileSpace = ( byte1 >> 6 ) & 0x3; + const tierFlag = ( byte1 >> 5 ) & 0x1; + const profileIdc = byte1 & 0x1f; + + const compatFlags = r.u32(); + const constraintBytes = r.bytes( 6 ); + const levelIdc = r.u8(); + + // Profile: optional space prefix (A/B/C) + profile_idc. + const spacePrefix = + profileSpace > 0 ? String.fromCharCode( 64 + profileSpace ) : ''; + + // Compatibility flags: bit-reversed, as hex. + const compatHex = reverseBits32( compatFlags ).toString( 16 ).toUpperCase(); + + // Tier: 'L' (Main) or 'H' (High). + const tierChar = tierFlag ? 'H' : 'L'; + + // Constraint indicator flags: each byte as hex, trailing zeros removed. + let lastNonZero = -1; + for ( let i = 5; i >= 0; i-- ) { + if ( constraintBytes[ i ] !== 0 ) { + lastNonZero = i; + break; + } + } + let constraintStr = ''; + if ( lastNonZero >= 0 ) { + const parts: string[] = []; + for ( let i = 0; i <= lastNonZero; i++ ) { + parts.push( constraintBytes[ i ].toString( 16 ).toUpperCase() ); + } + constraintStr = '.' + parts.join( '.' ); + } + + return `hvc1.${ spacePrefix }${ profileIdc }.${ compatHex }.${ tierChar }${ levelIdc }${ constraintStr }`; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Read item data by concatenating all extents for a given item location. + * + * @param buffer File ArrayBuffer. + * @param loc Item location with extents. + */ +function readItemData( buffer: ArrayBuffer, loc: ItemLocation ): Uint8Array { + if ( loc.extents.length === 1 ) { + const ext = loc.extents[ 0 ]; + return new Uint8Array( + buffer.slice( ext.offset, ext.offset + ext.length ) + ); + } + let totalLength = 0; + for ( const ext of loc.extents ) { + totalLength += ext.length; + } + const data = new Uint8Array( totalLength ); + let pos = 0; + for ( const ext of loc.extents ) { + data.set( + new Uint8Array( + buffer.slice( ext.offset, ext.offset + ext.length ) + ), + pos + ); + pos += ext.length; + } + return data; +} + +/** + * Find hvcC and ispe property boxes for a given item. + * + * @param propIndices 1-based property indices from ipma. + * @param properties All property boxes from ipco. + */ +function findHvcProperties( + propIndices: number[], + properties: BoxInfo[] +): { hvcCBox: BoxInfo; ispeBox: BoxInfo } { + let hvcCBox: BoxInfo | undefined; + let ispeBox: BoxInfo | undefined; + + for ( const idx of propIndices ) { + if ( idx < 1 || idx > properties.length ) { + continue; + } + const prop = properties[ idx - 1 ]; + if ( prop.type === 'hvcC' && ! hvcCBox ) { + hvcCBox = prop; + } + if ( prop.type === 'ispe' && ! ispeBox ) { + ispeBox = prop; + } + } + + if ( ! hvcCBox ) { + throw new Error( 'No HEVC configuration (hvcC) found' ); + } + if ( ! ispeBox ) { + throw new Error( 'No image dimensions (ispe) found' ); + } + + return { hvcCBox, ispeBox }; +} + +// --------------------------------------------------------------------------- +// Main entry point +// --------------------------------------------------------------------------- + +/** + * Parse a HEIC file and extract the data needed for VideoDecoder. + * + * Handles both single-image HEIC files and grid/tiled images (the common + * iPhone format where the full image is split into multiple HEVC tiles). + * + * @param buffer Raw HEIC file contents. + * @return Parsed image data including codec config and HEVC tile data. + * @throws If the file is not a valid HEIC or lacks required boxes. + */ +export function parseHeic( buffer: ArrayBuffer ): HeicImageData { + const r = new Reader( buffer ); + const fileEnd = buffer.byteLength; + + // Find top-level 'meta' box. + const metaBox = findBox( r, 0, fileEnd, 'meta' ); + if ( ! metaBox ) { + throw new Error( 'No meta box found in HEIC file' ); + } + + // meta is a FullBox: children start after version (1) + flags (3). + const metaChildStart = metaBox.offset + metaBox.headerSize + 4; + const metaEnd = metaBox.offset + metaBox.size; + + // Locate required child boxes within meta. + const children = findBoxes( r, metaChildStart, metaEnd ); + const pitmBox = children.find( ( b ) => b.type === 'pitm' ); + const ilocBox = children.find( ( b ) => b.type === 'iloc' ); + const iprpBox = children.find( ( b ) => b.type === 'iprp' ); + const iinfBox = children.find( ( b ) => b.type === 'iinf' ); + const irefBox = children.find( ( b ) => b.type === 'iref' ); + + if ( ! pitmBox || ! ilocBox || ! iprpBox ) { + throw new Error( 'Missing required boxes (pitm, iloc, iprp) in HEIC' ); + } + + // Primary item ID. + const primaryId = parsePitm( r, pitmBox ); + + // Item locations. + const locations = parseIloc( r, ilocBox ); + + // Item properties: iprp contains ipco (properties) + ipma (associations). + const iprpStart = iprpBox.offset + iprpBox.headerSize; + const iprpEnd = iprpBox.offset + iprpBox.size; + const iprpChildren = findBoxes( r, iprpStart, iprpEnd ); + const ipcoBox = iprpChildren.find( ( b ) => b.type === 'ipco' ); + const ipmaBox = iprpChildren.find( ( b ) => b.type === 'ipma' ); + + if ( ! ipcoBox || ! ipmaBox ) { + throw new Error( 'Missing ipco or ipma in HEIC properties' ); + } + + const allAssoc = parseIpma( r, ipmaBox ); + + // Enumerate ipco children (properties are 1-indexed). + const ipcoStart = ipcoBox.offset + ipcoBox.headerSize; + const ipcoEnd = ipcoBox.offset + ipcoBox.size; + const properties = findBoxes( r, ipcoStart, ipcoEnd ); + + // Determine if the primary item is a grid or a direct HEVC image. + let primaryItemType = 'hvc1'; + if ( iinfBox ) { + const itemTypes = parseIinf( r, iinfBox ); + const t = itemTypes.get( primaryId ); + if ( t ) { + primaryItemType = t; + } + } + + if ( primaryItemType === 'grid' ) { + // --- Grid/tiled image (common iPhone format) --- + return parseGridImage( + r, + buffer, + primaryId, + locations, + allAssoc, + properties, + irefBox + ); + } + + // --- Single HEVC image --- + const primaryLoc = locations.get( primaryId ); + if ( ! primaryLoc || primaryLoc.extents.length === 0 ) { + throw new Error( `No location data for primary item ${ primaryId }` ); + } + + const primaryPropIndices = allAssoc.get( primaryId ); + if ( ! primaryPropIndices || primaryPropIndices.length === 0 ) { + throw new Error( 'No property associations for primary item' ); + } + + const { hvcCBox, ispeBox } = findHvcProperties( + primaryPropIndices, + properties + ); + + const hvcCDataStart = hvcCBox.offset + hvcCBox.headerSize; + const hvcCDataSize = hvcCBox.size - hvcCBox.headerSize; + const description = new Uint8Array( + buffer.slice( hvcCDataStart, hvcCDataStart + hvcCDataSize ) + ); + const codecString = buildCodecString( r, hvcCDataStart ); + const { width, height } = parseIspe( r, ispeBox ); + + return { + codecString, + description, + tiles: [ { data: readItemData( buffer, primaryLoc ), x: 0, y: 0 } ], + tileWidth: width, + tileHeight: height, + outputWidth: width, + outputHeight: height, + }; +} + +/** + * Parse a grid/tiled HEIC image. + * + * @param r Binary reader. + * @param buffer File ArrayBuffer. + * @param gridItemId The grid item's ID. + * @param locations Parsed iloc data. + * @param allAssoc Parsed ipma data. + * @param properties ipco property boxes. + * @param irefBox iref box (required for grid). + */ +function parseGridImage( + r: Reader, + buffer: ArrayBuffer, + gridItemId: number, + locations: Map< number, ItemLocation >, + allAssoc: Map< number, number[] >, + properties: BoxInfo[], + irefBox: BoxInfo | undefined +): HeicImageData { + // Parse grid descriptor from the grid item's data. + const gridLoc = locations.get( gridItemId ); + if ( ! gridLoc || gridLoc.extents.length === 0 ) { + throw new Error( 'No location data for grid item' ); + } + const gridData = readItemData( buffer, gridLoc ); + + // Grid descriptor format: + // version (1 byte), flags (1 byte), + // rows_minus_one (1 byte), columns_minus_one (1 byte), + // output_width (2 or 4 bytes), output_height (2 or 4 bytes) + const gridFlags = gridData[ 1 ]; + const rows = gridData[ 2 ] + 1; + const columns = gridData[ 3 ] + 1; + const largeFields = ( gridFlags & 1 ) !== 0; + + const gv = new DataView( gridData.buffer, gridData.byteOffset ); + let outputWidth: number; + let outputHeight: number; + if ( largeFields ) { + outputWidth = gv.getUint32( 4 ); + outputHeight = gv.getUint32( 8 ); + } else { + outputWidth = gv.getUint16( 4 ); + outputHeight = gv.getUint16( 6 ); + } + + // Find tile item IDs from iref 'dimg' references. + if ( ! irefBox ) { + throw new Error( 'Grid image requires iref box' ); + } + const dimgRefs = parseIref( r, irefBox, 'dimg' ); + const tileItemIds = dimgRefs.get( gridItemId ); + if ( ! tileItemIds || tileItemIds.length === 0 ) { + throw new Error( 'No tile references found for grid item' ); + } + + if ( tileItemIds.length !== rows * columns ) { + throw new Error( + `Grid expects ${ rows * columns } tiles but found ${ + tileItemIds.length + }` + ); + } + + // Get hvcC and ispe from the first tile item's properties. + // All tiles in a grid share the same HEVC configuration. + const firstTileProps = allAssoc.get( tileItemIds[ 0 ] ); + if ( ! firstTileProps || firstTileProps.length === 0 ) { + throw new Error( 'No property associations for tile item' ); + } + + const { hvcCBox, ispeBox } = findHvcProperties( + firstTileProps, + properties + ); + + const hvcCDataStart = hvcCBox.offset + hvcCBox.headerSize; + const hvcCDataSize = hvcCBox.size - hvcCBox.headerSize; + const description = new Uint8Array( + buffer.slice( hvcCDataStart, hvcCDataStart + hvcCDataSize ) + ); + const codecString = buildCodecString( r, hvcCDataStart ); + const { width: tileWidth, height: tileHeight } = parseIspe( r, ispeBox ); + + // Extract tile data in raster scan order (left→right, top→bottom). + const tiles: HeicTile[] = []; + for ( let row = 0; row < rows; row++ ) { + for ( let col = 0; col < columns; col++ ) { + const tileIdx = row * columns + col; + const tileId = tileItemIds[ tileIdx ]; + const tileLoc = locations.get( tileId ); + if ( ! tileLoc || tileLoc.extents.length === 0 ) { + throw new Error( `No location data for tile item ${ tileId }` ); + } + tiles.push( { + data: readItemData( buffer, tileLoc ), + x: col * tileWidth, + y: row * tileHeight, + } ); + } + } + + return { + codecString, + description, + tiles, + tileWidth, + tileHeight, + outputWidth, + outputHeight, + }; +} + +/* eslint-enable no-bitwise */ diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 832571c29b7b9a..a338cf35a0a033 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -1108,18 +1108,6 @@ export function generateThumbnails( id: QueueItemId ) { const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); let jpegConversion: File | null = null; - // The server can't read HEIC files, so missing_image_sizes will be - // empty even though no sub-sizes were generated. Derive the list - // from the registered image sizes in settings instead. - if ( - isHeicSource && - ( ! attachment.missing_image_sizes || - attachment.missing_image_sizes.length === 0 ) - ) { - const allImageSizes = settings.allImageSizes || {}; - attachment.missing_image_sizes = Object.keys( allImageSizes ); - } - if ( isHeicSource && attachment.id ) { // Sideload the original HEIC to ensure it's tracked in metadata // as original_image. The sideload response includes missing_image_sizes. From 63d75e13ee4b0b9aee73064c360285bbc727e37c Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 21:54:55 -0700 Subject: [PATCH 05/24] Fix HEIC upload handling and parser issues - Register HEIC/HEIF as allowed upload MIME types so macOS file picker doesn't silently convert them to JPEG - Handle idat construction method in HEIC container parser for grid descriptor data stored inside the meta box - Relax grid tile count validation to allow extra iref entries (alpha planes, thumbnails) - Disable server-side sub-size generation and format conversion for HEIC uploads (client handles these) - Add debug logging throughout HEIC upload pipeline --- ...-gutenberg-rest-attachments-controller.php | 6 + lib/media/load.php | 18 +++ packages/upload-media/src/canvas-utils.ts | 37 +++++- packages/upload-media/src/heic-parser.ts | 66 +++++++---- .../upload-media/src/store/private-actions.ts | 107 ++++++++++++++++-- 5 files changed, 197 insertions(+), 37 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 9bda193bd7c0c1..c764a4c00fb024 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -285,6 +285,12 @@ public function prepare_item_for_response( $item, $request ): WP_REST_Response { * @return WP_REST_Response|WP_Error Response object on success, WP_Error object on failure. */ public function create_item( $request ) { + $files = $request->get_file_params(); + $file_type = ! empty( $files['file']['type'] ) ? $files['file']['type'] : 'unknown'; + $file_name = ! empty( $files['file']['name'] ) ? $files['file']['name'] : 'unknown'; + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log + error_log( "[HEIC DEBUG] create_item: file={$file_name} type={$file_type} generate_sub_sizes=" . var_export( $request['generate_sub_sizes'], true ) . " convert_format=" . var_export( $request['convert_format'], true ) ); + if ( ! $request['generate_sub_sizes'] ) { add_filter( 'intermediate_image_sizes_advanced', '__return_empty_array', 100 ); add_filter( 'fallback_intermediate_image_sizes', '__return_empty_array', 100 ); diff --git a/lib/media/load.php b/lib/media/load.php index bae118918a3146..01d16b1a4b9fb3 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -121,6 +121,24 @@ function gutenberg_filter_attachment_post_type_args( array $args, string $post_t add_filter( 'register_post_type_args', 'gutenberg_filter_attachment_post_type_args', 10, 2 ); +/** + * Registers HEIC/HEIF as allowed upload MIME types. + * + * When client-side media processing is enabled, HEIC images can be + * decoded in the browser (via canvas/VideoDecoder). Registering these + * MIME types ensures the file picker's accept attribute includes them, + * preventing macOS from silently converting HEIC to JPEG on selection. + * + * @param array $mimes Allowed MIME types (extension => type). + * @return array Modified MIME types. + */ +function gutenberg_add_heic_upload_mimes( array $mimes ): array { + $mimes['heic'] = 'image/heic'; + $mimes['heif'] = 'image/heif'; + return $mimes; +} + +add_filter( 'upload_mimes', 'gutenberg_add_heic_upload_mimes' ); /** * Registers additional REST fields for attachments. diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 232b209003abf1..4f1e4a2459981b 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -52,9 +52,12 @@ export async function canvasConvertToJpeg( } finally { bitmap.close(); } - } catch { - // createImageBitmap doesn't support HEIC in this browser. - // Fall through to strategy 2. + } catch ( e ) { + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Strategy 1 (createImageBitmap) failed:', + e + ); } // Strategy 2: WebCodecs ImageDecoder API. @@ -96,6 +99,11 @@ export async function canvasConvertToJpeg( } } + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Strategy 2 (ImageDecoder) skipped or unsupported' + ); + // Strategy 3: HEIC container parsing + WebCodecs VideoDecoder. // Chrome 107+ on macOS supports HEVC *video* decoding via platform codecs // (macOS VideoToolbox), even though it doesn't support HEIC through image @@ -103,12 +111,29 @@ export async function canvasConvertToJpeg( // we parse the container and decode each tile via VideoDecoder. if ( typeof VideoDecoder !== 'undefined' ) { try { + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Strategy 3: parsing HEIC container...' + ); const heicData = parseHeic( await file.arrayBuffer() ); const support = await VideoDecoder.isConfigSupported( { codec: heicData.codecString, } ); + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Strategy 3: parsed OK, codec=', + heicData.codecString, + 'tiles=', + heicData.tiles.length, + 'output=', + heicData.outputWidth, + 'x', + heicData.outputHeight, + 'supported=', + support.supported + ); if ( support.supported ) { const canvas = new OffscreenCanvas( heicData.outputWidth, @@ -145,9 +170,9 @@ export async function canvasConvertToJpeg( type: 'image/jpeg', } ); } - } catch { - // VideoDecoder HEVC not available or HEIC parsing failed. - // Fall through to error. + } catch ( e ) { + // eslint-disable-next-line no-console + console.warn( '[HEIC DEBUG] Strategy 3 (VideoDecoder) failed:', e ); } } diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts index a4aea7b9aad36e..3b2d9352e6484a 100644 --- a/packages/upload-media/src/heic-parser.ts +++ b/packages/upload-media/src/heic-parser.ts @@ -200,6 +200,8 @@ interface ItemExtent { } interface ItemLocation { + /** 0 = file offset (mdat), 1 = idat offset. */ + constructionMethod: number; extents: ItemExtent[]; } @@ -228,8 +230,10 @@ function parseIloc( r: Reader, box: BoxInfo ): Map< number, ItemLocation > { for ( let i = 0; i < itemCount; i++ ) { const itemId = version < 2 ? r.u16() : r.u32(); + let constructionMethod = 0; if ( version === 1 || version === 2 ) { - r.u16(); // construction_method + reserved + const cm = r.u16(); + constructionMethod = cm & 0xf; // lower 4 bits } r.u16(); // data_reference_index @@ -249,7 +253,7 @@ function parseIloc( r: Reader, box: BoxInfo ): Map< number, ItemLocation > { } ); } - items.set( itemId, { extents } ); + items.set( itemId, { constructionMethod, extents } ); } return items; @@ -466,15 +470,24 @@ function buildCodecString( r: Reader, recordOffset: number ): string { /** * Read item data by concatenating all extents for a given item location. * - * @param buffer File ArrayBuffer. - * @param loc Item location with extents. + * For construction_method 0, offsets are absolute file positions. + * For construction_method 1, offsets are relative to the idat box data. + * + * @param buffer File ArrayBuffer. + * @param loc Item location with extents. + * @param idatOffset Byte offset of the idat box's data within the file. */ -function readItemData( buffer: ArrayBuffer, loc: ItemLocation ): Uint8Array { +function readItemData( + buffer: ArrayBuffer, + loc: ItemLocation, + idatOffset: number +): Uint8Array { + const baseOffset = loc.constructionMethod === 1 ? idatOffset : 0; + if ( loc.extents.length === 1 ) { const ext = loc.extents[ 0 ]; - return new Uint8Array( - buffer.slice( ext.offset, ext.offset + ext.length ) - ); + const start = baseOffset + ext.offset; + return new Uint8Array( buffer.slice( start, start + ext.length ) ); } let totalLength = 0; for ( const ext of loc.extents ) { @@ -483,10 +496,9 @@ function readItemData( buffer: ArrayBuffer, loc: ItemLocation ): Uint8Array { const data = new Uint8Array( totalLength ); let pos = 0; for ( const ext of loc.extents ) { + const start = baseOffset + ext.offset; data.set( - new Uint8Array( - buffer.slice( ext.offset, ext.offset + ext.length ) - ), + new Uint8Array( buffer.slice( start, start + ext.length ) ), pos ); pos += ext.length; @@ -565,6 +577,10 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { const iprpBox = children.find( ( b ) => b.type === 'iprp' ); const iinfBox = children.find( ( b ) => b.type === 'iinf' ); const irefBox = children.find( ( b ) => b.type === 'iref' ); + const idatBox = children.find( ( b ) => b.type === 'idat' ); + + // idat data offset (for construction_method 1 items). + const idatOffset = idatBox ? idatBox.offset + idatBox.headerSize : 0; if ( ! pitmBox || ! ilocBox || ! iprpBox ) { throw new Error( 'Missing required boxes (pitm, iloc, iprp) in HEIC' ); @@ -613,7 +629,8 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { locations, allAssoc, properties, - irefBox + irefBox, + idatOffset ); } @@ -644,7 +661,13 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { return { codecString, description, - tiles: [ { data: readItemData( buffer, primaryLoc ), x: 0, y: 0 } ], + tiles: [ + { + data: readItemData( buffer, primaryLoc, idatOffset ), + x: 0, + y: 0, + }, + ], tileWidth: width, tileHeight: height, outputWidth: width, @@ -662,6 +685,7 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { * @param allAssoc Parsed ipma data. * @param properties ipco property boxes. * @param irefBox iref box (required for grid). + * @param idatOffset Byte offset of idat box data (for construction_method 1). */ function parseGridImage( r: Reader, @@ -670,14 +694,15 @@ function parseGridImage( locations: Map< number, ItemLocation >, allAssoc: Map< number, number[] >, properties: BoxInfo[], - irefBox: BoxInfo | undefined + irefBox: BoxInfo | undefined, + idatOffset: number ): HeicImageData { // Parse grid descriptor from the grid item's data. const gridLoc = locations.get( gridItemId ); if ( ! gridLoc || gridLoc.extents.length === 0 ) { throw new Error( 'No location data for grid item' ); } - const gridData = readItemData( buffer, gridLoc ); + const gridData = readItemData( buffer, gridLoc, idatOffset ); // Grid descriptor format: // version (1 byte), flags (1 byte), @@ -709,11 +734,12 @@ function parseGridImage( throw new Error( 'No tile references found for grid item' ); } - if ( tileItemIds.length !== rows * columns ) { + // The iref may include extra references (alpha planes, thumbnails). + // Use at least rows * columns tiles; ignore any surplus. + const expectedTiles = rows * columns; + if ( tileItemIds.length < expectedTiles ) { throw new Error( - `Grid expects ${ rows * columns } tiles but found ${ - tileItemIds.length - }` + `Grid expects ${ expectedTiles } tiles but found ${ tileItemIds.length }` ); } @@ -748,7 +774,7 @@ function parseGridImage( throw new Error( `No location data for tile item ${ tileId }` ); } tiles.push( { - data: readItemData( buffer, tileLoc ), + data: readItemData( buffer, tileLoc, idatOffset ), x: col * tileWidth, y: row * tileHeight, } ); diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index a338cf35a0a033..9e10ce1d025de8 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -734,6 +734,16 @@ export function prepareItem( id: QueueItemId ) { } const { file } = item; + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] prepareItem START:', + file.name, + file.type, + 'sourceFile:', + item.sourceFile?.name, + item.sourceFile?.type + ); + const operations: Operation[] = []; const settings = select.getSettings(); @@ -743,6 +753,17 @@ export function prepareItem( id: QueueItemId ) { ); const isHeic = HEIC_MIME_TYPES.includes( file.type ); + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] prepareItem:', + 'isImage=', + isImage, + 'isVipsSupported=', + isVipsSupported, + 'isHeic=', + isHeic + ); + // For images that can be processed by vips, check if we need to scale down based on threshold. if ( isImage && isVipsSupported ) { const { imageOutputFormats } = settings; @@ -788,17 +809,30 @@ export function prepareItem( id: QueueItemId ) { // If the file is not processed by vips, tell the server to // generate sub-sizes since they won't be created client-side. // Exception: HEIC images — the client handles sub-sizes via - // canvas fallback, so tell the server NOT to generate them. - const updates = - ( ! isVipsSupported && ! isHeic ) || ! isImage - ? { - additionalData: { - ...item.additionalData, - generate_sub_sizes: true, - convert_format: true, - }, - } - : {}; + // canvas fallback, so tell the server NOT to generate them + // (and disable format conversion to prevent a duplicate JPEG). + let updates = {}; + if ( isHeic ) { + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] prepareItem: setting generate_sub_sizes=false, convert_format=false for HEIC upload' + ); + updates = { + additionalData: { + ...item.additionalData, + generate_sub_sizes: false, + convert_format: false, + }, + }; + } else if ( ! isVipsSupported || ! isImage ) { + updates = { + additionalData: { + ...item.additionalData, + generate_sub_sizes: true, + convert_format: true, + }, + }; + } dispatch.finishOperation( id, updates ); }; @@ -816,18 +850,44 @@ export function uploadItem( id: QueueItemId ) { return; } + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] uploadItem:', + item.file.name, + item.file.type, + 'additionalData:', + JSON.stringify( item.additionalData ) + ); select.getSettings().mediaUpload( { filesList: [ item.file ], additionalData: item.additionalData, signal: item.abortController?.signal, onFileChange: ( [ attachment ] ) => { if ( attachment && ! isBlobURL( attachment.url ) ) { + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] uploadItem onFileChange:', + attachment.url, + 'mime:', + attachment.mime_type, + 'filename:', + attachment.filename + ); dispatch.finishOperation( id, { attachment, } ); } }, onSuccess: ( [ attachment ] ) => { + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] uploadItem onSuccess:', + attachment.url, + 'mime:', + attachment.mime_type, + 'filename:', + attachment.filename + ); dispatch.finishOperation( id, { attachment, } ); @@ -1102,6 +1162,17 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] generateThumbnails:', + item.sourceFile.name, + item.sourceFile.type, + 'attachment.filename=', + attachment.filename, + 'missing_image_sizes=', + attachment.missing_image_sizes + ); + // HEIC/HEIF canvas fallback handling. // When the source is HEIC, sideload the original and convert to JPEG // using the browser's native decoder for sub-size generation. @@ -1111,6 +1182,12 @@ export function generateThumbnails( id: QueueItemId ) { if ( isHeicSource && attachment.id ) { // Sideload the original HEIC to ensure it's tracked in metadata // as original_image. The sideload response includes missing_image_sizes. + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Sideloading original HEIC:', + item.sourceFile.name, + 'image_size=original' + ); dispatch.addSideloadItem( { file: item.sourceFile, batchId: uuidv4(), @@ -1161,6 +1238,14 @@ export function generateThumbnails( id: QueueItemId ) { ) : jpegConversion; + // eslint-disable-next-line no-console + console.warn( + '[HEIC DEBUG] Sideloading scaled JPEG:', + scaledFile.name, + 'image_size=scaled', + 'attachment.filename=', + attachment.filename + ); dispatch.addSideloadItem( { file: scaledFile, onChange: ( [ updatedAttachment ] ) => { From b5f35ccdceda87557a931f0667c31ac7820e6d9f Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 21:58:16 -0700 Subject: [PATCH 06/24] Remove redundant original HEIC sideload The initial upload already saves the HEIC file on the server. The scaled sideload handler records it as original_image in metadata, so a separate original sideload is unnecessary and creates a duplicate file. --- .../upload-media/src/store/private-actions.ts | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 9e10ce1d025de8..1a3b5797a391b5 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -1179,27 +1179,9 @@ export function generateThumbnails( id: QueueItemId ) { const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); let jpegConversion: File | null = null; - if ( isHeicSource && attachment.id ) { - // Sideload the original HEIC to ensure it's tracked in metadata - // as original_image. The sideload response includes missing_image_sizes. - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Sideloading original HEIC:', - item.sourceFile.name, - 'image_size=original' - ); - dispatch.addSideloadItem( { - file: item.sourceFile, - batchId: uuidv4(), - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'original', - convert_format: false, - }, - operations: [ OperationType.Upload ], - } ); - } + // The initial upload already saved the HEIC file on the server. + // The 'scaled' sideload handler records it as original_image + // in attachment metadata, so no separate original sideload is needed. if ( isHeicSource && From af72719459608e895e59aa380078f676f59aeb81 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 22:34:50 -0700 Subject: [PATCH 07/24] Skip HEIC URLs in editor and remove debug logging Don't update blocks with HEIC attachment URLs since browsers can't display them. The scaled JPEG sideload will provide a usable URL once client-side conversion completes. Also removes all HEIC debug logging added during development. --- ...-gutenberg-rest-attachments-controller.php | 6 -- packages/upload-media/src/canvas-utils.ts | 37 ++------- .../upload-media/src/store/private-actions.ts | 79 ++----------------- 3 files changed, 14 insertions(+), 108 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index c764a4c00fb024..9bda193bd7c0c1 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -285,12 +285,6 @@ public function prepare_item_for_response( $item, $request ): WP_REST_Response { * @return WP_REST_Response|WP_Error Response object on success, WP_Error object on failure. */ public function create_item( $request ) { - $files = $request->get_file_params(); - $file_type = ! empty( $files['file']['type'] ) ? $files['file']['type'] : 'unknown'; - $file_name = ! empty( $files['file']['name'] ) ? $files['file']['name'] : 'unknown'; - // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log - error_log( "[HEIC DEBUG] create_item: file={$file_name} type={$file_type} generate_sub_sizes=" . var_export( $request['generate_sub_sizes'], true ) . " convert_format=" . var_export( $request['convert_format'], true ) ); - if ( ! $request['generate_sub_sizes'] ) { add_filter( 'intermediate_image_sizes_advanced', '__return_empty_array', 100 ); add_filter( 'fallback_intermediate_image_sizes', '__return_empty_array', 100 ); diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 4f1e4a2459981b..232b209003abf1 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -52,12 +52,9 @@ export async function canvasConvertToJpeg( } finally { bitmap.close(); } - } catch ( e ) { - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Strategy 1 (createImageBitmap) failed:', - e - ); + } catch { + // createImageBitmap doesn't support HEIC in this browser. + // Fall through to strategy 2. } // Strategy 2: WebCodecs ImageDecoder API. @@ -99,11 +96,6 @@ export async function canvasConvertToJpeg( } } - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Strategy 2 (ImageDecoder) skipped or unsupported' - ); - // Strategy 3: HEIC container parsing + WebCodecs VideoDecoder. // Chrome 107+ on macOS supports HEVC *video* decoding via platform codecs // (macOS VideoToolbox), even though it doesn't support HEIC through image @@ -111,29 +103,12 @@ export async function canvasConvertToJpeg( // we parse the container and decode each tile via VideoDecoder. if ( typeof VideoDecoder !== 'undefined' ) { try { - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Strategy 3: parsing HEIC container...' - ); const heicData = parseHeic( await file.arrayBuffer() ); const support = await VideoDecoder.isConfigSupported( { codec: heicData.codecString, } ); - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Strategy 3: parsed OK, codec=', - heicData.codecString, - 'tiles=', - heicData.tiles.length, - 'output=', - heicData.outputWidth, - 'x', - heicData.outputHeight, - 'supported=', - support.supported - ); if ( support.supported ) { const canvas = new OffscreenCanvas( heicData.outputWidth, @@ -170,9 +145,9 @@ export async function canvasConvertToJpeg( type: 'image/jpeg', } ); } - } catch ( e ) { - // eslint-disable-next-line no-console - console.warn( '[HEIC DEBUG] Strategy 3 (VideoDecoder) failed:', e ); + } catch { + // VideoDecoder HEVC not available or HEIC parsing failed. + // Fall through to error. } } diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 1a3b5797a391b5..5de99b73efdbea 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -353,7 +353,14 @@ export function processItem( id: QueueItemId ) { } if ( attachment ) { - onChange?.( [ attachment ] ); + // Don't update the block with a HEIC URL — the browser can't + // display it. The scaled JPEG sideload will call onChange + // with a usable URL once the client-side conversion completes. + const isHeicUrl = + attachment.url && /\.hei[cf]$/i.test( attachment.url ); + if ( ! isHeicUrl ) { + onChange?.( [ attachment ] ); + } } /* @@ -734,16 +741,6 @@ export function prepareItem( id: QueueItemId ) { } const { file } = item; - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] prepareItem START:', - file.name, - file.type, - 'sourceFile:', - item.sourceFile?.name, - item.sourceFile?.type - ); - const operations: Operation[] = []; const settings = select.getSettings(); @@ -753,17 +750,6 @@ export function prepareItem( id: QueueItemId ) { ); const isHeic = HEIC_MIME_TYPES.includes( file.type ); - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] prepareItem:', - 'isImage=', - isImage, - 'isVipsSupported=', - isVipsSupported, - 'isHeic=', - isHeic - ); - // For images that can be processed by vips, check if we need to scale down based on threshold. if ( isImage && isVipsSupported ) { const { imageOutputFormats } = settings; @@ -813,10 +799,6 @@ export function prepareItem( id: QueueItemId ) { // (and disable format conversion to prevent a duplicate JPEG). let updates = {}; if ( isHeic ) { - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] prepareItem: setting generate_sub_sizes=false, convert_format=false for HEIC upload' - ); updates = { additionalData: { ...item.additionalData, @@ -850,44 +832,18 @@ export function uploadItem( id: QueueItemId ) { return; } - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] uploadItem:', - item.file.name, - item.file.type, - 'additionalData:', - JSON.stringify( item.additionalData ) - ); select.getSettings().mediaUpload( { filesList: [ item.file ], additionalData: item.additionalData, signal: item.abortController?.signal, onFileChange: ( [ attachment ] ) => { if ( attachment && ! isBlobURL( attachment.url ) ) { - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] uploadItem onFileChange:', - attachment.url, - 'mime:', - attachment.mime_type, - 'filename:', - attachment.filename - ); dispatch.finishOperation( id, { attachment, } ); } }, onSuccess: ( [ attachment ] ) => { - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] uploadItem onSuccess:', - attachment.url, - 'mime:', - attachment.mime_type, - 'filename:', - attachment.filename - ); dispatch.finishOperation( id, { attachment, } ); @@ -1162,17 +1118,6 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] generateThumbnails:', - item.sourceFile.name, - item.sourceFile.type, - 'attachment.filename=', - attachment.filename, - 'missing_image_sizes=', - attachment.missing_image_sizes - ); - // HEIC/HEIF canvas fallback handling. // When the source is HEIC, sideload the original and convert to JPEG // using the browser's native decoder for sub-size generation. @@ -1220,14 +1165,6 @@ export function generateThumbnails( id: QueueItemId ) { ) : jpegConversion; - // eslint-disable-next-line no-console - console.warn( - '[HEIC DEBUG] Sideloading scaled JPEG:', - scaledFile.name, - 'image_size=scaled', - 'attachment.filename=', - attachment.filename - ); dispatch.addSideloadItem( { file: scaledFile, onChange: ( [ updatedAttachment ] ) => { From b326856011bc2ff1036d46e5d422a0d534a7ae98 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 22:41:11 -0700 Subject: [PATCH 08/24] Add unit tests for HEIC container parser Tests cover: - Bit reversal for HEVC codec string construction - Single-image HEIC parsing with synthetic binary fixtures - Codec string generation (e.g. hvc1.1.6.L93.B0) - HEVCDecoderConfigurationRecord extraction - Error cases: empty buffer, missing meta/pitm/iloc boxes --- packages/upload-media/src/heic-parser.ts | 2 +- packages/upload-media/src/test/heic-parser.ts | 363 ++++++++++++++++++ 2 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 packages/upload-media/src/test/heic-parser.ts diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts index 3b2d9352e6484a..9c29c8861445cb 100644 --- a/packages/upload-media/src/heic-parser.ts +++ b/packages/upload-media/src/heic-parser.ts @@ -402,7 +402,7 @@ function parseIref( * * @param n 32-bit unsigned integer. */ -function reverseBits32( n: number ): number { +export function reverseBits32( n: number ): number { n = ( ( n >>> 1 ) & 0x55555555 ) | ( ( n & 0x55555555 ) << 1 ); n = ( ( n >>> 2 ) & 0x33333333 ) | ( ( n & 0x33333333 ) << 2 ); n = ( ( n >>> 4 ) & 0x0f0f0f0f ) | ( ( n & 0x0f0f0f0f ) << 4 ); diff --git a/packages/upload-media/src/test/heic-parser.ts b/packages/upload-media/src/test/heic-parser.ts new file mode 100644 index 00000000000000..81372832efbc08 --- /dev/null +++ b/packages/upload-media/src/test/heic-parser.ts @@ -0,0 +1,363 @@ +/* eslint-disable no-bitwise, jsdoc/require-param */ + +/** + * Internal dependencies + */ +import { parseHeic, reverseBits32 } from '../heic-parser'; + +// --------------------------------------------------------------------------- +// Helpers for constructing synthetic ISOBMFF structures +// --------------------------------------------------------------------------- + +/** Write a big-endian uint32 into a DataView. */ +function writeU32( view: DataView, offset: number, value: number ) { + view.setUint32( offset, value ); +} + +/** Write a big-endian uint16 into a DataView. */ +function writeU16( view: DataView, offset: number, value: number ) { + view.setUint16( offset, value ); +} + +/** Build an ISOBMFF box (size + fourcc + data). */ +function buildBox( type: string, data: Uint8Array ): Uint8Array { + const size = 8 + data.length; + const buf = new Uint8Array( size ); + const view = new DataView( buf.buffer ); + writeU32( view, 0, size ); + for ( let i = 0; i < 4; i++ ) { + buf[ 4 + i ] = type.charCodeAt( i ); + } + buf.set( data, 8 ); + return buf; +} + +/** Build a FullBox (size + fourcc + version + flags + data). */ +function buildFullBox( + type: string, + version: number, + flags: number, + data: Uint8Array +): Uint8Array { + const inner = new Uint8Array( 4 + data.length ); + inner[ 0 ] = version; + inner[ 1 ] = ( flags >> 16 ) & 0xff; + inner[ 2 ] = ( flags >> 8 ) & 0xff; + inner[ 3 ] = flags & 0xff; + inner.set( data, 4 ); + return buildBox( type, inner ); +} + +/** Concatenate multiple Uint8Arrays. */ +function concat( ...arrays: Uint8Array[] ): Uint8Array { + const total = arrays.reduce( ( sum, a ) => sum + a.length, 0 ); + const result = new Uint8Array( total ); + let offset = 0; + for ( const a of arrays ) { + result.set( a, offset ); + offset += a.length; + } + return result; +} + +/** Build a pitm box (Primary Item). */ +function buildPitm( primaryItemId: number ): Uint8Array { + const data = new Uint8Array( 2 ); + const view = new DataView( data.buffer ); + writeU16( view, 0, primaryItemId ); + return buildFullBox( 'pitm', 0, 0, data ); +} + +/** + * Build a minimal hvcC box (HEVCDecoderConfigurationRecord). + * + * Fields: configVersion=1, profileSpace=0, tier=0, profileIdc=1, + * compatFlags=0x60000000, constraintBytes=[0xB0,0,0,0,0,0], + * levelIdc=93, then zeros for remaining fields + 0 NAL arrays. + */ +function buildHvcC(): Uint8Array { + // HEVCDecoderConfigurationRecord (23 bytes minimum with 0 arrays) + const record = new Uint8Array( 23 ); + record[ 0 ] = 1; // configurationVersion + // byte1: profileSpace=0 (bits 6-7), tier=0 (bit 5), profileIdc=1 (bits 0-4) + record[ 1 ] = 0x01; + // general_profile_compatibility_flags = 0x60000000 + record[ 2 ] = 0x60; + record[ 3 ] = 0x00; + record[ 4 ] = 0x00; + record[ 5 ] = 0x00; + // general_constraint_indicator_flags (6 bytes) = [0xB0, 0, 0, 0, 0, 0] + record[ 6 ] = 0xb0; + // bytes 7-11 are zero (remaining constraint bytes) + // general_level_idc = 93 + record[ 12 ] = 93; + // remaining fields: min_spatial_segmentation_idc, parallelismType, + // chromaFormat, bitDepthLuma, bitDepthChroma, avgFrameRate, misc, numOfArrays + // All zero is valid for our test purposes. + return buildBox( 'hvcC', record ); +} + +/** Build an ispe (Image Spatial Extents) box. */ +function buildIspe( width: number, height: number ): Uint8Array { + const data = new Uint8Array( 8 ); + const view = new DataView( data.buffer ); + writeU32( view, 0, width ); + writeU32( view, 4, height ); + return buildFullBox( 'ispe', 0, 0, data ); +} + +/** Build an ipco (Item Property Container) with the given property boxes. */ +function buildIpco( ...properties: Uint8Array[] ): Uint8Array { + return buildBox( 'ipco', concat( ...properties ) ); +} + +/** + * Build an ipma (Item Property Association) box. + * + * @param associations Array of [itemId, propertyIndices[]] + */ +function buildIpma( associations: Array< [ number, number[] ] > ): Uint8Array { + // Calculate data size: 4 (entry_count) + per entry: 2 (itemId) + 1 (assocCount) + N (indices) + let dataSize = 4; + for ( const [ , indices ] of associations ) { + dataSize += 2 + 1 + indices.length; + } + const data = new Uint8Array( dataSize ); + const view = new DataView( data.buffer ); + writeU32( view, 0, associations.length ); + let pos = 4; + for ( const [ itemId, indices ] of associations ) { + writeU16( view, pos, itemId ); + pos += 2; + data[ pos ] = indices.length; + pos += 1; + for ( const idx of indices ) { + data[ pos ] = idx & 0x7f; // 7-bit index, essential=0 + pos += 1; + } + } + return buildFullBox( 'ipma', 0, 0, data ); +} + +/** Build an iprp box containing ipco + ipma. */ +function buildIprp( ipco: Uint8Array, ipma: Uint8Array ): Uint8Array { + return buildBox( 'iprp', concat( ipco, ipma ) ); +} + +/** + * Build an iloc box for version 0, with 4-byte offsets and 4-byte lengths. + * + * @param items Array of [itemId, [[offset, length], ...]] + */ +function buildIloc( + items: Array< [ number, Array< [ number, number ] > ] > +): Uint8Array { + // version 0: offsetSize=4, lengthSize=4, baseOffsetSize=0 + // per item: 2 (itemId) + 2 (data_reference_index) + 0 (base_offset) + 2 (extent_count) + N*(4+4) + let dataSize = 2 + 2; // sizes byte + item_count + for ( const [ , extents ] of items ) { + dataSize += 2 + 2 + 2 + extents.length * 8; + } + const data = new Uint8Array( dataSize ); + const view = new DataView( data.buffer ); + // offset_size=4 (upper nibble), length_size=4 (lower nibble) + data[ 0 ] = 0x44; + // base_offset_size=0 (upper nibble), reserved=0 + data[ 1 ] = 0x00; + // item_count + writeU16( view, 2, items.length ); + let pos = 4; + for ( const [ itemId, extents ] of items ) { + writeU16( view, pos, itemId ); + pos += 2; + // data_reference_index = 0 + writeU16( view, pos, 0 ); + pos += 2; + // no base_offset (size=0) + // extent_count + writeU16( view, pos, extents.length ); + pos += 2; + for ( const [ offset, length ] of extents ) { + writeU32( view, pos, offset ); + pos += 4; + writeU32( view, pos, length ); + pos += 4; + } + } + return buildFullBox( 'iloc', 0, 0, data ); +} + +/** Build an hdlr (Handler) box with handler_type='pict'. */ +function buildHdlr(): Uint8Array { + // Minimal hdlr: 4 bytes pre_defined + 4 bytes handler_type + 12 bytes reserved + 1 byte name (null) + const data = new Uint8Array( 21 ); + // handler_type = 'pict' at offset 4 + data[ 4 ] = 0x70; // p + data[ 5 ] = 0x69; // i + data[ 6 ] = 0x63; // c + data[ 7 ] = 0x74; // t + return buildFullBox( 'hdlr', 0, 0, data ); +} + +/** + * Build a minimal single-image HEIC file as an ArrayBuffer. + * + * The image data is fake (not decodable) but the container structure + * is valid for testing the parser. + */ +function buildSingleImageHeic( { + width = 100, + height = 80, + imageData = new Uint8Array( [ 0xde, 0xad, 0xbe, 0xef ] ), +}: { + width?: number; + height?: number; + imageData?: Uint8Array; +} = {} ): ArrayBuffer { + const primaryItemId = 1; + + // Build property boxes (1-indexed: 1=ispe, 2=hvcC) + const ispe = buildIspe( width, height ); + const hvcC = buildHvcC(); + const ipco = buildIpco( ispe, hvcC ); + const ipma = buildIpma( [ [ primaryItemId, [ 1, 2 ] ] ] ); + const iprp = buildIprp( ipco, ipma ); + + // We need to know where mdat data will be placed. + // Build everything except mdat first to calculate the offset. + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); // brand='heic' + const pitm = buildPitm( primaryItemId ); + // iloc will reference the image data at an absolute file offset. + // We'll calculate the actual offset after constructing the meta box. + // Use a placeholder first, then fix it up. + + // Build meta children (without iloc - we'll add it after calculating offset) + const hdlr = buildHdlr(); + const metaChildrenWithoutIloc = concat( hdlr, pitm, iprp ); + + // Calculate sizes to determine mdat data offset: + // ftyp + meta box header (8 + 4 fullbox) + metaChildren + iloc + mdat header (8) + // iloc size depends on items, so build a placeholder iloc to get its size. + const placeholderIloc = buildIloc( [ + [ primaryItemId, [ [ 0, imageData.length ] ] ], + ] ); + const metaSize = + 8 + 4 + metaChildrenWithoutIloc.length + placeholderIloc.length; + const mdatDataOffset = ftyp.length + metaSize + 8; // +8 for mdat box header + + // Now build the real iloc with the correct offset + const iloc = buildIloc( [ + [ primaryItemId, [ [ mdatDataOffset, imageData.length ] ] ], + ] ); + + // Build meta box (FullBox) + const metaChildren = concat( hdlr, pitm, iloc, iprp ); + const meta = buildFullBox( 'meta', 0, 0, metaChildren ); + + // Build mdat + const mdat = buildBox( 'mdat', imageData ); + + // Assemble full file + const file = concat( ftyp, meta, mdat ); + return file.buffer; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe( 'heic-parser', () => { + describe( 'reverseBits32', () => { + it( 'should reverse bits of 0x60000000 to 0x00000006', () => { + expect( reverseBits32( 0x60000000 ) ).toBe( 0x00000006 ); + } ); + + it( 'should reverse bits of 0 to 0', () => { + expect( reverseBits32( 0 ) ).toBe( 0 ); + } ); + + it( 'should reverse bits of 0xFFFFFFFF to 0xFFFFFFFF', () => { + expect( reverseBits32( 0xffffffff ) ).toBe( 0xffffffff ); + } ); + + it( 'should reverse bits of 1 to 0x80000000', () => { + expect( reverseBits32( 1 ) ).toBe( 0x80000000 ); + } ); + + it( 'should reverse bits of 0x80000000 to 1', () => { + expect( reverseBits32( 0x80000000 ) ).toBe( 1 ); + } ); + } ); + + describe( 'parseHeic – single image', () => { + it( 'should parse a minimal single-image HEIC', () => { + const imageData = new Uint8Array( [ 1, 2, 3, 4, 5, 6 ] ); + const buffer = buildSingleImageHeic( { + width: 200, + height: 150, + imageData, + } ); + + const result = parseHeic( buffer ); + + expect( result.outputWidth ).toBe( 200 ); + expect( result.outputHeight ).toBe( 150 ); + expect( result.tileWidth ).toBe( 200 ); + expect( result.tileHeight ).toBe( 150 ); + expect( result.tiles ).toHaveLength( 1 ); + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 0 ].y ).toBe( 0 ); + expect( result.tiles[ 0 ].data ).toEqual( imageData ); + } ); + + it( 'should build correct codec string for Main Profile L3.1', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + // profileIdc=1, compatFlags=0x60000000→reversed=6, + // tier=L, level=93, constraints=B0 + expect( result.codecString ).toBe( 'hvc1.1.6.L93.B0' ); + } ); + + it( 'should extract the HEVCDecoderConfigurationRecord', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + expect( result.description ).toBeInstanceOf( Uint8Array ); + expect( result.description.length ).toBe( 23 ); // minimal hvcC record + expect( result.description[ 0 ] ).toBe( 1 ); // configurationVersion + } ); + } ); + + describe( 'parseHeic – error cases', () => { + it( 'should throw for empty buffer', () => { + expect( () => parseHeic( new ArrayBuffer( 0 ) ) ).toThrow( + 'No meta box found' + ); + } ); + + it( 'should throw for buffer without meta box', () => { + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); + expect( () => parseHeic( ftyp.buffer ) ).toThrow( + 'No meta box found' + ); + } ); + + it( 'should throw when required boxes are missing', () => { + // meta box with only hdlr (no pitm, iloc, iprp) + const hdlr = buildHdlr(); + const meta = buildFullBox( 'meta', 0, 0, hdlr ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'Missing required boxes' + ); + } ); + } ); +} ); + +/* eslint-enable no-bitwise, jsdoc/require-param */ From 781b9f518f7d77df979d375e3d32e6b248d5636f Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Fri, 20 Mar 2026 22:45:08 -0700 Subject: [PATCH 09/24] Add bounds check for HEIC grid descriptor Validate grid descriptor data length before accessing fields to avoid out-of-bounds reads on malformed files. --- packages/upload-media/src/heic-parser.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts index 9c29c8861445cb..71830c202c5465 100644 --- a/packages/upload-media/src/heic-parser.ts +++ b/packages/upload-media/src/heic-parser.ts @@ -708,10 +708,16 @@ function parseGridImage( // version (1 byte), flags (1 byte), // rows_minus_one (1 byte), columns_minus_one (1 byte), // output_width (2 or 4 bytes), output_height (2 or 4 bytes) - const gridFlags = gridData[ 1 ]; + const largeFields = gridData.length > 1 && ( gridData[ 1 ] & 1 ) !== 0; + const minGridSize = largeFields ? 12 : 8; + if ( gridData.length < minGridSize ) { + throw new Error( + `Grid descriptor too short: ${ gridData.length } bytes` + ); + } + const rows = gridData[ 2 ] + 1; const columns = gridData[ 3 ] + 1; - const largeFields = ( gridFlags & 1 ) !== 0; const gv = new DataView( gridData.buffer, gridData.byteOffset ); let outputWidth: number; From ec42421f05164aea0240cf12b85636d5dddb44ab Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 23 Mar 2026 13:30:21 -0700 Subject: [PATCH 10/24] Add Safari HEIC support via canvas conversion Safari can decode HEIC natively via createImageBitmap() but lacks SharedArrayBuffer for VIPS. Split the PHP gate so HEIC infrastructure (MIME types, REST controller, sideload endpoint) loads independently of VIPS requirements. Add a HEIC-only mode that converts HEIC to JPEG client-side and delegates sub-size generation to the server. --- ...-gutenberg-rest-attachments-controller.php | 28 ++- lib/media/load.php | 236 ++++++++++-------- .../src/components/provider/index.js | 143 ++++++++++- packages/block-library/src/image/edit.js | 6 +- .../provider/use-upload-save-lock.js | 2 +- packages/media-utils/src/utils/types.ts | 2 + .../media-utils/src/utils/upload-media.ts | 1 + .../upload-media/src/feature-detection.ts | 17 ++ packages/upload-media/src/index.ts | 1 + .../upload-media/src/store/private-actions.ts | 16 ++ 10 files changed, 336 insertions(+), 116 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 9bda193bd7c0c1..a682277faf5539 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -41,16 +41,21 @@ public function register_routes(): void { 'callback' => array( $this, 'sideload_item' ), 'permission_callback' => array( $this, 'sideload_item_permissions_check' ), 'args' => array( - 'id' => array( + 'id' => array( 'description' => __( 'Unique identifier for the attachment.', 'gutenberg' ), 'type' => 'integer', ), - 'image_size' => array( + 'image_size' => array( 'description' => __( 'Image size.', 'gutenberg' ), 'type' => 'string', 'enum' => $valid_image_sizes, 'required' => true, ), + 'generate_sub_sizes' => array( + 'description' => __( 'Whether to generate image sub sizes from the sideloaded file.', 'gutenberg' ), + 'type' => 'boolean', + 'default' => false, + ), ), ), 'allow_batch' => $this->allow_batch, @@ -549,6 +554,25 @@ public function sideload_item( WP_REST_Request $request ) { wp_update_attachment_metadata( $attachment_id, $metadata ); + // When generate_sub_sizes is true (e.g. HEIC-only mode on Safari), + // generate all image sub-sizes server-side from the sideloaded JPEG. + // This handles the case where the client converted HEIC to JPEG via + // canvas but lacks VIPS/WASM for client-side thumbnail generation. + if ( $request['generate_sub_sizes'] && 'scaled' === $image_size ) { + // Use wp_create_image_subsizes which generates all registered + // sub-sizes and updates the attachment metadata. + $new_metadata = wp_create_image_subsizes( $path, $attachment_id ); + + if ( ! is_wp_error( $new_metadata ) ) { + // Preserve the original_image reference from HEIC. + if ( ! empty( $metadata['original_image'] ) ) { + $new_metadata['original_image'] = $metadata['original_image']; + } + + wp_update_attachment_metadata( $attachment_id, $new_metadata ); + } + } + $response_request = new WP_REST_Request( WP_REST_Server::READABLE, rest_get_route_for_post( $attachment_id ) diff --git a/lib/media/load.php b/lib/media/load.php index 01d16b1a4b9fb3..9a474e3b5ea0ad 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -2,6 +2,16 @@ /** * Adds media-related functionality for client-side media processing. * + * This file is structured in two tiers: + * + * 1. HEIC infrastructure — loaded whenever the feature filter is enabled. + * Browsers like Safari can decode HEIC via createImageBitmap() even + * without VIPS/SharedArrayBuffer, so HEIC MIME types, the custom REST + * controller, and REST field/index registrations are always needed. + * + * 2. Full VIPS/WASM processing — loaded only when the feature filter is + * enabled AND requires cross-origin isolation (DIP) at runtime. + * * @package gutenberg */ @@ -9,98 +19,26 @@ return; } -/** - * Sets a global JS variable to indicate that client-side media processing is enabled. - */ -function gutenberg_set_client_side_media_processing_flag() { - if ( ! gutenberg_is_client_side_media_processing_enabled() ) { - return; - } - wp_add_inline_script( 'wp-block-editor', 'window.__clientSideMediaProcessing = true', 'before' ); -} -add_action( 'admin_init', 'gutenberg_set_client_side_media_processing_flag' ); +// ── Tier 1: HEIC infrastructure (always loaded) ───────────────────── /** - * Returns a list of all available image sizes. + * Registers HEIC/HEIF as allowed upload MIME types. * - * @return array Existing image sizes. - */ -function gutenberg_get_all_image_sizes(): array { - $sizes = wp_get_registered_image_subsizes(); - - foreach ( $sizes as $name => &$size ) { - $size['height'] = (int) $size['height']; - $size['width'] = (int) $size['width']; - $size['name'] = $name; - } - unset( $size ); - - return $sizes; -} - -/** - * Returns the default output format mapping for the supported image formats. + * HEIC images can be decoded in the browser (via canvas/VideoDecoder). + * Registering these MIME types ensures the file picker's accept attribute + * includes them, preventing macOS from silently converting HEIC to JPEG + * on selection. * - * @return array Map of input formats to output formats. - */ -function gutenberg_get_default_image_output_formats() { - $input_formats = array( - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - 'image/avif', - 'image/heic', - 'image/heif', - ); - - $output_formats = array(); - - foreach ( $input_formats as $mime_type ) { - /** This filter is documented in wp-includes/media.php */ - $output_formats = apply_filters( - 'image_editor_output_format', // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - $output_formats, - '', - $mime_type - ); - } - - return $output_formats; -} - -/** - * Filters the REST API root index data to add custom settings. - * - * @param WP_REST_Response $response Response data. + * @param array $mimes Allowed MIME types (extension => type). + * @return array Modified MIME types. */ -function gutenberg_media_processing_filter_rest_index( WP_REST_Response $response ) { - /** This filter is documented in wp-admin/includes/images.php */ - $image_size_threshold = (int) apply_filters( 'big_image_size_threshold', 2560, array( 0, 0 ), '', 0 ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - - $default_image_output_formats = gutenberg_get_default_image_output_formats(); - - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $jpeg_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/jpeg' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $png_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/png' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $gif_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/gif' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - - if ( current_user_can( 'upload_files' ) ) { - $response->data['image_sizes'] = gutenberg_get_all_image_sizes(); - $response->data['image_size_threshold'] = $image_size_threshold; - $response->data['image_output_formats'] = (object) $default_image_output_formats; - $response->data['jpeg_interlaced'] = $jpeg_interlaced; - $response->data['png_interlaced'] = $png_interlaced; - $response->data['gif_interlaced'] = $gif_interlaced; - } - - return $response; +function gutenberg_add_heic_upload_mimes( array $mimes ): array { + $mimes['heic'] = 'image/heic'; + $mimes['heif'] = 'image/heif'; + return $mimes; } -add_filter( 'rest_index', 'gutenberg_media_processing_filter_rest_index' ); - +add_filter( 'upload_mimes', 'gutenberg_add_heic_upload_mimes' ); /** * Overrides the REST controller for the attachment post type. @@ -121,25 +59,6 @@ function gutenberg_filter_attachment_post_type_args( array $args, string $post_t add_filter( 'register_post_type_args', 'gutenberg_filter_attachment_post_type_args', 10, 2 ); -/** - * Registers HEIC/HEIF as allowed upload MIME types. - * - * When client-side media processing is enabled, HEIC images can be - * decoded in the browser (via canvas/VideoDecoder). Registering these - * MIME types ensures the file picker's accept attribute includes them, - * preventing macOS from silently converting HEIC to JPEG on selection. - * - * @param array $mimes Allowed MIME types (extension => type). - * @return array Modified MIME types. - */ -function gutenberg_add_heic_upload_mimes( array $mimes ): array { - $mimes['heic'] = 'image/heic'; - $mimes['heif'] = 'image/heif'; - return $mimes; -} - -add_filter( 'upload_mimes', 'gutenberg_add_heic_upload_mimes' ); - /** * Registers additional REST fields for attachments. */ @@ -220,6 +139,115 @@ function gutenberg_rest_get_attachment_filesize( array $post ): ?int { return null; } +/** + * Returns a list of all available image sizes. + * + * @return array Existing image sizes. + */ +function gutenberg_get_all_image_sizes(): array { + $sizes = wp_get_registered_image_subsizes(); + + foreach ( $sizes as $name => &$size ) { + $size['height'] = (int) $size['height']; + $size['width'] = (int) $size['width']; + $size['name'] = $name; + } + unset( $size ); + + return $sizes; +} + +/** + * Returns the default output format mapping for the supported image formats. + * + * @return array Map of input formats to output formats. + */ +function gutenberg_get_default_image_output_formats() { + $input_formats = array( + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + 'image/avif', + 'image/heic', + 'image/heif', + ); + + $output_formats = array(); + + foreach ( $input_formats as $mime_type ) { + /** This filter is documented in wp-includes/media.php */ + $output_formats = apply_filters( + 'image_editor_output_format', // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + $output_formats, + '', + $mime_type + ); + } + + return $output_formats; +} + +/** + * Filters the REST API root index data to add custom settings. + * + * @param WP_REST_Response $response Response data. + */ +function gutenberg_media_processing_filter_rest_index( WP_REST_Response $response ) { + /** This filter is documented in wp-admin/includes/images.php */ + $image_size_threshold = (int) apply_filters( 'big_image_size_threshold', 2560, array( 0, 0 ), '', 0 ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + + $default_image_output_formats = gutenberg_get_default_image_output_formats(); + + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $jpeg_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/jpeg' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $png_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/png' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $gif_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/gif' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + + if ( current_user_can( 'upload_files' ) ) { + $response->data['image_sizes'] = gutenberg_get_all_image_sizes(); + $response->data['image_size_threshold'] = $image_size_threshold; + $response->data['image_output_formats'] = (object) $default_image_output_formats; + $response->data['jpeg_interlaced'] = $jpeg_interlaced; + $response->data['png_interlaced'] = $png_interlaced; + $response->data['gif_interlaced'] = $gif_interlaced; + } + + return $response; +} + +add_filter( 'rest_index', 'gutenberg_media_processing_filter_rest_index' ); + +/** + * Sets a global JS variable to indicate that HEIC canvas-based upload support is available. + * + * This flag is set whenever the media processing feature is enabled, + * regardless of whether the browser supports full VIPS-based processing. + * Browsers like Safari can use createImageBitmap() to decode HEIC images + * and convert them to JPEG for server-side sub-size generation. + */ +function gutenberg_set_heic_upload_support_flag() { + wp_add_inline_script( 'wp-block-editor', 'window.__heicUploadSupport = true', 'before' ); +} +add_action( 'admin_init', 'gutenberg_set_heic_upload_support_flag' ); + +// ── Tier 2: Full client-side processing (VIPS/WASM) ───────────────── +// Everything below requires cross-origin isolation (Document-Isolation-Policy) +// and SharedArrayBuffer support, which is only available in Chromium 137+. + +/** + * Sets a global JS variable to indicate that client-side media processing is enabled. + */ +function gutenberg_set_client_side_media_processing_flag() { + if ( ! gutenberg_is_client_side_media_processing_enabled() ) { + return; + } + wp_add_inline_script( 'wp-block-editor', 'window.__clientSideMediaProcessing = true', 'before' ); +} +add_action( 'admin_init', 'gutenberg_set_client_side_media_processing_flag' ); + /** * Filters the list of rewrite rules formatted for output to an .htaccess file. * diff --git a/packages/block-editor/src/components/provider/index.js b/packages/block-editor/src/components/provider/index.js index 18185c57c010aa..c6e2e758de00d6 100644 --- a/packages/block-editor/src/components/provider/index.js +++ b/packages/block-editor/src/components/provider/index.js @@ -8,6 +8,7 @@ import { MediaUploadProvider, store as uploadStore, detectClientSideMediaSupport, + isHeicCanvasSupported, } from '@wordpress/upload-media'; /** @@ -38,6 +39,17 @@ let hasLoggedFallback = false; */ let isClientSideMediaEnabledCache = null; +/** + * Cached result of whether HEIC-only canvas processing should be enabled. + */ +let isHeicCanvasEnabledCache = null; + +/** + * HEIC MIME types that should be routed through the upload-media pipeline + * when in HEIC-only mode. + */ +const HEIC_MIME_TYPES = [ 'image/heic', 'image/heif' ]; + /** * Checks if client-side media processing should be enabled. * @@ -85,6 +97,44 @@ function shouldEnableClientSideMediaProcessing() { return true; } +/** + * Checks if HEIC-only canvas processing should be enabled. + * + * Returns true when: + * 1. Full client-side processing is NOT available (otherwise it handles HEIC already) + * 2. The server has set the __heicUploadSupport flag + * 3. The browser supports createImageBitmap + OffscreenCanvas (e.g. Safari) + * + * @return {boolean} Whether HEIC-only canvas processing should be enabled. + */ +function shouldEnableHeicCanvasProcessing() { + if ( isHeicCanvasEnabledCache !== null ) { + return isHeicCanvasEnabledCache; + } + + // If full client-side processing is enabled, it already handles HEIC. + if ( shouldEnableClientSideMediaProcessing() ) { + isHeicCanvasEnabledCache = false; + return false; + } + + if ( ! window.__heicUploadSupport ) { + isHeicCanvasEnabledCache = false; + return false; + } + + if ( + typeof isHeicCanvasSupported !== 'function' || + ! isHeicCanvasSupported() + ) { + isHeicCanvasEnabledCache = false; + return false; + } + + isHeicCanvasEnabledCache = true; + return true; +} + /** * Upload a media file when the file upload button is activated * or when adding a file to the editor via drag & drop. @@ -128,6 +178,78 @@ function mediaUpload( } ); } +/** + * Upload interceptor for HEIC-only mode. + * + * Routes HEIC files through the upload-media pipeline for canvas-based + * conversion, while passing non-HEIC files to the original mediaUpload + * function for standard server-side processing. + * + * @param {WPDataRegistry} registry + * @param {Object} settings Block editor settings. + * @param {Object} $3 Parameters object passed to the function. + * @param {Array} $3.allowedTypes Array with the types of media that can be uploaded, if unset all types are allowed. + * @param {Object} $3.additionalData Additional data to include in the request. + * @param {Array} $3.filesList List of files. + * @param {Function} $3.onError Function called when an error happens. + * @param {Function} $3.onFileChange Function called each time a file or a temporary representation of the file is available. + * @param {Function} $3.onSuccess Function called once a file has completely finished uploading, including thumbnails. + * @param {Function} $3.onBatchSuccess Function called once all files in a group have completely finished uploading, including thumbnails. + */ +function heicMediaUpload( + registry, + settings, + { + allowedTypes, + additionalData = {}, + filesList, + onError = noop, + onFileChange, + onSuccess, + onBatchSuccess, + } +) { + const files = Array.from( filesList ); + const heicFiles = files.filter( ( file ) => + HEIC_MIME_TYPES.includes( file.type ) + ); + const otherFiles = files.filter( + ( file ) => ! HEIC_MIME_TYPES.includes( file.type ) + ); + + // Route HEIC files through the upload-media pipeline. + if ( heicFiles.length > 0 ) { + void registry.dispatch( uploadStore ).addItems( { + files: heicFiles, + onChange: onFileChange, + onSuccess: ( attachments ) => { + settings?.[ mediaUploadOnSuccessKey ]?.( attachments ); + onSuccess?.( attachments ); + }, + onBatchSuccess, + onError: ( error ) => + onError( + typeof error === 'string' ? error : error?.message ?? '' + ), + additionalData, + allowedTypes, + } ); + } + + // Pass non-HEIC files to the original server-side upload function. + if ( otherFiles.length > 0 && settings?.mediaUpload ) { + settings.mediaUpload( { + allowedTypes, + additionalData, + filesList: otherFiles, + onError, + onFileChange, + onSuccess, + onBatchSuccess, + } ); + } +} + /** * Calls useBlockSync as a child of SelectionContext.Provider so that the * hook can read selection state from the context provided by this tree @@ -152,6 +274,9 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( const isClientSideMediaEnabled = shouldEnableClientSideMediaProcessing(); + const isHeicCanvasEnabled = shouldEnableHeicCanvasProcessing(); + const useUploadMediaPipeline = + isClientSideMediaEnabled || isHeicCanvasEnabled; // Nested providers (e.g. from useBlockPreview) inherit settings // where mediaUpload has already been replaced with the @@ -162,16 +287,17 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( const settings = useMemo( () => { if ( - isClientSideMediaEnabled && + useUploadMediaPipeline && _settings?.mediaUpload && ! isMediaUploadIntercepted ) { - // Create a new object so that the original props.settings.mediaUpload is not modified. - const interceptor = mediaUpload.bind( - null, - registry, - _settings - ); + // Choose the right interceptor: + // - Full mode: all uploads go through upload-media pipeline. + // - HEIC-only mode: only HEIC files go through, rest use legacy path. + const uploadFn = isClientSideMediaEnabled + ? mediaUpload + : heicMediaUpload; + const interceptor = uploadFn.bind( null, registry, _settings ); interceptor.__isMediaUploadInterceptor = true; return { ..._settings, @@ -182,6 +308,7 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( }, [ _settings, registry, + useUploadMediaPipeline, isClientSideMediaEnabled, isMediaUploadIntercepted, ] ); @@ -258,7 +385,7 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( // overwrite the store's server-side function with the // interceptor, causing uploads to loop instead of reaching // the server. - if ( isClientSideMediaEnabled && ! isMediaUploadIntercepted ) { + if ( useUploadMediaPipeline && ! isMediaUploadIntercepted ) { return ( { - if ( ! window.__clientSideMediaProcessing || ! id ) { + if ( + ( ! window.__clientSideMediaProcessing && + ! window.__heicUploadSupport ) || + ! id + ) { return false; } return select( uploadStore ).isUploadingById( id ); diff --git a/packages/editor/src/components/provider/use-upload-save-lock.js b/packages/editor/src/components/provider/use-upload-save-lock.js index 8c0473e061774f..c4803d478db025 100644 --- a/packages/editor/src/components/provider/use-upload-save-lock.js +++ b/packages/editor/src/components/provider/use-upload-save-lock.js @@ -20,7 +20,7 @@ const LOCK_NAME = 'upload-in-progress'; */ export default function useUploadSaveLock() { const isClientSideMediaProcessingEnabled = - window.__clientSideMediaProcessing; + window.__clientSideMediaProcessing || window.__heicUploadSupport; const isUploading = useSelect( ( select ) => { diff --git a/packages/media-utils/src/utils/types.ts b/packages/media-utils/src/utils/types.ts index c4c6882ea2532e..4593f8d16205e8 100644 --- a/packages/media-utils/src/utils/types.ts +++ b/packages/media-utils/src/utils/types.ts @@ -208,6 +208,8 @@ export type AdditionalData = BetterOmit< CreateRestAttachment, 'meta' >; export interface CreateSideloadFile { image_size?: string; upload_request?: string; + generate_sub_sizes?: boolean; + convert_format?: boolean; } export interface SideloadAdditionalData { diff --git a/packages/media-utils/src/utils/upload-media.ts b/packages/media-utils/src/utils/upload-media.ts index 278b5c4ababf7a..79c6fe0697101a 100644 --- a/packages/media-utils/src/utils/upload-media.ts +++ b/packages/media-utils/src/utils/upload-media.ts @@ -22,6 +22,7 @@ import { UploadError } from './upload-error'; declare global { interface Window { __clientSideMediaProcessing?: boolean; + __heicUploadSupport?: boolean; } } diff --git a/packages/upload-media/src/feature-detection.ts b/packages/upload-media/src/feature-detection.ts index a69d7cfaad8d17..1b05187594effd 100644 --- a/packages/upload-media/src/feature-detection.ts +++ b/packages/upload-media/src/feature-detection.ts @@ -165,6 +165,23 @@ export function isClientSideMediaSupported(): boolean { return detectClientSideMediaSupport().supported; } +/** + * Detects whether the browser can decode HEIC images via canvas APIs. + * + * This checks for createImageBitmap and OffscreenCanvas support, + * which are sufficient to convert HEIC to JPEG without VIPS/WASM. + * Safari supports both APIs and can natively decode HEIC via + * createImageBitmap(), leveraging macOS platform codecs. + * + * @return Whether HEIC canvas-based processing is supported. + */ +export function isHeicCanvasSupported(): boolean { + return ( + typeof createImageBitmap !== 'undefined' && + typeof OffscreenCanvas !== 'undefined' + ); +} + /** * Clears the cached feature detection result. * diff --git a/packages/upload-media/src/index.ts b/packages/upload-media/src/index.ts index ec4225139c4b1b..032a99367c2c94 100644 --- a/packages/upload-media/src/index.ts +++ b/packages/upload-media/src/index.ts @@ -10,6 +10,7 @@ export { UploadError } from './upload-error'; export { detectClientSideMediaSupport, isClientSideMediaSupported, + isHeicCanvasSupported, clearFeatureDetectionCache, } from './feature-detection'; diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 5de99b73efdbea..1d27390f29ead0 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -20,6 +20,7 @@ import { renameFile, } from '../utils'; import { canvasConvertToJpeg } from '../canvas-utils'; +import { isClientSideMediaSupported } from '../feature-detection'; import { CLIENT_SIDE_SUPPORTED_MIME_TYPES, HEIC_MIME_TYPES } from './constants'; import { StubFile } from '../stub-file'; import { UploadError } from '../upload-error'; @@ -1165,6 +1166,10 @@ export function generateThumbnails( id: QueueItemId ) { ) : jpegConversion; + // When VIPS is not available (e.g. Safari), tell the server + // to generate all sub-sizes from the sideloaded JPEG. + const vipsAvailable = isClientSideMediaSupported(); + dispatch.addSideloadItem( { file: scaledFile, onChange: ( [ updatedAttachment ] ) => { @@ -1179,9 +1184,20 @@ export function generateThumbnails( id: QueueItemId ) { post: attachment.id, image_size: 'scaled', convert_format: false, + ...( ! vipsAvailable && { + generate_sub_sizes: true, + } ), }, operations: [ OperationType.Upload ], } ); + + // When VIPS is not available, the server generates sub-sizes + // from the sideloaded JPEG, so skip client-side thumbnail + // generation entirely. + if ( ! vipsAvailable ) { + dispatch.finishOperation( id, {} ); + return; + } } } From 17df2d70a93bb6b938d60fba86352c0f58220095 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 24 Mar 2026 10:55:44 -0700 Subject: [PATCH 11/24] Add unit tests for canvas utils, grid parsing, and utils Cover previously untested HEIC functionality: canvas conversion strategies, grid/tiled image parsing (iPhone format), isHeicCanvasSupported feature detection, and renameFile/cloneFile utilities. Also add error case tests for missing ipco/ipma, missing location data, and missing property associations in the HEIC parser. --- .../upload-media/src/test/canvas-utils.ts | 198 +++++++++++ .../src/test/feature-detection.ts | 62 ++++ packages/upload-media/src/test/heic-parser.ts | 335 ++++++++++++++++++ .../src/test/rename-and-clone-file.ts | 87 +++++ 4 files changed, 682 insertions(+) create mode 100644 packages/upload-media/src/test/canvas-utils.ts create mode 100644 packages/upload-media/src/test/rename-and-clone-file.ts diff --git a/packages/upload-media/src/test/canvas-utils.ts b/packages/upload-media/src/test/canvas-utils.ts new file mode 100644 index 00000000000000..8eb3c5a64d49d5 --- /dev/null +++ b/packages/upload-media/src/test/canvas-utils.ts @@ -0,0 +1,198 @@ +/** + * Internal dependencies + */ +import { canvasConvertToJpeg } from '../canvas-utils'; + +describe( 'canvasConvertToJpeg', () => { + const originalCreateImageBitmap = global.createImageBitmap; + const originalOffscreenCanvas = global.OffscreenCanvas; + const originalImageDecoder = ( global as any ).ImageDecoder; + const originalVideoDecoder = ( global as any ).VideoDecoder; + + afterEach( () => { + // Restore all globals. + if ( originalCreateImageBitmap ) { + global.createImageBitmap = originalCreateImageBitmap; + } else { + // @ts-ignore + delete global.createImageBitmap; + } + if ( originalOffscreenCanvas ) { + global.OffscreenCanvas = originalOffscreenCanvas; + } else { + // @ts-ignore + delete global.OffscreenCanvas; + } + if ( originalImageDecoder ) { + ( global as any ).ImageDecoder = originalImageDecoder; + } else { + delete ( global as any ).ImageDecoder; + } + if ( originalVideoDecoder ) { + ( global as any ).VideoDecoder = originalVideoDecoder; + } else { + delete ( global as any ).VideoDecoder; + } + } ); + + describe( 'Strategy 1: createImageBitmap + OffscreenCanvas', () => { + it( 'should convert via createImageBitmap when available', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + + const mockBitmap = { + width: 200, + height: 150, + close: jest.fn(), + }; + + const mockCtx = { + drawImage: jest.fn(), + }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest.fn().mockReturnValue( mockCtx ), + convertToBlob: jest.fn().mockResolvedValue( jpegBlob ), + } ) ); + + const file = new File( [ 'heic-data' ], 'photo.heic', { + type: 'image/heic', + } ); + const result = await canvasConvertToJpeg( file ); + + expect( result ).toBeInstanceOf( File ); + expect( result.name ).toBe( 'photo.jpeg' ); + expect( result.type ).toBe( 'image/jpeg' ); + expect( mockBitmap.close ).toHaveBeenCalled(); + expect( global.createImageBitmap ).toHaveBeenCalledWith( file ); + } ); + + it( 'should use the specified quality', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + + const mockConvertToBlob = jest.fn().mockResolvedValue( jpegBlob ); + const mockBitmap = { width: 100, height: 100, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest + .fn() + .mockReturnValue( { drawImage: jest.fn() } ), + convertToBlob: mockConvertToBlob, + } ) ); + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + await canvasConvertToJpeg( file, 0.5 ); + + expect( mockConvertToBlob ).toHaveBeenCalledWith( { + type: 'image/jpeg', + quality: 0.5, + } ); + } ); + + it( 'should strip the extension and use .jpeg', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + const mockBitmap = { width: 10, height: 10, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest + .fn() + .mockReturnValue( { drawImage: jest.fn() } ), + convertToBlob: jest.fn().mockResolvedValue( jpegBlob ), + } ) ); + + const file = new File( [ 'data' ], 'my-photo.HEIC', { + type: 'image/heic', + } ); + const result = await canvasConvertToJpeg( file ); + expect( result.name ).toBe( 'my-photo.jpeg' ); + } ); + + it( 'should close the bitmap even if canvas context fails', async () => { + const mockBitmap = { width: 10, height: 10, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest.fn().mockReturnValue( null ), + convertToBlob: jest.fn(), + } ) ); + + // Remove other decoders so it falls through to the final error. + delete ( global as any ).ImageDecoder; + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + expect( mockBitmap.close ).toHaveBeenCalled(); + } ); + } ); + + describe( 'fallback behavior', () => { + it( 'should throw when no strategy is available', async () => { + // createImageBitmap throws (doesn't support HEIC). + global.createImageBitmap = jest + .fn() + .mockRejectedValue( new Error( 'Unsupported format' ) ); + // No ImageDecoder or VideoDecoder. + delete ( global as any ).ImageDecoder; + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + } ); + + it( 'should fall through Strategy 1 failure to subsequent strategies', async () => { + // Strategy 1 fails. + global.createImageBitmap = jest + .fn() + .mockRejectedValue( new Error( 'Unsupported' ) ); + + // Strategy 2: ImageDecoder not supported for this type. + ( global as any ).ImageDecoder = { + isTypeSupported: jest.fn().mockResolvedValue( false ), + }; + + // No VideoDecoder. + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + + expect( + ( global as any ).ImageDecoder.isTypeSupported + ).toHaveBeenCalledWith( 'image/heic' ); + } ); + } ); +} ); diff --git a/packages/upload-media/src/test/feature-detection.ts b/packages/upload-media/src/test/feature-detection.ts index b96ced8a50df7c..3c4572892747df 100644 --- a/packages/upload-media/src/test/feature-detection.ts +++ b/packages/upload-media/src/test/feature-detection.ts @@ -4,6 +4,7 @@ import { detectClientSideMediaSupport, isClientSideMediaSupported, + isHeicCanvasSupported, clearFeatureDetectionCache, } from '../feature-detection'; @@ -289,6 +290,67 @@ describe( 'feature-detection', () => { } ); } ); + describe( 'isHeicCanvasSupported', () => { + const originalCreateImageBitmap = + global.createImageBitmap as typeof createImageBitmap; + const originalOffscreenCanvas = + global.OffscreenCanvas as typeof OffscreenCanvas; + + afterEach( () => { + // Restore globals after each test. + if ( originalCreateImageBitmap !== undefined ) { + global.createImageBitmap = + originalCreateImageBitmap as typeof createImageBitmap; + } else { + // @ts-ignore + delete global.createImageBitmap; + } + if ( originalOffscreenCanvas !== undefined ) { + global.OffscreenCanvas = + originalOffscreenCanvas as typeof OffscreenCanvas; + } else { + // @ts-ignore + delete global.OffscreenCanvas; + } + } ); + + it( 'returns true when both createImageBitmap and OffscreenCanvas are available', () => { + global.createImageBitmap = + jest.fn() as unknown as typeof createImageBitmap; + global.OffscreenCanvas = + jest.fn() as unknown as typeof OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( true ); + } ); + + it( 'returns false when createImageBitmap is unavailable', () => { + // @ts-ignore + delete global.createImageBitmap; + global.OffscreenCanvas = + jest.fn() as unknown as typeof OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + + it( 'returns false when OffscreenCanvas is unavailable', () => { + global.createImageBitmap = + jest.fn() as unknown as typeof createImageBitmap; + // @ts-ignore + delete global.OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + + it( 'returns false when both are unavailable', () => { + // @ts-ignore + delete global.createImageBitmap; + // @ts-ignore + delete global.OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + } ); + describe( 'clearFeatureDetectionCache', () => { it( 'clears the cached result', () => { const result1 = detectClientSideMediaSupport(); diff --git a/packages/upload-media/src/test/heic-parser.ts b/packages/upload-media/src/test/heic-parser.ts index 81372832efbc08..a3a2885411ffe2 100644 --- a/packages/upload-media/src/test/heic-parser.ts +++ b/packages/upload-media/src/test/heic-parser.ts @@ -332,6 +332,282 @@ describe( 'heic-parser', () => { } ); } ); + describe( 'parseHeic – grid/tiled image', () => { + /** + * Build a grid HEIC file with multiple tiles (like iPhone photos). + * + * The structure is: ftyp, meta (hdlr, pitm, iinf, iref, iloc, iprp), mdat. + * The primary item is type 'grid' referencing tile items of type 'hvc1'. + */ + function buildGridHeic( { + rows = 2, + columns = 2, + tileWidth = 512, + tileHeight = 512, + outputWidth = 1024, + outputHeight = 1024, + tileData = new Uint8Array( [ 0xca, 0xfe ] ), + }: { + rows?: number; + columns?: number; + tileWidth?: number; + tileHeight?: number; + outputWidth?: number; + outputHeight?: number; + tileData?: Uint8Array; + } = {} ): ArrayBuffer { + const gridItemId = 1; + const tileCount = rows * columns; + // Tile item IDs start at 2. + const tileItemIds = Array.from( + { length: tileCount }, + ( _, i ) => i + 2 + ); + + // Grid descriptor: version(1), flags(1), rows_minus_one(1), columns_minus_one(1), + // output_width(2), output_height(2) = 8 bytes (small fields). + const gridDescriptor = new Uint8Array( 8 ); + gridDescriptor[ 0 ] = 0; // version + gridDescriptor[ 1 ] = 0; // flags (no large fields) + gridDescriptor[ 2 ] = rows - 1; + gridDescriptor[ 3 ] = columns - 1; + const gdv = new DataView( gridDescriptor.buffer ); + gdv.setUint16( 4, outputWidth ); + gdv.setUint16( 6, outputHeight ); + + // Build property boxes (1=hvcC, 2=ispe) + const hvcC = buildHvcC(); + const ispe = buildIspe( tileWidth, tileHeight ); + const ipco = buildIpco( hvcC, ispe ); + + // ipma: each tile item → [1, 2] (hvcC at index 1, ispe at index 2) + const associations: Array< [ number, number[] ] > = tileItemIds.map( + ( id ) => [ id, [ 1, 2 ] ] as [ number, number[] ] + ); + const ipma = buildIpma( associations ); + const iprp = buildIprp( ipco, ipma ); + + // iinf: grid item type 'grid', tile items type 'hvc1'. + const iinf = buildIinf( [ + [ gridItemId, 'grid' ], + ...tileItemIds.map( + ( id ) => [ id, 'hvc1' ] as [ number, string ] + ), + ] ); + + // iref: dimg reference from grid to tiles. + const iref = buildIref( gridItemId, tileItemIds ); + + // Build everything except iloc and mdat first to calculate offsets. + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); + const hdlr = buildHdlr(); + const pitm = buildPitm( gridItemId ); + + // We need iloc for grid item + all tile items. + // First, build a placeholder iloc to calculate meta size. + const placeholderItems: Array< + [ number, Array< [ number, number ] > ] + > = [ + [ gridItemId, [ [ 0, gridDescriptor.length ] ] ], + ...tileItemIds.map( + ( id ) => + [ id, [ [ 0, tileData.length ] ] ] as [ + number, + Array< [ number, number ] >, + ] + ), + ]; + const placeholderIloc = buildIloc( placeholderItems ); + + const metaChildren = concat( + hdlr, + pitm, + iinf, + iref, + placeholderIloc, + iprp + ); + const metaSize = 8 + 4 + metaChildren.length; // box header + fullbox + const mdatHeaderSize = 8; + const mdatDataStart = ftyp.length + metaSize + mdatHeaderSize; + + // Grid descriptor is at the start of mdat, tiles follow. + const gridOffset = mdatDataStart; + let currentOffset = gridOffset + gridDescriptor.length; + const tileOffsets: number[] = []; + for ( let i = 0; i < tileCount; i++ ) { + tileOffsets.push( currentOffset ); + currentOffset += tileData.length; + } + + // Build real iloc. + const realItems: Array< [ number, Array< [ number, number ] > ] > = + [ + [ gridItemId, [ [ gridOffset, gridDescriptor.length ] ] ], + ...tileItemIds.map( + ( id, i ) => + [ + id, + [ [ tileOffsets[ i ], tileData.length ] ], + ] as [ number, Array< [ number, number ] > ] + ), + ]; + const iloc = buildIloc( realItems ); + + const realMetaChildren = concat( + hdlr, + pitm, + iinf, + iref, + iloc, + iprp + ); + const meta = buildFullBox( 'meta', 0, 0, realMetaChildren ); + + // Build mdat: grid descriptor + tile data. + const mdatPayload = concat( + gridDescriptor, + ...Array( tileCount ).fill( tileData ) + ); + const mdat = buildBox( 'mdat', mdatPayload ); + + return concat( ftyp, meta, mdat ).buffer; + } + + /** Build an iinf box with infe entries. */ + function buildIinf( items: Array< [ number, string ] > ): Uint8Array { + // Each infe: version=2, flags=0, itemId(u16), protection_index(u16), item_type(4 bytes) + const infeBoxes = items.map( ( [ itemId, itemType ] ) => { + const infeData = new Uint8Array( 8 ); + const idv = new DataView( infeData.buffer ); + idv.setUint16( 0, itemId ); + idv.setUint16( 2, 0 ); // protection index + for ( let k = 0; k < 4; k++ ) { + infeData[ 4 + k ] = itemType.charCodeAt( k ); + } + return buildFullBox( 'infe', 2, 0, infeData ); + } ); + + // iinf: version=0, entry_count(u16), then infe boxes. + const countData = new Uint8Array( 2 ); + new DataView( countData.buffer ).setUint16( 0, items.length ); + return buildFullBox( + 'iinf', + 0, + 0, + concat( countData, ...infeBoxes ) + ); + } + + /** Build an iref box with a single dimg reference. */ + function buildIref( fromId: number, toIds: number[] ): Uint8Array { + // dimg reference box: fromId(u16), refCount(u16), toIds(u16 each) + const dimgData = new Uint8Array( 4 + toIds.length * 2 ); + const dv = new DataView( dimgData.buffer ); + dv.setUint16( 0, fromId ); + dv.setUint16( 2, toIds.length ); + for ( let i = 0; i < toIds.length; i++ ) { + dv.setUint16( 4 + i * 2, toIds[ i ] ); + } + const dimgBox = buildBox( 'dimg', dimgData ); + + // iref is a FullBox (version=0). + return buildFullBox( 'iref', 0, 0, dimgBox ); + } + + it( 'should parse a 2x2 grid HEIC', () => { + const tileData = new Uint8Array( [ 0x11, 0x22, 0x33 ] ); + const buffer = buildGridHeic( { + rows: 2, + columns: 2, + tileWidth: 512, + tileHeight: 512, + outputWidth: 1024, + outputHeight: 1024, + tileData, + } ); + + const result = parseHeic( buffer ); + + expect( result.outputWidth ).toBe( 1024 ); + expect( result.outputHeight ).toBe( 1024 ); + expect( result.tileWidth ).toBe( 512 ); + expect( result.tileHeight ).toBe( 512 ); + expect( result.tiles ).toHaveLength( 4 ); + } ); + + it( 'should place tiles in correct grid positions', () => { + const buffer = buildGridHeic( { + rows: 2, + columns: 2, + tileWidth: 256, + tileHeight: 256, + } ); + + const result = parseHeic( buffer ); + + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 0 ].y ).toBe( 0 ); + expect( result.tiles[ 1 ].x ).toBe( 256 ); + expect( result.tiles[ 1 ].y ).toBe( 0 ); + expect( result.tiles[ 2 ].x ).toBe( 0 ); + expect( result.tiles[ 2 ].y ).toBe( 256 ); + expect( result.tiles[ 3 ].x ).toBe( 256 ); + expect( result.tiles[ 3 ].y ).toBe( 256 ); + } ); + + it( 'should extract tile data for each tile', () => { + const tileData = new Uint8Array( [ 0xaa, 0xbb ] ); + const buffer = buildGridHeic( { tileData } ); + + const result = parseHeic( buffer ); + + for ( const tile of result.tiles ) { + expect( tile.data ).toEqual( tileData ); + } + } ); + + it( 'should extract codec string from grid tiles', () => { + const buffer = buildGridHeic(); + const result = parseHeic( buffer ); + + // Same hvcC as single image tests. + expect( result.codecString ).toBe( 'hvc1.1.6.L93.B0' ); + } ); + + it( 'should parse a 1x3 grid', () => { + const buffer = buildGridHeic( { + rows: 1, + columns: 3, + tileWidth: 100, + tileHeight: 300, + outputWidth: 300, + outputHeight: 300, + } ); + + const result = parseHeic( buffer ); + + expect( result.tiles ).toHaveLength( 3 ); + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 1 ].x ).toBe( 100 ); + expect( result.tiles[ 2 ].x ).toBe( 200 ); + expect( result.outputWidth ).toBe( 300 ); + expect( result.outputHeight ).toBe( 300 ); + } ); + + it( 'should extract the HEVCDecoderConfigurationRecord from grid', () => { + const buffer = buildGridHeic(); + const result = parseHeic( buffer ); + + expect( result.description ).toBeInstanceOf( Uint8Array ); + expect( result.description.length ).toBe( 23 ); + expect( result.description[ 0 ] ).toBe( 1 ); // configurationVersion + } ); + } ); + describe( 'parseHeic – error cases', () => { it( 'should throw for empty buffer', () => { expect( () => parseHeic( new ArrayBuffer( 0 ) ) ).toThrow( @@ -357,6 +633,65 @@ describe( 'heic-parser', () => { 'Missing required boxes' ); } ); + + it( 'should throw for missing ipco or ipma inside iprp', () => { + // meta with pitm, iloc, iprp but iprp is empty (no ipco/ipma) + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 1, [ [ 0, 4 ] ] ] ] ); + const emptyIprp = buildBox( 'iprp', new Uint8Array( 0 ) ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, emptyIprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'Missing ipco or ipma' + ); + } ); + + it( 'should throw when primary item has no location data', () => { + // iloc with item ID 99, but pitm says primary is 1 + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 99, [ [ 0, 4 ] ] ] ] ); + const hvcC = buildHvcC(); + const ispe = buildIspe( 100, 100 ); + const ipco = buildIpco( hvcC, ispe ); + const ipma = buildIpma( [ [ 1, [ 1, 2 ] ] ] ); + const iprp = buildIprp( ipco, ipma ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, iprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'No location data for primary item' + ); + } ); + + it( 'should throw when primary item has no property associations', () => { + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 1, [ [ 0, 4 ] ] ] ] ); + const hvcC = buildHvcC(); + const ispe = buildIspe( 100, 100 ); + const ipco = buildIpco( hvcC, ispe ); + // ipma associates item 99, not item 1 + const ipma = buildIpma( [ [ 99, [ 1, 2 ] ] ] ); + const iprp = buildIprp( ipco, ipma ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, iprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'No property associations' + ); + } ); } ); } ); diff --git a/packages/upload-media/src/test/rename-and-clone-file.ts b/packages/upload-media/src/test/rename-and-clone-file.ts new file mode 100644 index 00000000000000..65e86e40d67765 --- /dev/null +++ b/packages/upload-media/src/test/rename-and-clone-file.ts @@ -0,0 +1,87 @@ +/** + * Internal dependencies + */ +import { renameFile, cloneFile } from '../utils'; + +describe( 'renameFile', () => { + it( 'should return a new File with the given name', () => { + const original = new File( [ 'content' ], 'old-name.jpg', { + type: 'image/jpeg', + lastModified: 1700000000000, + } ); + + const result = renameFile( original, 'new-name.png' ); + + expect( result ).toBeInstanceOf( File ); + expect( result.name ).toBe( 'new-name.png' ); + expect( result ).not.toBe( original ); + } ); + + it( 'should preserve the original file type', () => { + const original = new File( [ 'content' ], 'photo.heic', { + type: 'image/heic', + } ); + + const result = renameFile( original, 'photo.jpeg' ); + + expect( result.type ).toBe( 'image/heic' ); + } ); + + it( 'should preserve lastModified timestamp', () => { + const timestamp = 1600000000000; + const original = new File( [ 'content' ], 'file.txt', { + type: 'text/plain', + lastModified: timestamp, + } ); + + const result = renameFile( original, 'renamed.txt' ); + + expect( result.lastModified ).toBe( timestamp ); + } ); + + it( 'should preserve file size', () => { + const original = new File( [ 'hello world' ], 'file.txt', { + type: 'text/plain', + } ); + + const result = renameFile( original, 'new.txt' ); + + expect( result.size ).toBe( original.size ); + } ); +} ); + +describe( 'cloneFile', () => { + it( 'should return a new File with the same name', () => { + const original = new File( [ 'content' ], 'photo.jpg', { + type: 'image/jpeg', + } ); + + const result = cloneFile( original ); + + expect( result ).toBeInstanceOf( File ); + expect( result ).not.toBe( original ); + expect( result.name ).toBe( 'photo.jpg' ); + } ); + + it( 'should preserve type and lastModified', () => { + const original = new File( [ 'content' ], 'doc.pdf', { + type: 'application/pdf', + lastModified: 1500000000000, + } ); + + const result = cloneFile( original ); + + expect( result.type ).toBe( 'application/pdf' ); + expect( result.lastModified ).toBe( 1500000000000 ); + } ); + + it( 'should preserve file size', () => { + const original = new File( [ 'test data' ], 'test.bin', { + type: 'application/octet-stream', + } ); + + const result = cloneFile( original ); + + expect( result.size ).toBe( original.size ); + } ); +} ); From cbf4dfb71e1b217982a3c699a9d711f65058fdaf Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 30 Mar 2026 17:40:59 -0700 Subject: [PATCH 12/24] Fix ImageDecoder resource leak and duplicate onBatchSuccess callback Wrap ImageDecoder strategy in try/finally to ensure decoder.close() and videoFrame.close() are always called, even on decode failure. Coordinate onBatchSuccess in heicMediaUpload so it fires once when a batch contains both HEIC and non-HEIC files routed through separate upload paths. --- .../src/components/provider/index.js | 18 ++++++- packages/upload-media/src/canvas-utils.ts | 47 ++++++++++--------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/packages/block-editor/src/components/provider/index.js b/packages/block-editor/src/components/provider/index.js index c6e2e758de00d6..379aad102606a9 100644 --- a/packages/block-editor/src/components/provider/index.js +++ b/packages/block-editor/src/components/provider/index.js @@ -217,6 +217,20 @@ function heicMediaUpload( ( file ) => ! HEIC_MIME_TYPES.includes( file.type ) ); + // When the batch contains both HEIC and non-HEIC files, coordinate + // onBatchSuccess so it fires only after *both* paths have completed. + const hasBothPaths = + heicFiles.length > 0 && otherFiles.length > 0 && settings?.mediaUpload; + let pathsRemaining = hasBothPaths ? 2 : 1; + const coordinatedBatchSuccess = hasBothPaths + ? () => { + pathsRemaining--; + if ( pathsRemaining <= 0 ) { + onBatchSuccess?.(); + } + } + : onBatchSuccess; + // Route HEIC files through the upload-media pipeline. if ( heicFiles.length > 0 ) { void registry.dispatch( uploadStore ).addItems( { @@ -226,7 +240,7 @@ function heicMediaUpload( settings?.[ mediaUploadOnSuccessKey ]?.( attachments ); onSuccess?.( attachments ); }, - onBatchSuccess, + onBatchSuccess: coordinatedBatchSuccess, onError: ( error ) => onError( typeof error === 'string' ? error : error?.message ?? '' @@ -245,7 +259,7 @@ function heicMediaUpload( onError, onFileChange, onSuccess, - onBatchSuccess, + onBatchSuccess: coordinatedBatchSuccess, } ); } } diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 232b209003abf1..217dfecd010f34 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -67,32 +67,35 @@ export async function canvasConvertToJpeg( type: file.type, data: file.stream(), } ); - const { image: videoFrame } = await decoder.decode(); - - const canvas = new OffscreenCanvas( - videoFrame.displayWidth, - videoFrame.displayHeight - ); - const ctx = canvas.getContext( '2d' ); + try { + const { image: videoFrame } = await decoder.decode(); + try { + const canvas = new OffscreenCanvas( + videoFrame.displayWidth, + videoFrame.displayHeight + ); + const ctx = canvas.getContext( '2d' ); - if ( ! ctx ) { - videoFrame.close(); - decoder.close(); - throw new Error( 'Could not get canvas 2d context' ); - } + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } - ctx.drawImage( videoFrame, 0, 0 ); - videoFrame.close(); - decoder.close(); + ctx.drawImage( videoFrame, 0, 0 ); - const jpegBlob = await canvas.convertToBlob( { - type: 'image/jpeg', - quality, - } ); + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); - return new File( [ jpegBlob ], `${ baseName }.jpeg`, { - type: 'image/jpeg', - } ); + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } finally { + videoFrame.close(); + } + } finally { + decoder.close(); + } } } From 7721b4ca1be27fdae704f9d26c969062e4bdb247 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 2 Apr 2026 14:42:40 -0700 Subject: [PATCH 13/24] Ensure wp-admin/includes/image.php is loaded before calling wp_create_image_subsizes The function wp_create_image_subsizes() is defined in wp-admin/includes/image.php which is not automatically loaded in REST API context, causing a fatal error when sideloading HEIC-converted JPEGs in Safari. --- lib/media/class-gutenberg-rest-attachments-controller.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index a682277faf5539..1178b12392316e 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -559,6 +559,11 @@ public function sideload_item( WP_REST_Request $request ) { // This handles the case where the client converted HEIC to JPEG via // canvas but lacks VIPS/WASM for client-side thumbnail generation. if ( $request['generate_sub_sizes'] && 'scaled' === $image_size ) { + // Ensure the image functions are available (not loaded by default in REST context). + if ( ! function_exists( 'wp_create_image_subsizes' ) ) { + require_once ABSPATH . 'wp-admin/includes/image.php'; + } + // Use wp_create_image_subsizes which generates all registered // sub-sizes and updates the attachment metadata. $new_metadata = wp_create_image_subsizes( $path, $attachment_id ); From cecea0028f50f1ee7a1fea96cea3147fa0ccf62d Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 2 Apr 2026 14:56:49 -0700 Subject: [PATCH 14/24] Parse irot box from HEIC container and apply rotation in canvas conversion iPhone portrait photos are stored as landscape pixels with an irot (image rotation) property in the ISOBMFF container. Safari handles this automatically via createImageBitmap, but Chrome's Strategy 3 (manual HEIC parsing + VideoDecoder) was ignoring the irot box, resulting in rotated images. Parse the irot property box for both single-image and grid HEIC files, and apply the rotation to the canvas output. --- packages/upload-media/src/canvas-utils.ts | 42 ++++++++++++++++- packages/upload-media/src/heic-parser.ts | 46 +++++++++++++++++-- packages/upload-media/src/test/heic-parser.ts | 41 +++++++++++++++-- 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts index 217dfecd010f34..909e7072df72d5 100644 --- a/packages/upload-media/src/canvas-utils.ts +++ b/packages/upload-media/src/canvas-utils.ts @@ -139,7 +139,10 @@ export async function canvasConvertToJpeg( } } - const jpegBlob = await canvas.convertToBlob( { + // Apply ISOBMFF irot rotation if present. + const outputCanvas = applyRotation( canvas, heicData.rotation ); + + const jpegBlob = await outputCanvas.convertToBlob( { type: 'image/jpeg', quality, } ); @@ -159,6 +162,43 @@ export async function canvasConvertToJpeg( ); } +/** + * Apply ISOBMFF irot rotation to a canvas. + * + * Returns the original canvas if no rotation is needed, or a new + * OffscreenCanvas with the rotation applied. + * + * @param source Source canvas with the decoded image. + * @param rotation Rotation angle in degrees counter-clockwise (0, 90, 180, 270). + * @return Canvas with rotation applied. + */ +function applyRotation( + source: OffscreenCanvas, + rotation: number +): OffscreenCanvas { + if ( rotation === 0 ) { + return source; + } + + const swap = rotation === 90 || rotation === 270; + const w = swap ? source.height : source.width; + const h = swap ? source.width : source.height; + + const rotated = new OffscreenCanvas( w, h ); + const ctx = rotated.getContext( '2d' ); + + if ( ! ctx ) { + return source; + } + + ctx.translate( w / 2, h / 2 ); + // irot angle is CCW; canvas rotate() is CW, so negate. + ctx.rotate( ( -rotation * Math.PI ) / 180 ); + ctx.drawImage( source, -source.width / 2, -source.height / 2 ); + + return rotated; +} + /** * Decode a single HEVC key frame using the WebCodecs VideoDecoder API. * diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts index 71830c202c5465..4625263486dc61 100644 --- a/packages/upload-media/src/heic-parser.ts +++ b/packages/upload-media/src/heic-parser.ts @@ -37,6 +37,8 @@ export interface HeicImageData { outputWidth: number; /** Final output height in pixels. */ outputHeight: number; + /** Rotation angle in degrees counter-clockwise (0, 90, 180, 270). */ + rotation: number; } // --------------------------------------------------------------------------- @@ -309,6 +311,20 @@ function parseIspe( return { width: r.u32(), height: r.u32() }; } +/** + * Parse Image Rotation box → rotation angle in degrees CCW. + * + * Format: 1 byte with reserved (6 bits) + angle (2 bits). + * angle * 90 = rotation in degrees counter-clockwise. + * + * @param r Binary reader. + * @param box BoxInfo for the irot box. + */ +function parseIrot( r: Reader, box: BoxInfo ): number { + r.pos = box.offset + box.headerSize; + return ( r.u8() & 0x3 ) * 90; +} + /** * Parse Item Info Box → map of item ID to item type (4-char code). * @@ -507,7 +523,7 @@ function readItemData( } /** - * Find hvcC and ispe property boxes for a given item. + * Find hvcC, ispe, and irot property boxes for a given item. * * @param propIndices 1-based property indices from ipma. * @param properties All property boxes from ipco. @@ -515,9 +531,10 @@ function readItemData( function findHvcProperties( propIndices: number[], properties: BoxInfo[] -): { hvcCBox: BoxInfo; ispeBox: BoxInfo } { +): { hvcCBox: BoxInfo; ispeBox: BoxInfo; irotBox?: BoxInfo } { let hvcCBox: BoxInfo | undefined; let ispeBox: BoxInfo | undefined; + let irotBox: BoxInfo | undefined; for ( const idx of propIndices ) { if ( idx < 1 || idx > properties.length ) { @@ -530,6 +547,9 @@ function findHvcProperties( if ( prop.type === 'ispe' && ! ispeBox ) { ispeBox = prop; } + if ( prop.type === 'irot' && ! irotBox ) { + irotBox = prop; + } } if ( ! hvcCBox ) { @@ -539,7 +559,7 @@ function findHvcProperties( throw new Error( 'No image dimensions (ispe) found' ); } - return { hvcCBox, ispeBox }; + return { hvcCBox, ispeBox, irotBox }; } // --------------------------------------------------------------------------- @@ -645,7 +665,7 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { throw new Error( 'No property associations for primary item' ); } - const { hvcCBox, ispeBox } = findHvcProperties( + const { hvcCBox, ispeBox, irotBox } = findHvcProperties( primaryPropIndices, properties ); @@ -657,6 +677,7 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { ); const codecString = buildCodecString( r, hvcCDataStart ); const { width, height } = parseIspe( r, ispeBox ); + const rotation = irotBox ? parseIrot( r, irotBox ) : 0; return { codecString, @@ -672,6 +693,7 @@ export function parseHeic( buffer: ArrayBuffer ): HeicImageData { tileHeight: height, outputWidth: width, outputHeight: height, + rotation, }; } @@ -761,6 +783,19 @@ function parseGridImage( properties ); + // irot is associated with the grid item, not the tiles. + const gridProps = allAssoc.get( gridItemId ) || []; + let irotBox: BoxInfo | undefined; + for ( const idx of gridProps ) { + if ( idx >= 1 && idx <= properties.length ) { + const prop = properties[ idx - 1 ]; + if ( prop.type === 'irot' ) { + irotBox = prop; + break; + } + } + } + const hvcCDataStart = hvcCBox.offset + hvcCBox.headerSize; const hvcCDataSize = hvcCBox.size - hvcCBox.headerSize; const description = new Uint8Array( @@ -787,6 +822,8 @@ function parseGridImage( } } + const rotation = irotBox ? parseIrot( r, irotBox ) : 0; + return { codecString, description, @@ -795,6 +832,7 @@ function parseGridImage( tileHeight, outputWidth, outputHeight, + rotation, }; } diff --git a/packages/upload-media/src/test/heic-parser.ts b/packages/upload-media/src/test/heic-parser.ts index a3a2885411ffe2..c7c523d80d61d9 100644 --- a/packages/upload-media/src/test/heic-parser.ts +++ b/packages/upload-media/src/test/heic-parser.ts @@ -106,6 +106,12 @@ function buildIspe( width: number, height: number ): Uint8Array { return buildFullBox( 'ispe', 0, 0, data ); } +/** Build an irot (Image Rotation) box. angle is 0-3 (multiplied by 90°). */ +function buildIrot( angle: number ): Uint8Array { + const data = new Uint8Array( [ angle & 0x3 ] ); + return buildBox( 'irot', data ); +} + /** Build an ipco (Item Property Container) with the given property boxes. */ function buildIpco( ...properties: Uint8Array[] ): Uint8Array { return buildBox( 'ipco', concat( ...properties ) ); @@ -209,18 +215,26 @@ function buildSingleImageHeic( { width = 100, height = 80, imageData = new Uint8Array( [ 0xde, 0xad, 0xbe, 0xef ] ), + rotation, }: { width?: number; height?: number; imageData?: Uint8Array; + rotation?: number; } = {} ): ArrayBuffer { const primaryItemId = 1; - // Build property boxes (1-indexed: 1=ispe, 2=hvcC) + // Build property boxes (1-indexed: 1=ispe, 2=hvcC, optionally 3=irot) const ispe = buildIspe( width, height ); const hvcC = buildHvcC(); - const ipco = buildIpco( ispe, hvcC ); - const ipma = buildIpma( [ [ primaryItemId, [ 1, 2 ] ] ] ); + const propBoxes: Uint8Array[] = [ ispe, hvcC ]; + const propIndices = [ 1, 2 ]; + if ( rotation !== undefined ) { + propBoxes.push( buildIrot( rotation / 90 ) ); + propIndices.push( 3 ); + } + const ipco = buildIpco( ...propBoxes ); + const ipma = buildIpma( [ [ primaryItemId, propIndices ] ] ); const iprp = buildIprp( ipco, ipma ); // We need to know where mdat data will be placed. @@ -330,6 +344,27 @@ describe( 'heic-parser', () => { expect( result.description.length ).toBe( 23 ); // minimal hvcC record expect( result.description[ 0 ] ).toBe( 1 ); // configurationVersion } ); + + it( 'should return rotation 0 when no irot box present', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 0 ); + } ); + + it( 'should parse 90° CCW rotation from irot box', () => { + const buffer = buildSingleImageHeic( { rotation: 90 } ); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 90 ); + } ); + + it( 'should parse 270° CCW rotation from irot box', () => { + const buffer = buildSingleImageHeic( { rotation: 270 } ); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 270 ); + } ); } ); describe( 'parseHeic – grid/tiled image', () => { From 42db17634ebdd03ea9ea91d8866785e03f4f8369 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 2 Apr 2026 15:53:57 -0700 Subject: [PATCH 15/24] Convert HEIC to JPEG before upload instead of sideloading Instead of uploading the HEIC file and then sideloading a converted JPEG, convert HEIC to JPEG client-side in prepareItem and upload the JPEG directly. The server handles it like any normal JPEG upload (threshold scaling, sub-sizes, EXIF rotation). This matches iOS behavior where HEIC is converted to JPEG on the fly, avoids orphaned files from conflicting original_image metadata, and simplifies the server-side code by removing the generate_sub_sizes handling in the sideload endpoint and the HEIC MIME type bypass. --- ...-gutenberg-rest-attachments-controller.php | 58 +------ .../upload-media/src/store/private-actions.ts | 149 +++++------------- 2 files changed, 42 insertions(+), 165 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 1178b12392316e..ea9adbc8cdaf2e 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -89,9 +89,7 @@ public function register_routes(): void { * Checks if a given request has access to create an attachment. * * Skips the server-side image type support check when the client - * will handle image processing (generate_sub_sizes is false), or - * when the file is HEIC/HEIF (client-side canvas fallback handles - * processing when the server's image editor doesn't support them). + * will handle image processing (generate_sub_sizes is false). * * @param WP_REST_Request $request Full details about the request. * @return true|WP_Error True if the request has access to create items, WP_Error object otherwise. @@ -99,19 +97,6 @@ public function register_routes(): void { public function create_item_permissions_check( $request ) { $bypass_mime_check = false === $request['generate_sub_sizes']; - // Always allow HEIC/HEIF uploads through even if the server's image - // editor doesn't support them. The client-side canvas fallback will - // handle processing using the browser's native HEVC decoder. - if ( ! $bypass_mime_check ) { - $files = $request->get_file_params(); - if ( - ! empty( $files['file']['type'] ) && - in_array( $files['file']['type'], array( 'image/heic', 'image/heif' ), true ) - ) { - $bypass_mime_check = true; - } - } - if ( $bypass_mime_check ) { add_filter( 'wp_prevent_unsupported_mime_type_uploads', '__return_false' ); } @@ -247,23 +232,6 @@ public function prepare_item_for_response( $item, $request ): WP_REST_Response { $data['missing_image_sizes'] = $missing_image_sizes; } - // HEIC/HEIF: the server's image editor cannot read these files, - // so missing_image_sizes is empty even though no sub-sizes were - // generated. Report all registered sizes as missing so the - // client-side canvas fallback can generate them. - if ( in_array( $mime_type, array( 'image/heic', 'image/heif' ), true ) ) { - $metadata = wp_get_attachment_metadata( $item->ID, true ); - - if ( ! is_array( $metadata ) ) { - $metadata = array(); - } - - $metadata['sizes'] = $metadata['sizes'] ?? array(); - - $registered_sizes = wp_get_registered_image_subsizes(); - $missing_image_sizes = array_diff( array_keys( $registered_sizes ), array_keys( $metadata['sizes'] ) ); - $data['missing_image_sizes'] = array_values( $missing_image_sizes ); - } } $context = ! empty( $request['context'] ) ? $request['context'] : 'view'; @@ -554,30 +522,6 @@ public function sideload_item( WP_REST_Request $request ) { wp_update_attachment_metadata( $attachment_id, $metadata ); - // When generate_sub_sizes is true (e.g. HEIC-only mode on Safari), - // generate all image sub-sizes server-side from the sideloaded JPEG. - // This handles the case where the client converted HEIC to JPEG via - // canvas but lacks VIPS/WASM for client-side thumbnail generation. - if ( $request['generate_sub_sizes'] && 'scaled' === $image_size ) { - // Ensure the image functions are available (not loaded by default in REST context). - if ( ! function_exists( 'wp_create_image_subsizes' ) ) { - require_once ABSPATH . 'wp-admin/includes/image.php'; - } - - // Use wp_create_image_subsizes which generates all registered - // sub-sizes and updates the attachment metadata. - $new_metadata = wp_create_image_subsizes( $path, $attachment_id ); - - if ( ! is_wp_error( $new_metadata ) ) { - // Preserve the original_image reference from HEIC. - if ( ! empty( $metadata['original_image'] ) ) { - $new_metadata['original_image'] = $metadata['original_image']; - } - - wp_update_attachment_metadata( $attachment_id, $new_metadata ); - } - } - $response_request = new WP_REST_Request( WP_REST_Server::READABLE, rest_get_route_for_post( $attachment_id ) diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 1d27390f29ead0..d09d6abcbbaf6d 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -13,12 +13,7 @@ type WPDataRegistry = ReturnType< typeof createRegistry >; /** * Internal dependencies */ -import { - cloneFile, - convertBlobToFile, - getFileBasename, - renameFile, -} from '../utils'; +import { cloneFile, convertBlobToFile, renameFile } from '../utils'; import { canvasConvertToJpeg } from '../canvas-utils'; import { isClientSideMediaSupported } from '../feature-detection'; import { CLIENT_SIDE_SUPPORTED_MIME_TYPES, HEIC_MIME_TYPES } from './constants'; @@ -744,6 +739,7 @@ export function prepareItem( id: QueueItemId ) { const operations: Operation[] = []; const settings = select.getSettings(); + let heicJpeg: File | null = null; const isImage = file.type.startsWith( 'image/' ); const isVipsSupported = CLIENT_SIDE_SUPPORTED_MIME_TYPES.includes( @@ -775,9 +771,30 @@ export function prepareItem( id: QueueItemId ) { OperationType.Finalize ); } else if ( isImage && isHeic ) { - // HEIC/HEIF: upload with server processing first. - // If the server can't generate sub-sizes, the client falls back - // to canvas-based decoding using the browser's native HEVC codec. + // HEIC/HEIF: convert to JPEG client-side before upload. + // The server may not support HEIC, so decode it using the + // browser's native HEVC codec (createImageBitmap or VideoDecoder) + // and upload the resulting JPEG. The server then handles it like + // any normal JPEG (threshold scaling, sub-sizes, etc.). + // This matches iOS behavior where HEIC is converted on the fly. + try { + heicJpeg = await canvasConvertToJpeg( + file, + settings.imageQuality ?? DEFAULT_OUTPUT_QUALITY + ); + } catch { + dispatch.cancelItem( + id, + new UploadError( { + code: 'HEIC_DECODE_ERROR', + message: + 'This browser cannot decode HEIC images and the server does not support them either. Please convert to JPEG before uploading.', + file, + } ) + ); + return; + } + operations.push( OperationType.Upload, OperationType.ThumbnailGeneration, @@ -795,16 +812,19 @@ export function prepareItem( id: QueueItemId ) { // If the file is not processed by vips, tell the server to // generate sub-sizes since they won't be created client-side. - // Exception: HEIC images — the client handles sub-sizes via - // canvas fallback, so tell the server NOT to generate them - // (and disable format conversion to prevent a duplicate JPEG). - let updates = {}; - if ( isHeic ) { + let updates: Partial< QueueItem > = {}; + if ( isHeic && heicJpeg ) { + // HEIC was converted to JPEG client-side. Upload the JPEG + // and let the server handle it normally (threshold scaling, + // sub-sizes, format conversion). + const vipsAvailable = isClientSideMediaSupported(); updates = { + file: heicJpeg, + sourceFile: heicJpeg, additionalData: { ...item.additionalData, - generate_sub_sizes: false, - convert_format: false, + generate_sub_sizes: ! vipsAvailable, + convert_format: true, }, }; } else if ( ! isVipsSupported || ! isImage ) { @@ -1119,93 +1139,10 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); - // HEIC/HEIF canvas fallback handling. - // When the source is HEIC, sideload the original and convert to JPEG - // using the browser's native decoder for sub-size generation. - const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); - let jpegConversion: File | null = null; - - // The initial upload already saved the HEIC file on the server. - // The 'scaled' sideload handler records it as original_image - // in attachment metadata, so no separate original sideload is needed. - - if ( - isHeicSource && - attachment.missing_image_sizes && - attachment.missing_image_sizes.length > 0 - ) { - // Convert HEIC to JPEG using browser's native HEVC decoder. - // Uses createImageBitmap() which leverages OS/browser-licensed codecs, - // avoiding patent concerns with shipping our own HEVC decoder. - try { - jpegConversion = await canvasConvertToJpeg( - item.sourceFile, - settings.imageQuality ?? DEFAULT_OUTPUT_QUALITY - ); - } catch { - dispatch.cancelItem( - id, - new UploadError( { - code: 'HEIC_DECODE_ERROR', - message: - 'This browser cannot decode HEIC images and the server does not support them either. Please convert to JPEG before uploading.', - file: item.sourceFile, - } ) - ); - return; - } - - // Sideload the full-size JPEG as the "scaled" version. - // This makes the JPEG the main displayable image while keeping - // the original HEIC stored as original_image in metadata. - if ( attachment.id ) { - const scaledFile = attachment.filename - ? renameFile( - jpegConversion, - getFileBasename( attachment.filename ) + '.jpeg' - ) - : jpegConversion; - - // When VIPS is not available (e.g. Safari), tell the server - // to generate all sub-sizes from the sideloaded JPEG. - const vipsAvailable = isClientSideMediaSupported(); - - dispatch.addSideloadItem( { - file: scaledFile, - onChange: ( [ updatedAttachment ] ) => { - if ( isBlobURL( updatedAttachment.url ) ) { - return; - } - item.onChange?.( [ updatedAttachment ] ); - }, - batchId: uuidv4(), - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'scaled', - convert_format: false, - ...( ! vipsAvailable && { - generate_sub_sizes: true, - } ), - }, - operations: [ OperationType.Upload ], - } ); - - // When VIPS is not available, the server generates sub-sizes - // from the sideloaded JPEG, so skip client-side thumbnail - // generation entirely. - if ( ! vipsAvailable ) { - dispatch.finishOperation( id, {} ); - return; - } - } - } - - // Check if image needs rotation (non-HEIC images only). + // Check if image needs rotation. // If exif_orientation is not 1, the image needs rotation. // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. - // HEIC rotation is handled by the browser's native decoder. - if ( ! isHeicSource ) { + { const needsRotation = attachment.exif_orientation && attachment.exif_orientation !== 1 && @@ -1256,10 +1193,7 @@ export function generateThumbnails( id: QueueItemId ) { const sizesToGenerate: string[] = attachment.missing_image_sizes as string[]; - // Use the JPEG conversion for HEIC images, or the sourceFile otherwise. - // For HEIC, vips can resize the JPEG efficiently. - // For other formats, vips auto-rotates based on EXIF orientation. - const thumbnailSource = jpegConversion ?? item.sourceFile; + const thumbnailSource = item.sourceFile; const file = attachment.filename ? renameFile( thumbnailSource, attachment.filename ) : thumbnailSource; @@ -1338,9 +1272,8 @@ export function generateThumbnails( id: QueueItemId ) { } ); } - // Create and sideload the scaled version (non-HEIC only). - // For HEIC, the scaled JPEG version is already sideloaded above. - if ( ! isHeicSource ) { + // Create and sideload the scaled version if it exceeds the threshold. + { const { bigImageSizeThreshold } = settings; if ( bigImageSizeThreshold && attachment.id ) { // Check if the image actually exceeds the threshold. From 2ce8b2695b0e1663e7bed792408bfff71aa2a9cc Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 2 Apr 2026 20:32:24 -0700 Subject: [PATCH 16/24] Fix PHPCS: remove blank line after control structure --- lib/media/class-gutenberg-rest-attachments-controller.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index ea9adbc8cdaf2e..6cd928b6965db1 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -231,7 +231,6 @@ public function prepare_item_for_response( $item, $request ): WP_REST_Response { $missing_image_sizes = array_diff( $merged_sizes, array_keys( $metadata['sizes'] ) ); $data['missing_image_sizes'] = $missing_image_sizes; } - } $context = ! empty( $request['context'] ) ? $request['context'] : 'view'; From 2f41440eb1acf04f3ca1bee2d7e5f2f8b8e7b069 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 7 Apr 2026 17:44:51 -0700 Subject: [PATCH 17/24] Preserve original HEIC file when uploading Keep the original HEIC in sourceFile instead of replacing it with the converted JPEG. After upload, sideload the HEIC as the "original" so WordPress stores it in attachment metadata (original_image). The uploaded JPEG remains the main display file and is used for all thumbnail generation. This addresses reviewer feedback that the original HEIC appeared lost after upload since both file and sourceFile were replaced with JPEG. Now the media library preserves the HEIC as the original image while all display and sub-size files remain as JPEG. Also skip vips rotation for HEIC sources since rotation is already handled during the canvas conversion step. --- .../upload-media/src/store/private-actions.ts | 31 ++- .../src/store/test/private-actions.js | 254 +++++++++++++++++- 2 files changed, 280 insertions(+), 5 deletions(-) diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index d09d6abcbbaf6d..60dcae2fb71271 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -816,11 +816,12 @@ export function prepareItem( id: QueueItemId ) { if ( isHeic && heicJpeg ) { // HEIC was converted to JPEG client-side. Upload the JPEG // and let the server handle it normally (threshold scaling, - // sub-sizes, format conversion). + // sub-sizes, format conversion). Keep the original HEIC in + // sourceFile so it can be sideloaded as the "original" after + // upload, preserving the user's original file. const vipsAvailable = isClientSideMediaSupported(); updates = { file: heicJpeg, - sourceFile: heicJpeg, additionalData: { ...item.additionalData, generate_sub_sizes: ! vipsAvailable, @@ -1139,10 +1140,30 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); + // HEIC/HEIF: the uploaded file is a JPEG conversion, but + // sourceFile still holds the original HEIC. Sideload it as + // "original" so WordPress preserves it in metadata. + const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); + if ( isHeicSource && attachment.id ) { + dispatch.addSideloadItem( { + file: item.sourceFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } + // Check if image needs rotation. // If exif_orientation is not 1, the image needs rotation. // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. - { + // HEIC rotation is handled by the browser's native decoder during + // canvas conversion, so skip vips rotation for HEIC sources. + if ( ! isHeicSource ) { const needsRotation = attachment.exif_orientation && attachment.exif_orientation !== 1 && @@ -1193,7 +1214,9 @@ export function generateThumbnails( id: QueueItemId ) { const sizesToGenerate: string[] = attachment.missing_image_sizes as string[]; - const thumbnailSource = item.sourceFile; + // For HEIC, sourceFile is the original HEIC which vips cannot + // process. Use the uploaded JPEG (item.file) instead. + const thumbnailSource = isHeicSource ? item.file : item.sourceFile; const file = attachment.filename ? renameFile( thumbnailSource, attachment.filename ) : thumbnailSource; diff --git a/packages/upload-media/src/store/test/private-actions.js b/packages/upload-media/src/store/test/private-actions.js index 5da1dace106aea..112779c6f0a5b9 100644 --- a/packages/upload-media/src/store/test/private-actions.js +++ b/packages/upload-media/src/store/test/private-actions.js @@ -6,19 +6,46 @@ import { createBlobURL, revokeBlobURL } from '@wordpress/blob'; /** * Internal dependencies */ -import { getTranscodeImageOperation, finalizeItem } from '../private-actions'; +import { + getTranscodeImageOperation, + finalizeItem, + prepareItem, + generateThumbnails, +} from '../private-actions'; import { OperationType } from '../types'; import { vipsHasTransparency } from '../utils'; +import { canvasConvertToJpeg } from '../../canvas-utils'; +import { isClientSideMediaSupported } from '../../feature-detection'; // Mock @wordpress/blob jest.mock( '@wordpress/blob', () => ( { createBlobURL: jest.fn( () => 'blob:mock-url' ), revokeBlobURL: jest.fn(), + isBlobURL: jest.fn( () => false ), } ) ); // Mock vips utilities jest.mock( '../utils', () => ( { vipsHasTransparency: jest.fn(), + vipsResizeImage: jest.fn(), + vipsRotateImage: jest.fn(), + vipsConvertImageFormat: jest.fn(), + terminateVipsWorker: jest.fn(), +} ) ); + +// Mock canvas-utils +jest.mock( '../../canvas-utils', () => ( { + canvasConvertToJpeg: jest.fn(), +} ) ); + +// Mock feature-detection +jest.mock( '../../feature-detection', () => ( { + isClientSideMediaSupported: jest.fn( () => true ), +} ) ); + +// Mock uuid +jest.mock( 'uuid', () => ( { + v4: jest.fn( () => 'mock-uuid' ), } ) ); describe( 'private actions', () => { @@ -346,4 +373,229 @@ describe( 'private actions', () => { expect( finishOperation ).not.toHaveBeenCalled(); } ); } ); + + describe( 'prepareItem - HEIC handling', () => { + const heicFile = new File( [ 'heic-data' ], 'photo.heic', { + type: 'image/heic', + } ); + + const jpegFile = new File( [ 'jpeg-data' ], 'photo.jpeg', { + type: 'image/jpeg', + } ); + + beforeEach( () => { + jest.clearAllMocks(); + canvasConvertToJpeg.mockResolvedValue( jpegFile ); + isClientSideMediaSupported.mockReturnValue( true ); + } ); + + it( 'should preserve original HEIC in sourceFile and set file to JPEG', async () => { + const finishOperation = jest.fn(); + const dispatch = Object.assign( jest.fn(), { + finishOperation, + cancelItem: jest.fn(), + } ); + + const select = { + getItem: () => ( { + id: 'test-id', + file: heicFile, + sourceFile: heicFile, + additionalData: { + generate_sub_sizes: false, + convert_format: false, + }, + } ), + getSettings: () => ( { + imageQuality: 0.82, + } ), + }; + + const thunk = prepareItem( 'test-id' ); + await thunk( { select, dispatch } ); + + // finishOperation should be called with file=JPEG but NOT sourceFile + expect( finishOperation ).toHaveBeenCalledWith( + 'test-id', + expect.objectContaining( { + file: jpegFile, + additionalData: expect.objectContaining( { + convert_format: true, + } ), + } ) + ); + + // sourceFile should NOT be in the updates (preserving original HEIC) + const updates = finishOperation.mock.calls[ 0 ][ 1 ]; + expect( updates ).not.toHaveProperty( 'sourceFile' ); + } ); + + it( 'should cancel item when HEIC conversion fails', async () => { + canvasConvertToJpeg.mockRejectedValue( + new Error( 'Decode failed' ) + ); + + const cancelItem = jest.fn(); + const dispatch = Object.assign( jest.fn(), { + finishOperation: jest.fn(), + cancelItem, + } ); + + const select = { + getItem: () => ( { + id: 'test-id', + file: heicFile, + sourceFile: heicFile, + additionalData: { + generate_sub_sizes: false, + convert_format: false, + }, + } ), + getSettings: () => ( {} ), + }; + + const thunk = prepareItem( 'test-id' ); + await thunk( { select, dispatch } ); + + expect( cancelItem ).toHaveBeenCalledWith( + 'test-id', + expect.objectContaining( { + code: 'HEIC_DECODE_ERROR', + } ) + ); + } ); + } ); + + describe( 'generateThumbnails - HEIC handling', () => { + const heicFile = new File( [ 'heic-data' ], 'photo.heic', { + type: 'image/heic', + } ); + + const jpegFile = new File( [ 'jpeg-data' ], 'photo.jpeg', { + type: 'image/jpeg', + } ); + + beforeEach( () => { + jest.clearAllMocks(); + } ); + + it( 'should sideload original HEIC as "original" when source is HEIC', async () => { + const addSideloadItem = jest.fn(); + const finishOperation = jest.fn(); + const dispatch = Object.assign( jest.fn(), { + addSideloadItem, + finishOperation, + } ); + + const select = { + getItem: () => ( { + id: 'test-id', + file: jpegFile, + sourceFile: heicFile, + attachment: { + id: 42, + missing_image_sizes: [], + }, + abortController: new AbortController(), + } ), + getSettings: () => ( {} ), + }; + + const thunk = generateThumbnails( 'test-id' ); + await thunk( { select, dispatch } ); + + // Should sideload the HEIC as "original" + expect( addSideloadItem ).toHaveBeenCalledWith( + expect.objectContaining( { + file: heicFile, + additionalData: expect.objectContaining( { + post: 42, + image_size: 'original', + } ), + } ) + ); + } ); + + it( 'should not sideload HEIC as "original" for non-HEIC files', async () => { + const addSideloadItem = jest.fn(); + const finishOperation = jest.fn(); + const dispatch = Object.assign( jest.fn(), { + addSideloadItem, + finishOperation, + } ); + + const select = { + getItem: () => ( { + id: 'test-id', + file: jpegFile, + sourceFile: jpegFile, + attachment: { + id: 42, + missing_image_sizes: [], + }, + abortController: new AbortController(), + } ), + getSettings: () => ( {} ), + }; + + const thunk = generateThumbnails( 'test-id' ); + await thunk( { select, dispatch } ); + + // Should NOT sideload as "original" for non-HEIC + const originalSideloads = addSideloadItem.mock.calls.filter( + ( call ) => call[ 0 ].additionalData.image_size === 'original' + ); + expect( originalSideloads ).toHaveLength( 0 ); + } ); + + it( 'should use JPEG file for thumbnail generation when source is HEIC', async () => { + // Mock createImageBitmap for the scaled check + const closeMock = jest.fn(); + global.createImageBitmap = jest.fn().mockResolvedValue( { + width: 100, + height: 100, + close: closeMock, + } ); + + const addSideloadItem = jest.fn(); + const finishOperation = jest.fn(); + const dispatch = Object.assign( jest.fn(), { + addSideloadItem, + finishOperation, + } ); + + const select = { + getItem: () => ( { + id: 'test-id', + file: jpegFile, + sourceFile: heicFile, + attachment: { + id: 42, + filename: 'photo.jpeg', + missing_image_sizes: [ 'thumbnail' ], + }, + abortController: new AbortController(), + } ), + getSettings: () => ( { + allImageSizes: { + thumbnail: { width: 150, height: 150 }, + }, + bigImageSizeThreshold: 2560, + } ), + }; + + const thunk = generateThumbnails( 'test-id' ); + await thunk( { select, dispatch } ); + + // Find the thumbnail sideload (not the "original" sideload) + const thumbnailSideloads = addSideloadItem.mock.calls.filter( + ( call ) => call[ 0 ].additionalData.image_size === 'thumbnail' + ); + expect( thumbnailSideloads ).toHaveLength( 1 ); + + // The file used for thumbnails should be JPEG-based (not HEIC) + const thumbnailFile = thumbnailSideloads[ 0 ][ 0 ].file; + expect( thumbnailFile.type ).toBe( 'image/jpeg' ); + } ); + } ); } ); From 657ff0dd14a2bc47252534c304520e6f1506f56a Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Sun, 12 Apr 2026 20:11:44 -0600 Subject: [PATCH 18/24] Revert "Preserve original HEIC file when uploading" This reverts commit 2f41440eb1acf04f3ca1bee2d7e5f2f8b8e7b069. --- .../upload-media/src/store/private-actions.ts | 31 +-- .../src/store/test/private-actions.js | 254 +----------------- 2 files changed, 5 insertions(+), 280 deletions(-) diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 60dcae2fb71271..d09d6abcbbaf6d 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -816,12 +816,11 @@ export function prepareItem( id: QueueItemId ) { if ( isHeic && heicJpeg ) { // HEIC was converted to JPEG client-side. Upload the JPEG // and let the server handle it normally (threshold scaling, - // sub-sizes, format conversion). Keep the original HEIC in - // sourceFile so it can be sideloaded as the "original" after - // upload, preserving the user's original file. + // sub-sizes, format conversion). const vipsAvailable = isClientSideMediaSupported(); updates = { file: heicJpeg, + sourceFile: heicJpeg, additionalData: { ...item.additionalData, generate_sub_sizes: ! vipsAvailable, @@ -1140,30 +1139,10 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); - // HEIC/HEIF: the uploaded file is a JPEG conversion, but - // sourceFile still holds the original HEIC. Sideload it as - // "original" so WordPress preserves it in metadata. - const isHeicSource = HEIC_MIME_TYPES.includes( item.sourceFile.type ); - if ( isHeicSource && attachment.id ) { - dispatch.addSideloadItem( { - file: item.sourceFile, - batchId: uuidv4(), - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'original', - convert_format: false, - }, - operations: [ OperationType.Upload ], - } ); - } - // Check if image needs rotation. // If exif_orientation is not 1, the image needs rotation. // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. - // HEIC rotation is handled by the browser's native decoder during - // canvas conversion, so skip vips rotation for HEIC sources. - if ( ! isHeicSource ) { + { const needsRotation = attachment.exif_orientation && attachment.exif_orientation !== 1 && @@ -1214,9 +1193,7 @@ export function generateThumbnails( id: QueueItemId ) { const sizesToGenerate: string[] = attachment.missing_image_sizes as string[]; - // For HEIC, sourceFile is the original HEIC which vips cannot - // process. Use the uploaded JPEG (item.file) instead. - const thumbnailSource = isHeicSource ? item.file : item.sourceFile; + const thumbnailSource = item.sourceFile; const file = attachment.filename ? renameFile( thumbnailSource, attachment.filename ) : thumbnailSource; diff --git a/packages/upload-media/src/store/test/private-actions.js b/packages/upload-media/src/store/test/private-actions.js index 112779c6f0a5b9..5da1dace106aea 100644 --- a/packages/upload-media/src/store/test/private-actions.js +++ b/packages/upload-media/src/store/test/private-actions.js @@ -6,46 +6,19 @@ import { createBlobURL, revokeBlobURL } from '@wordpress/blob'; /** * Internal dependencies */ -import { - getTranscodeImageOperation, - finalizeItem, - prepareItem, - generateThumbnails, -} from '../private-actions'; +import { getTranscodeImageOperation, finalizeItem } from '../private-actions'; import { OperationType } from '../types'; import { vipsHasTransparency } from '../utils'; -import { canvasConvertToJpeg } from '../../canvas-utils'; -import { isClientSideMediaSupported } from '../../feature-detection'; // Mock @wordpress/blob jest.mock( '@wordpress/blob', () => ( { createBlobURL: jest.fn( () => 'blob:mock-url' ), revokeBlobURL: jest.fn(), - isBlobURL: jest.fn( () => false ), } ) ); // Mock vips utilities jest.mock( '../utils', () => ( { vipsHasTransparency: jest.fn(), - vipsResizeImage: jest.fn(), - vipsRotateImage: jest.fn(), - vipsConvertImageFormat: jest.fn(), - terminateVipsWorker: jest.fn(), -} ) ); - -// Mock canvas-utils -jest.mock( '../../canvas-utils', () => ( { - canvasConvertToJpeg: jest.fn(), -} ) ); - -// Mock feature-detection -jest.mock( '../../feature-detection', () => ( { - isClientSideMediaSupported: jest.fn( () => true ), -} ) ); - -// Mock uuid -jest.mock( 'uuid', () => ( { - v4: jest.fn( () => 'mock-uuid' ), } ) ); describe( 'private actions', () => { @@ -373,229 +346,4 @@ describe( 'private actions', () => { expect( finishOperation ).not.toHaveBeenCalled(); } ); } ); - - describe( 'prepareItem - HEIC handling', () => { - const heicFile = new File( [ 'heic-data' ], 'photo.heic', { - type: 'image/heic', - } ); - - const jpegFile = new File( [ 'jpeg-data' ], 'photo.jpeg', { - type: 'image/jpeg', - } ); - - beforeEach( () => { - jest.clearAllMocks(); - canvasConvertToJpeg.mockResolvedValue( jpegFile ); - isClientSideMediaSupported.mockReturnValue( true ); - } ); - - it( 'should preserve original HEIC in sourceFile and set file to JPEG', async () => { - const finishOperation = jest.fn(); - const dispatch = Object.assign( jest.fn(), { - finishOperation, - cancelItem: jest.fn(), - } ); - - const select = { - getItem: () => ( { - id: 'test-id', - file: heicFile, - sourceFile: heicFile, - additionalData: { - generate_sub_sizes: false, - convert_format: false, - }, - } ), - getSettings: () => ( { - imageQuality: 0.82, - } ), - }; - - const thunk = prepareItem( 'test-id' ); - await thunk( { select, dispatch } ); - - // finishOperation should be called with file=JPEG but NOT sourceFile - expect( finishOperation ).toHaveBeenCalledWith( - 'test-id', - expect.objectContaining( { - file: jpegFile, - additionalData: expect.objectContaining( { - convert_format: true, - } ), - } ) - ); - - // sourceFile should NOT be in the updates (preserving original HEIC) - const updates = finishOperation.mock.calls[ 0 ][ 1 ]; - expect( updates ).not.toHaveProperty( 'sourceFile' ); - } ); - - it( 'should cancel item when HEIC conversion fails', async () => { - canvasConvertToJpeg.mockRejectedValue( - new Error( 'Decode failed' ) - ); - - const cancelItem = jest.fn(); - const dispatch = Object.assign( jest.fn(), { - finishOperation: jest.fn(), - cancelItem, - } ); - - const select = { - getItem: () => ( { - id: 'test-id', - file: heicFile, - sourceFile: heicFile, - additionalData: { - generate_sub_sizes: false, - convert_format: false, - }, - } ), - getSettings: () => ( {} ), - }; - - const thunk = prepareItem( 'test-id' ); - await thunk( { select, dispatch } ); - - expect( cancelItem ).toHaveBeenCalledWith( - 'test-id', - expect.objectContaining( { - code: 'HEIC_DECODE_ERROR', - } ) - ); - } ); - } ); - - describe( 'generateThumbnails - HEIC handling', () => { - const heicFile = new File( [ 'heic-data' ], 'photo.heic', { - type: 'image/heic', - } ); - - const jpegFile = new File( [ 'jpeg-data' ], 'photo.jpeg', { - type: 'image/jpeg', - } ); - - beforeEach( () => { - jest.clearAllMocks(); - } ); - - it( 'should sideload original HEIC as "original" when source is HEIC', async () => { - const addSideloadItem = jest.fn(); - const finishOperation = jest.fn(); - const dispatch = Object.assign( jest.fn(), { - addSideloadItem, - finishOperation, - } ); - - const select = { - getItem: () => ( { - id: 'test-id', - file: jpegFile, - sourceFile: heicFile, - attachment: { - id: 42, - missing_image_sizes: [], - }, - abortController: new AbortController(), - } ), - getSettings: () => ( {} ), - }; - - const thunk = generateThumbnails( 'test-id' ); - await thunk( { select, dispatch } ); - - // Should sideload the HEIC as "original" - expect( addSideloadItem ).toHaveBeenCalledWith( - expect.objectContaining( { - file: heicFile, - additionalData: expect.objectContaining( { - post: 42, - image_size: 'original', - } ), - } ) - ); - } ); - - it( 'should not sideload HEIC as "original" for non-HEIC files', async () => { - const addSideloadItem = jest.fn(); - const finishOperation = jest.fn(); - const dispatch = Object.assign( jest.fn(), { - addSideloadItem, - finishOperation, - } ); - - const select = { - getItem: () => ( { - id: 'test-id', - file: jpegFile, - sourceFile: jpegFile, - attachment: { - id: 42, - missing_image_sizes: [], - }, - abortController: new AbortController(), - } ), - getSettings: () => ( {} ), - }; - - const thunk = generateThumbnails( 'test-id' ); - await thunk( { select, dispatch } ); - - // Should NOT sideload as "original" for non-HEIC - const originalSideloads = addSideloadItem.mock.calls.filter( - ( call ) => call[ 0 ].additionalData.image_size === 'original' - ); - expect( originalSideloads ).toHaveLength( 0 ); - } ); - - it( 'should use JPEG file for thumbnail generation when source is HEIC', async () => { - // Mock createImageBitmap for the scaled check - const closeMock = jest.fn(); - global.createImageBitmap = jest.fn().mockResolvedValue( { - width: 100, - height: 100, - close: closeMock, - } ); - - const addSideloadItem = jest.fn(); - const finishOperation = jest.fn(); - const dispatch = Object.assign( jest.fn(), { - addSideloadItem, - finishOperation, - } ); - - const select = { - getItem: () => ( { - id: 'test-id', - file: jpegFile, - sourceFile: heicFile, - attachment: { - id: 42, - filename: 'photo.jpeg', - missing_image_sizes: [ 'thumbnail' ], - }, - abortController: new AbortController(), - } ), - getSettings: () => ( { - allImageSizes: { - thumbnail: { width: 150, height: 150 }, - }, - bigImageSizeThreshold: 2560, - } ), - }; - - const thunk = generateThumbnails( 'test-id' ); - await thunk( { select, dispatch } ); - - // Find the thumbnail sideload (not the "original" sideload) - const thumbnailSideloads = addSideloadItem.mock.calls.filter( - ( call ) => call[ 0 ].additionalData.image_size === 'thumbnail' - ); - expect( thumbnailSideloads ).toHaveLength( 1 ); - - // The file used for thumbnails should be JPEG-based (not HEIC) - const thumbnailFile = thumbnailSideloads[ 0 ][ 0 ].file; - expect( thumbnailFile.type ).toBe( 'image/jpeg' ); - } ); - } ); } ); From 1549a7aebd8a4c4f0ebb39d09656a3d763568e6e Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Sun, 12 Apr 2026 20:14:50 -0600 Subject: [PATCH 19/24] Preserve original HEIC via dedicated sideload field Keep the converted JPEG in both file and sourceFile so all downstream paths (vips, thumbnails, retries) see an editor-supported image and never leak HEIC into the main /wp/v2/media create endpoint. Store the original HEIC on item.originalHeicFile instead. In generateThumbnails, dispatch a single addSideloadItem with parentId set, which guarantees processItem routes it to sideloadItem and the /wp/v2/media/{id}/sideload endpoint. The server accepts HEIC there because the mime type is registered via upload_mimes and the sideload permission check does not apply wp_prevent_unsupported_mime_type_uploads. --- .../upload-media/src/store/private-actions.ts | 25 ++++++++++++++++++- packages/upload-media/src/store/types.ts | 4 +++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index d09d6abcbbaf6d..85d132f0a9228b 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -816,11 +816,15 @@ export function prepareItem( id: QueueItemId ) { if ( isHeic && heicJpeg ) { // HEIC was converted to JPEG client-side. Upload the JPEG // and let the server handle it normally (threshold scaling, - // sub-sizes, format conversion). + // sub-sizes, format conversion). Keep the original HEIC in + // a separate field so it can be sideloaded as the "original" + // after upload, preserving the user's file without leaking it + // into paths that expect an editor-supported image. const vipsAvailable = isClientSideMediaSupported(); updates = { file: heicJpeg, sourceFile: heicJpeg, + originalHeicFile: item.file, additionalData: { ...item.additionalData, generate_sub_sizes: ! vipsAvailable, @@ -1139,6 +1143,25 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); + // HEIC/HEIF: preserve the original file by sideloading it as the + // attachment's "original". The uploaded file is a JPEG conversion; + // the HEIC was kept on item.originalHeicFile for this purpose. + // parentId guarantees processItem routes this to the sideload + // endpoint, never the main create endpoint. + if ( item.originalHeicFile && attachment.id ) { + dispatch.addSideloadItem( { + file: item.originalHeicFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } + // Check if image needs rotation. // If exif_orientation is not 1, the image needs rotation. // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. diff --git a/packages/upload-media/src/store/types.ts b/packages/upload-media/src/store/types.ts index da9c1d11661f8c..6a170c9d7c114e 100644 --- a/packages/upload-media/src/store/types.ts +++ b/packages/upload-media/src/store/types.ts @@ -8,6 +8,10 @@ export interface QueueItem { id: QueueItemId; sourceFile: File; file: File; + // Original HEIC/HEIF file, kept separately so it can be sideloaded + // as the attachment's "original_image" after the converted JPEG is + // uploaded. Not set for non-HEIC items. + originalHeicFile?: File; poster?: File; attachment?: Partial< Attachment >; status: ItemStatus; From c3a664788c15dbef1e7ac8773331e2169f8858cf Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Sun, 12 Apr 2026 20:27:58 -0600 Subject: [PATCH 20/24] Remove orphaned JPEG when HEIC replaces original_image When a HEIC sideload overwrites an existing original_image value (set by the scaled-sideload flow to point at the un-scaled JPEG), delete that JPEG file first. wp_delete_attachment_files() only consults the current value of original_image, so without this the pre-scaled JPEG would remain on disk forever when the attachment is deleted. --- ...class-gutenberg-rest-attachments-controller.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 6cd928b6965db1..4aefe0e5de682c 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -490,6 +490,20 @@ public function sideload_item( WP_REST_Request $request ) { } if ( 'original' === $image_size ) { + // If an original_image is already recorded (e.g. the un-scaled + // JPEG saved by the scaled-sideload flow), delete that file + // before overwriting the reference. Otherwise it becomes an + // orphan on disk that is never cleaned up on attachment + // deletion, since wp_delete_attachment_files() only consults + // the current value of original_image. + if ( ! empty( $metadata['original_image'] ) && wp_basename( $path ) !== $metadata['original_image'] ) { + $attached_file = get_attached_file( $attachment_id, true ); + $previous_origin = path_join( dirname( $attached_file ), $metadata['original_image'] ); + if ( file_exists( $previous_origin ) ) { + wp_delete_file( $previous_origin ); + } + } + $metadata['original_image'] = wp_basename( $path ); } elseif ( 'scaled' === $image_size ) { // The current attached file is the original; record it as original_image. From 4cea4e59cd225ed4ab2e88ce3db9ae5dd98ed77d Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Sun, 12 Apr 2026 20:35:38 -0600 Subject: [PATCH 21/24] Preserve HEIC under dedicated metadata key The HEIC sideload previously used image_size 'original', which set $metadata['original_image']. The scaled-JPEG sideload runs after and overwrites that same key with the un-scaled JPEG basename, losing the HEIC reference. The HEIC then lingered on disk after attachment delete, because wp_delete_attachment_files() only consults original_image. Use a dedicated 'original-heic' image_size that writes to a separate 'original_image_heic' meta key, so the two sideloads no longer collide. A delete_attachment hook removes the companion HEIC file when the attachment is deleted. --- ...-gutenberg-rest-attachments-controller.php | 10 ++++++ lib/media/load.php | 33 +++++++++++++++++++ .../upload-media/src/store/private-actions.ts | 13 ++++---- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 4aefe0e5de682c..bf67cec09752c9 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -27,6 +27,10 @@ public function register_routes(): void { // Special case to set 'original_image' in attachment metadata. $valid_image_sizes[] = 'original'; + // HEIC/HEIF companion original preserved alongside the JPEG derivative. + // Stored under its own meta key so it never collides with 'original' + // (which the scaled-sideload flow also writes to). + $valid_image_sizes[] = 'original-heic'; // Client-side big image threshold: sideload the scaled version. $valid_image_sizes[] = 'scaled'; // Used for PDF thumbnails. @@ -505,6 +509,12 @@ public function sideload_item( WP_REST_Request $request ) { } $metadata['original_image'] = wp_basename( $path ); + } elseif ( 'original-heic' === $image_size ) { + // HEIC companion original: stored under its own meta key so the + // scaled-sideload flow (which writes 'original_image') cannot + // clobber it. Cleanup on attachment delete is handled by a + // delete_attachment hook that reads this key. + $metadata['original_image_heic'] = wp_basename( $path ); } elseif ( 'scaled' === $image_size ) { // The current attached file is the original; record it as original_image. $current_file = get_attached_file( $attachment_id, true ); diff --git a/lib/media/load.php b/lib/media/load.php index 180190d93d9452..894b1d6c88cefb 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -238,6 +238,39 @@ function gutenberg_set_heic_upload_support_flag() { } add_action( 'admin_init', 'gutenberg_set_heic_upload_support_flag' ); +/** + * Deletes the HEIC companion file when its attachment is deleted. + * + * The HEIC is sideloaded alongside a JPEG derivative and recorded in + * $metadata['original_image_heic']. WordPress core's + * wp_delete_attachment_files() only knows about 'original_image', so + * without this hook the HEIC would linger on disk after the attachment + * is deleted. + * + * @param int $post_id Attachment ID being deleted. + */ +function gutenberg_delete_heic_companion_file( int $post_id ): void { + $metadata = wp_get_attachment_metadata( $post_id, true ); + + if ( empty( $metadata['original_image_heic'] ) ) { + return; + } + + $attached_file = get_attached_file( $post_id, true ); + + if ( ! $attached_file ) { + return; + } + + $heic_path = path_join( dirname( $attached_file ), $metadata['original_image_heic'] ); + + if ( file_exists( $heic_path ) ) { + wp_delete_file( $heic_path ); + } +} + +add_action( 'delete_attachment', 'gutenberg_delete_heic_companion_file' ); + // ── Tier 2: Full client-side processing (VIPS/WASM) ───────────────── // Everything below requires cross-origin isolation (Document-Isolation-Policy) // and SharedArrayBuffer support, which is only available in Chromium 137+. diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 85d132f0a9228b..71b1616990efbb 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -1143,11 +1143,12 @@ export function generateThumbnails( id: QueueItemId ) { const attachment = item.attachment; const settings = select.getSettings(); - // HEIC/HEIF: preserve the original file by sideloading it as the - // attachment's "original". The uploaded file is a JPEG conversion; - // the HEIC was kept on item.originalHeicFile for this purpose. - // parentId guarantees processItem routes this to the sideload - // endpoint, never the main create endpoint. + // HEIC/HEIF: preserve the original file under a dedicated metadata + // key so it never collides with `original_image`, which the scaled + // sideload flow owns. The HEIC was kept on item.originalHeicFile; + // the uploaded file is a JPEG conversion. parentId guarantees + // processItem routes this to the sideload endpoint, never the main + // create endpoint. if ( item.originalHeicFile && attachment.id ) { dispatch.addSideloadItem( { file: item.originalHeicFile, @@ -1155,7 +1156,7 @@ export function generateThumbnails( id: QueueItemId ) { parentId: item.id, additionalData: { post: attachment.id, - image_size: 'original', + image_size: 'original-heic', convert_format: false, }, operations: [ OperationType.Upload ], From 6671942fb54fb789e8dd646129b1a020f94a4fad Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Sun, 12 Apr 2026 20:38:01 -0600 Subject: [PATCH 22/24] Remove orphan-JPEG cleanup from 'original' sideload branch This cleanup was added to handle HEIC overwriting an existing original_image, but Option A (dedicated original_image_heic key) means HEIC no longer touches the 'original' branch. The remaining non-HEIC callers ('original' for rotated images) only fire once per upload, so there is nothing to clean up. --- ...class-gutenberg-rest-attachments-controller.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index bf67cec09752c9..754da649610551 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -494,20 +494,6 @@ public function sideload_item( WP_REST_Request $request ) { } if ( 'original' === $image_size ) { - // If an original_image is already recorded (e.g. the un-scaled - // JPEG saved by the scaled-sideload flow), delete that file - // before overwriting the reference. Otherwise it becomes an - // orphan on disk that is never cleaned up on attachment - // deletion, since wp_delete_attachment_files() only consults - // the current value of original_image. - if ( ! empty( $metadata['original_image'] ) && wp_basename( $path ) !== $metadata['original_image'] ) { - $attached_file = get_attached_file( $attachment_id, true ); - $previous_origin = path_join( dirname( $attached_file ), $metadata['original_image'] ); - if ( file_exists( $previous_origin ) ) { - wp_delete_file( $previous_origin ); - } - } - $metadata['original_image'] = wp_basename( $path ); } elseif ( 'original-heic' === $image_size ) { // HEIC companion original: stored under its own meta key so the From 6ff5a4cd4a577166fdfdcf7182d4af368aaa8720 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 13 Apr 2026 11:06:51 -0600 Subject: [PATCH 23/24] Rename HEIC companion meta key to 'original' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per PR discussion, align the meta key with the convention used in media-experiments (swissspidy/media-experiments) so a future switch between the two is seamless. 'original_image' continues to point at the web-viewable JPEG derivative — existing consumers rely on that. --- .../class-gutenberg-rest-attachments-controller.php | 8 +++++--- lib/media/load.php | 11 +++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 754da649610551..7a0c53038b460c 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -498,9 +498,11 @@ public function sideload_item( WP_REST_Request $request ) { } elseif ( 'original-heic' === $image_size ) { // HEIC companion original: stored under its own meta key so the // scaled-sideload flow (which writes 'original_image') cannot - // clobber it. Cleanup on attachment delete is handled by a - // delete_attachment hook that reads this key. - $metadata['original_image_heic'] = wp_basename( $path ); + // clobber it. 'original_image' must keep pointing at the + // web-viewable JPEG derivative so existing consumers still work. + // Cleanup on attachment delete is handled by a delete_attachment + // hook that reads this key. + $metadata['original'] = wp_basename( $path ); } elseif ( 'scaled' === $image_size ) { // The current attached file is the original; record it as original_image. $current_file = get_attached_file( $attachment_id, true ); diff --git a/lib/media/load.php b/lib/media/load.php index 894b1d6c88cefb..7f285f777661c8 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -242,17 +242,16 @@ function gutenberg_set_heic_upload_support_flag() { * Deletes the HEIC companion file when its attachment is deleted. * * The HEIC is sideloaded alongside a JPEG derivative and recorded in - * $metadata['original_image_heic']. WordPress core's - * wp_delete_attachment_files() only knows about 'original_image', so - * without this hook the HEIC would linger on disk after the attachment - * is deleted. + * $metadata['original']. WordPress core's wp_delete_attachment_files() + * only knows about 'original_image', so without this hook the HEIC + * would linger on disk after the attachment is deleted. * * @param int $post_id Attachment ID being deleted. */ function gutenberg_delete_heic_companion_file( int $post_id ): void { $metadata = wp_get_attachment_metadata( $post_id, true ); - if ( empty( $metadata['original_image_heic'] ) ) { + if ( empty( $metadata['original'] ) ) { return; } @@ -262,7 +261,7 @@ function gutenberg_delete_heic_companion_file( int $post_id ): void { return; } - $heic_path = path_join( dirname( $attached_file ), $metadata['original_image_heic'] ); + $heic_path = path_join( dirname( $attached_file ), $metadata['original'] ); if ( file_exists( $heic_path ) ) { wp_delete_file( $heic_path ); From 84cdd06b02e6c91000a4b8bd2c7f6a515f1abfab Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 20 Apr 2026 10:01:03 -0700 Subject: [PATCH 24/24] Reconcile HEIC changes with trunk's concurrent sideload API Adopt the sub_size_data accumulation pattern introduced in #75888 on trunk and fold the HEIC companion handling into it: PHP (class-gutenberg-rest-attachments-controller.php) - finalize route: add sub_sizes schema argument - finalize_item(): apply accumulated sub_sizes to $metadata, including a new 'original-heic' branch that writes to $metadata['original'] - sideload_item(): return a $sub_size_data shape; the 'original-heic' case returns file=wp_basename($path) for finalize to pick up TS (upload-media/src/store/private-actions.ts) - Remove shouldPauseForSideload and resumeItemByPostId; trunk gates concurrency via getActiveUploadCount / maxConcurrentUploads - sideloadItem(): replace onFileChange with onSuccess(subSize) that dispatches AccumulateSubSize against item.parentId - generateThumbnails(): drop onChange callbacks from thumbnail and scaled addSideloadItem dispatches; sideloads no longer return a full attachment - finalizeItem(): pass item.subSizes to mediaFinalize --- ...-gutenberg-rest-attachments-controller.php | 146 ++++++++++++------ .../upload-media/src/store/private-actions.ts | 96 ++---------- 2 files changed, 111 insertions(+), 131 deletions(-) diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 7a0c53038b460c..03ec72efd4fd23 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -77,10 +77,46 @@ public function register_routes(): void { 'callback' => array( $this, 'finalize_item' ), 'permission_callback' => array( $this, 'edit_media_item_permissions_check' ), 'args' => array( - 'id' => array( + 'id' => array( 'description' => __( 'Unique identifier for the attachment.', 'gutenberg' ), 'type' => 'integer', ), + 'sub_sizes' => array( + 'description' => __( 'Array of sub-size metadata collected from sideload responses.', 'gutenberg' ), + 'type' => 'array', + 'default' => array(), + 'items' => array( + 'type' => 'object', + 'properties' => array( + 'image_size' => array( + 'type' => 'string', + 'required' => true, + ), + 'width' => array( + 'type' => 'integer', + 'minimum' => 1, + ), + 'height' => array( + 'type' => 'integer', + 'minimum' => 1, + ), + 'file' => array( + 'type' => 'string', + ), + 'mime_type' => array( + 'type' => 'string', + 'pattern' => '^image/.*', + ), + 'filesize' => array( + 'type' => 'integer', + 'minimum' => 1, + ), + 'original_image' => array( + 'type' => 'string', + ), + ), + ), + ), ), ), 'allow_batch' => $this->allow_batch, @@ -311,6 +347,42 @@ public function finalize_item( WP_REST_Request $request ) { $metadata = array(); } + // Apply all sub-size metadata collected from sideload responses. + $sub_sizes = $request['sub_sizes'] ?? array(); + + foreach ( $sub_sizes as $sub_size ) { + $image_size = $sub_size['image_size']; + + if ( 'original' === $image_size ) { + $metadata['original_image'] = $sub_size['file']; + } elseif ( 'original-heic' === $image_size ) { + // HEIC companion original: stored under its own meta key so + // the scaled-sideload flow (which writes 'original_image') + // cannot clobber it. 'original_image' keeps pointing at the + // web-viewable JPEG derivative. Cleanup on attachment delete + // is handled by a delete_attachment hook that reads this key. + $metadata['original'] = $sub_size['file']; + } elseif ( 'scaled' === $image_size ) { + if ( ! empty( $sub_size['original_image'] ) ) { + $metadata['original_image'] = $sub_size['original_image']; + } + $metadata['width'] = $sub_size['width'] ?? 0; + $metadata['height'] = $sub_size['height'] ?? 0; + $metadata['filesize'] = $sub_size['filesize'] ?? 0; + $metadata['file'] = $sub_size['file'] ?? ''; + } else { + $metadata['sizes'] = $metadata['sizes'] ?? array(); + + $metadata['sizes'][ $image_size ] = array( + 'width' => $sub_size['width'] ?? 0, + 'height' => $sub_size['height'] ?? 0, + 'file' => $sub_size['file'] ?? '', + 'mime-type' => $sub_size['mime_type'] ?? '', + 'filesize' => $sub_size['filesize'] ?? 0, + ); + } + } + /** * Filters the attachment metadata after client-side processing. * @@ -487,67 +559,45 @@ public function sideload_item( WP_REST_Request $request ) { $image_size = $request['image_size']; - $metadata = wp_get_attachment_metadata( $attachment_id, true ); - - if ( ! $metadata ) { - $metadata = array(); - } + // Build sub-size data to return to the client. + // The client accumulates these and sends them all to the finalize endpoint. + $sub_size_data = array( + 'image_size' => $image_size, + ); if ( 'original' === $image_size ) { - $metadata['original_image'] = wp_basename( $path ); + $sub_size_data['file'] = wp_basename( $path ); } elseif ( 'original-heic' === $image_size ) { - // HEIC companion original: stored under its own meta key so the - // scaled-sideload flow (which writes 'original_image') cannot - // clobber it. 'original_image' must keep pointing at the - // web-viewable JPEG derivative so existing consumers still work. - // Cleanup on attachment delete is handled by a delete_attachment - // hook that reads this key. - $metadata['original'] = wp_basename( $path ); + // HEIC companion original. finalize_item() writes the filename to + // $metadata['original'] (separate from 'original_image', which the + // scaled-sideload flow owns). Cleanup on attachment delete is + // handled by a delete_attachment hook that reads this key. + $sub_size_data['file'] = wp_basename( $path ); } elseif ( 'scaled' === $image_size ) { - // The current attached file is the original; record it as original_image. - $current_file = get_attached_file( $attachment_id, true ); - $metadata['original_image'] = wp_basename( $current_file ); + // Record the current attached file as the original. + $current_file = get_attached_file( $attachment_id, true ); + $sub_size_data['original_image'] = wp_basename( $current_file ); // Update the attached file to point to the scaled version. + // This writes to _wp_attached_file meta, not _wp_attachment_metadata. update_attached_file( $attachment_id, $path ); $size = wp_getimagesize( $path ); - $metadata['width'] = $size ? $size[0] : 0; - $metadata['height'] = $size ? $size[1] : 0; - $metadata['filesize'] = wp_filesize( $path ); - $metadata['file'] = _wp_relative_upload_path( $path ); + $sub_size_data['width'] = $size ? $size[0] : 0; + $sub_size_data['height'] = $size ? $size[1] : 0; + $sub_size_data['filesize'] = wp_filesize( $path ); + $sub_size_data['file'] = _wp_relative_upload_path( $path ); } else { - $metadata['sizes'] = $metadata['sizes'] ?? array(); - $size = wp_getimagesize( $path ); - $metadata['sizes'][ $image_size ] = array( - 'width' => $size ? $size[0] : 0, - 'height' => $size ? $size[1] : 0, - 'file' => wp_basename( $path ), - 'mime-type' => $type, - 'filesize' => wp_filesize( $path ), - ); - } - - wp_update_attachment_metadata( $attachment_id, $metadata ); - - $response_request = new WP_REST_Request( - WP_REST_Server::READABLE, - rest_get_route_for_post( $attachment_id ) - ); - - $response_request['context'] = 'edit'; - - if ( isset( $request['_fields'] ) ) { - $response_request['_fields'] = $request['_fields']; + $sub_size_data['width'] = $size ? $size[0] : 0; + $sub_size_data['height'] = $size ? $size[1] : 0; + $sub_size_data['file'] = wp_basename( $path ); + $sub_size_data['mime_type'] = $type; + $sub_size_data['filesize'] = wp_filesize( $path ); } - $response = $this->prepare_item_for_response( get_post( $attachment_id ), $response_request ); - - $response->header( 'Location', rest_url( rest_get_route_for_post( $attachment_id ) ) ); - - return $response; + return rest_ensure_response( $sub_size_data ); } } diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index 71b1616990efbb..75e3c91731e90c 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -27,6 +27,7 @@ import { terminateVipsWorker, } from './utils'; import type { + AccumulateSubSizeAction, AddAction, AdditionalData, AddOperationsAction, @@ -45,12 +46,12 @@ import type { PauseQueueAction, QueueItem, QueueItemId, - ResumeItemAction, ResumeQueueAction, RevokeBlobUrlsAction, SideloadAdditionalData, Settings, State, + SubSizeData, UpdateProgressAction, UpdateSettingsAction, } from './types'; @@ -65,7 +66,6 @@ type ActionCreators = { addSideloadItem: typeof addSideloadItem; removeItem: typeof removeItem; pauseItem: typeof pauseItem; - resumeItemByPostId: typeof resumeItemByPostId; prepareItem: typeof prepareItem; processItem: typeof processItem; finishOperation: typeof finishOperation; @@ -96,32 +96,6 @@ type ThunkArgs = { registry: WPDataRegistry; }; -/** - * Determines if an upload should be paused to avoid race conditions. - * - * When sideloading thumbnails, we need to pause uploads if another - * upload to the same post is already in progress. - * - * @param item Queue item to check. - * @param operation Current operation type. - * @param select Store selectors. - * @return Whether the upload should be paused. - */ -function shouldPauseForSideload( - item: QueueItem, - operation: OperationType | undefined, - select: Selectors -): boolean { - if ( - operation !== OperationType.Upload || - ! item.parentId || - ! item.additionalData.post - ) { - return false; - } - return select.isUploadingToPost( item.additionalData.post as number ); -} - interface AddItemArgs { // It should always be a File, but some consumers might still pass Blobs only. file: File | Blob; @@ -308,16 +282,6 @@ export function processItem( id: QueueItemId ) { ? item.operations[ 0 ][ 1 ] : undefined; - // If we're sideloading a thumbnail, pause upload to avoid race conditions. - // It will be resumed after the previous upload finishes. - if ( shouldPauseForSideload( item, operation, select ) ) { - dispatch< PauseItemAction >( { - type: Type.PauseItem, - id, - } ); - return; - } - /* * If the next operation is an upload, check concurrency limit. * If at capacity, the item remains queued and will be processed @@ -525,28 +489,6 @@ export function pauseItem( id: QueueItemId ) { }; } -/** - * Resumes processing for a given post/attachment ID. - * - * This function looks up paused uploads by post ID and resumes them. - * It's typically called after a sideload completes to resume paused - * thumbnail uploads. - * - * @param postOrAttachmentId Post or attachment ID. - */ -export function resumeItemByPostId( postOrAttachmentId: number ) { - return async ( { select, dispatch }: ThunkArgs ) => { - const item = select.getPausedUploadForPost( postOrAttachmentId ); - if ( item ) { - dispatch< ResumeItemAction >( { - type: Type.ResumeItem, - id: item.id, - } ); - dispatch.processItem( item.id ); - } - }; -} - /** * Removes a specific item from the queue. * @@ -907,13 +849,19 @@ export function sideloadItem( id: QueueItemId ) { attachmentId: post as number, additionalData, signal: item.abortController?.signal, - onFileChange: ( [ attachment ] ) => { - dispatch.finishOperation( id, { attachment } ); - dispatch.resumeItemByPostId( post as number ); + onSuccess: ( subSize: SubSizeData ) => { + // Accumulate sub-size data on the parent item for finalize. + if ( item.parentId ) { + dispatch< AccumulateSubSizeAction >( { + type: Type.AccumulateSubSize, + id: item.parentId, + subSize, + } ); + } + dispatch.finishOperation( id, {} ); }, onError: ( error ) => { dispatch.cancelItem( id, error ); - dispatch.resumeItemByPostId( post as number ); }, } ); }; @@ -1271,18 +1219,6 @@ export function generateThumbnails( id: QueueItemId ) { dispatch.addSideloadItem( { file, - onChange: ( [ updatedAttachment ] ) => { - // If the sub-size is still being generated, there is no need - // to invoke the callback below. It would just override - // the main image in the editor with the sub-size. - if ( isBlobURL( updatedAttachment.url ) ) { - return; - } - - // This might be confusing, but the idea is to update the original - // image item in the editor with the new one with the added sub-size. - item.onChange?.( [ updatedAttachment ] ); - }, batchId, parentId: item.id, additionalData: { @@ -1340,12 +1276,6 @@ export function generateThumbnails( id: QueueItemId ) { dispatch.addSideloadItem( { file: sourceForScaled, - onChange: ( [ updatedAttachment ] ) => { - if ( isBlobURL( updatedAttachment.url ) ) { - return; - } - item.onChange?.( [ updatedAttachment ] ); - }, batchId, parentId: item.id, additionalData: { @@ -1386,7 +1316,7 @@ export function finalizeItem( id: QueueItemId ) { // Only finalize if we have an attachment ID and a mediaFinalize callback. if ( attachment?.id && mediaFinalize ) { try { - await mediaFinalize( attachment.id ); + await mediaFinalize( attachment.id, item.subSizes || [] ); } catch ( error ) { // Log but don't fail the upload if finalization fails. // eslint-disable-next-line no-console