diff --git a/backport-changelog/7.1/11323.md b/backport-changelog/7.1/11323.md new file mode 100644 index 00000000000000..fa30d7359b6685 --- /dev/null +++ b/backport-changelog/7.1/11323.md @@ -0,0 +1,3 @@ +https://github.com/WordPress/wordpress-develop/pull/11323 + +* https://github.com/WordPress/gutenberg/pull/76731 diff --git a/lib/media/class-gutenberg-rest-attachments-controller.php b/lib/media/class-gutenberg-rest-attachments-controller.php index 2b0576dc19b7fd..03ec72efd4fd23 100644 --- a/lib/media/class-gutenberg-rest-attachments-controller.php +++ b/lib/media/class-gutenberg-rest-attachments-controller.php @@ -27,6 +27,10 @@ public function register_routes(): void { // Special case to set 'original_image' in attachment metadata. $valid_image_sizes[] = 'original'; + // HEIC/HEIF companion original preserved alongside the JPEG derivative. + // Stored under its own meta key so it never collides with 'original' + // (which the scaled-sideload flow also writes to). + $valid_image_sizes[] = 'original-heic'; // Client-side big image threshold: sideload the scaled version. $valid_image_sizes[] = 'scaled'; // Used for PDF thumbnails. @@ -41,16 +45,21 @@ public function register_routes(): void { 'callback' => array( $this, 'sideload_item' ), 'permission_callback' => array( $this, 'sideload_item_permissions_check' ), 'args' => array( - 'id' => array( + 'id' => array( 'description' => __( 'Unique identifier for the attachment.', 'gutenberg' ), 'type' => 'integer', ), - 'image_size' => array( + 'image_size' => array( 'description' => __( 'Image size.', 'gutenberg' ), 'type' => 'string', 'enum' => $valid_image_sizes, 'required' => true, ), + 'generate_sub_sizes' => array( + 'description' => __( 'Whether to generate image sub sizes from the sideloaded file.', 'gutenberg' ), + 'type' => 'boolean', + 'default' => false, + ), ), ), 'allow_batch' => $this->allow_batch, @@ -126,13 +135,15 @@ public function register_routes(): void { * @return true|WP_Error True if the request has access to create items, WP_Error object otherwise. */ public function create_item_permissions_check( $request ) { - if ( false === $request['generate_sub_sizes'] ) { + $bypass_mime_check = false === $request['generate_sub_sizes']; + + if ( $bypass_mime_check ) { add_filter( 'wp_prevent_unsupported_mime_type_uploads', '__return_false' ); } $result = parent::create_item_permissions_check( $request ); - if ( false === $request['generate_sub_sizes'] ) { + if ( $bypass_mime_check ) { remove_filter( 'wp_prevent_unsupported_mime_type_uploads', '__return_false' ); } @@ -344,6 +355,13 @@ public function finalize_item( WP_REST_Request $request ) { if ( 'original' === $image_size ) { $metadata['original_image'] = $sub_size['file']; + } elseif ( 'original-heic' === $image_size ) { + // HEIC companion original: stored under its own meta key so + // the scaled-sideload flow (which writes 'original_image') + // cannot clobber it. 'original_image' keeps pointing at the + // web-viewable JPEG derivative. Cleanup on attachment delete + // is handled by a delete_attachment hook that reads this key. + $metadata['original'] = $sub_size['file']; } elseif ( 'scaled' === $image_size ) { if ( ! empty( $sub_size['original_image'] ) ) { $metadata['original_image'] = $sub_size['original_image']; @@ -549,6 +567,12 @@ public function sideload_item( WP_REST_Request $request ) { if ( 'original' === $image_size ) { $sub_size_data['file'] = wp_basename( $path ); + } elseif ( 'original-heic' === $image_size ) { + // HEIC companion original. finalize_item() writes the filename to + // $metadata['original'] (separate from 'original_image', which the + // scaled-sideload flow owns). Cleanup on attachment delete is + // handled by a delete_attachment hook that reads this key. + $sub_size_data['file'] = wp_basename( $path ); } elseif ( 'scaled' === $image_size ) { // Record the current attached file as the original. $current_file = get_attached_file( $attachment_id, true ); diff --git a/lib/media/load.php b/lib/media/load.php index c664d36285f83b..7f285f777661c8 100644 --- a/lib/media/load.php +++ b/lib/media/load.php @@ -2,6 +2,16 @@ /** * Adds media-related functionality for client-side media processing. * + * This file is structured in two tiers: + * + * 1. HEIC infrastructure — loaded whenever the feature filter is enabled. + * Browsers like Safari can decode HEIC via createImageBitmap() even + * without VIPS/SharedArrayBuffer, so HEIC MIME types, the custom REST + * controller, and REST field/index registrations are always needed. + * + * 2. Full VIPS/WASM processing — loaded only when the feature filter is + * enabled AND requires cross-origin isolation (DIP) at runtime. + * * @package gutenberg */ @@ -14,97 +24,26 @@ return; } -/** - * Sets a global JS variable to indicate that client-side media processing is enabled. - */ -function gutenberg_set_client_side_media_processing_flag() { - if ( ! gutenberg_is_client_side_media_processing_enabled() ) { - return; - } - wp_add_inline_script( 'wp-block-editor', 'window.__clientSideMediaProcessing = true', 'before' ); -} -add_action( 'admin_init', 'gutenberg_set_client_side_media_processing_flag' ); +// ── Tier 1: HEIC infrastructure (always loaded) ───────────────────── /** - * Returns a list of all available image sizes. + * Registers HEIC/HEIF as allowed upload MIME types. * - * @return array Existing image sizes. - */ -function gutenberg_get_all_image_sizes(): array { - $sizes = wp_get_registered_image_subsizes(); - - foreach ( $sizes as $name => &$size ) { - $size['height'] = (int) $size['height']; - $size['width'] = (int) $size['width']; - $size['name'] = $name; - } - unset( $size ); - - return $sizes; -} - -/** - * Returns the default output format mapping for the supported image formats. + * HEIC images can be decoded in the browser (via canvas/VideoDecoder). + * Registering these MIME types ensures the file picker's accept attribute + * includes them, preventing macOS from silently converting HEIC to JPEG + * on selection. * - * @return array Map of input formats to output formats. + * @param array $mimes Allowed MIME types (extension => type). + * @return array Modified MIME types. */ -function gutenberg_get_default_image_output_formats() { - $input_formats = array( - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - 'image/avif', - 'image/heic', - ); - - $output_formats = array(); - - foreach ( $input_formats as $mime_type ) { - /** This filter is documented in wp-includes/media.php */ - $output_formats = apply_filters( - 'image_editor_output_format', // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - $output_formats, - '', - $mime_type - ); - } - - return $output_formats; -} - -/** - * Filters the REST API root index data to add custom settings. - * - * @param WP_REST_Response $response Response data. - */ -function gutenberg_media_processing_filter_rest_index( WP_REST_Response $response ) { - /** This filter is documented in wp-admin/includes/images.php */ - $image_size_threshold = (int) apply_filters( 'big_image_size_threshold', 2560, array( 0, 0 ), '', 0 ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - - $default_image_output_formats = gutenberg_get_default_image_output_formats(); - - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $jpeg_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/jpeg' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $png_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/png' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ - $gif_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/gif' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound - - if ( current_user_can( 'upload_files' ) ) { - $response->data['image_sizes'] = gutenberg_get_all_image_sizes(); - $response->data['image_size_threshold'] = $image_size_threshold; - $response->data['image_output_formats'] = (object) $default_image_output_formats; - $response->data['jpeg_interlaced'] = $jpeg_interlaced; - $response->data['png_interlaced'] = $png_interlaced; - $response->data['gif_interlaced'] = $gif_interlaced; - } - - return $response; +function gutenberg_add_heic_upload_mimes( array $mimes ): array { + $mimes['heic'] = 'image/heic'; + $mimes['heif'] = 'image/heif'; + return $mimes; } -add_filter( 'rest_index', 'gutenberg_media_processing_filter_rest_index' ); - +add_filter( 'upload_mimes', 'gutenberg_add_heic_upload_mimes' ); /** * Overrides the REST controller for the attachment post type. @@ -125,7 +64,6 @@ function gutenberg_filter_attachment_post_type_args( array $args, string $post_t add_filter( 'register_post_type_args', 'gutenberg_filter_attachment_post_type_args', 10, 2 ); - /** * Registers additional REST fields for attachments. */ @@ -206,6 +144,147 @@ function gutenberg_rest_get_attachment_filesize( array $post ): ?int { return null; } +/** + * Returns a list of all available image sizes. + * + * @return array Existing image sizes. + */ +function gutenberg_get_all_image_sizes(): array { + $sizes = wp_get_registered_image_subsizes(); + + foreach ( $sizes as $name => &$size ) { + $size['height'] = (int) $size['height']; + $size['width'] = (int) $size['width']; + $size['name'] = $name; + } + unset( $size ); + + return $sizes; +} + +/** + * Returns the default output format mapping for the supported image formats. + * + * @return array Map of input formats to output formats. + */ +function gutenberg_get_default_image_output_formats() { + $input_formats = array( + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + 'image/avif', + 'image/heic', + 'image/heif', + ); + + $output_formats = array(); + + foreach ( $input_formats as $mime_type ) { + /** This filter is documented in wp-includes/media.php */ + $output_formats = apply_filters( + 'image_editor_output_format', // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + $output_formats, + '', + $mime_type + ); + } + + return $output_formats; +} + +/** + * Filters the REST API root index data to add custom settings. + * + * @param WP_REST_Response $response Response data. + */ +function gutenberg_media_processing_filter_rest_index( WP_REST_Response $response ) { + /** This filter is documented in wp-admin/includes/images.php */ + $image_size_threshold = (int) apply_filters( 'big_image_size_threshold', 2560, array( 0, 0 ), '', 0 ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + + $default_image_output_formats = gutenberg_get_default_image_output_formats(); + + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $jpeg_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/jpeg' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $png_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/png' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + /** This filter is documented in wp-includes/class-wp-image-editor-imagick.php */ + $gif_interlaced = (bool) apply_filters( 'image_save_progressive', false, 'image/gif' ); // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound + + if ( current_user_can( 'upload_files' ) ) { + $response->data['image_sizes'] = gutenberg_get_all_image_sizes(); + $response->data['image_size_threshold'] = $image_size_threshold; + $response->data['image_output_formats'] = (object) $default_image_output_formats; + $response->data['jpeg_interlaced'] = $jpeg_interlaced; + $response->data['png_interlaced'] = $png_interlaced; + $response->data['gif_interlaced'] = $gif_interlaced; + } + + return $response; +} + +add_filter( 'rest_index', 'gutenberg_media_processing_filter_rest_index' ); + +/** + * Sets a global JS variable to indicate that HEIC canvas-based upload support is available. + * + * This flag is set whenever the media processing feature is enabled, + * regardless of whether the browser supports full VIPS-based processing. + * Browsers like Safari can use createImageBitmap() to decode HEIC images + * and convert them to JPEG for server-side sub-size generation. + */ +function gutenberg_set_heic_upload_support_flag() { + wp_add_inline_script( 'wp-block-editor', 'window.__heicUploadSupport = true', 'before' ); +} +add_action( 'admin_init', 'gutenberg_set_heic_upload_support_flag' ); + +/** + * Deletes the HEIC companion file when its attachment is deleted. + * + * The HEIC is sideloaded alongside a JPEG derivative and recorded in + * $metadata['original']. WordPress core's wp_delete_attachment_files() + * only knows about 'original_image', so without this hook the HEIC + * would linger on disk after the attachment is deleted. + * + * @param int $post_id Attachment ID being deleted. + */ +function gutenberg_delete_heic_companion_file( int $post_id ): void { + $metadata = wp_get_attachment_metadata( $post_id, true ); + + if ( empty( $metadata['original'] ) ) { + return; + } + + $attached_file = get_attached_file( $post_id, true ); + + if ( ! $attached_file ) { + return; + } + + $heic_path = path_join( dirname( $attached_file ), $metadata['original'] ); + + if ( file_exists( $heic_path ) ) { + wp_delete_file( $heic_path ); + } +} + +add_action( 'delete_attachment', 'gutenberg_delete_heic_companion_file' ); + +// ── Tier 2: Full client-side processing (VIPS/WASM) ───────────────── +// Everything below requires cross-origin isolation (Document-Isolation-Policy) +// and SharedArrayBuffer support, which is only available in Chromium 137+. + +/** + * Sets a global JS variable to indicate that client-side media processing is enabled. + */ +function gutenberg_set_client_side_media_processing_flag() { + if ( ! gutenberg_is_client_side_media_processing_enabled() ) { + return; + } + wp_add_inline_script( 'wp-block-editor', 'window.__clientSideMediaProcessing = true', 'before' ); +} +add_action( 'admin_init', 'gutenberg_set_client_side_media_processing_flag' ); + /** * Filters the list of rewrite rules formatted for output to an .htaccess file. * diff --git a/packages/block-editor/src/components/provider/index.js b/packages/block-editor/src/components/provider/index.js index 18185c57c010aa..379aad102606a9 100644 --- a/packages/block-editor/src/components/provider/index.js +++ b/packages/block-editor/src/components/provider/index.js @@ -8,6 +8,7 @@ import { MediaUploadProvider, store as uploadStore, detectClientSideMediaSupport, + isHeicCanvasSupported, } from '@wordpress/upload-media'; /** @@ -38,6 +39,17 @@ let hasLoggedFallback = false; */ let isClientSideMediaEnabledCache = null; +/** + * Cached result of whether HEIC-only canvas processing should be enabled. + */ +let isHeicCanvasEnabledCache = null; + +/** + * HEIC MIME types that should be routed through the upload-media pipeline + * when in HEIC-only mode. + */ +const HEIC_MIME_TYPES = [ 'image/heic', 'image/heif' ]; + /** * Checks if client-side media processing should be enabled. * @@ -85,6 +97,44 @@ function shouldEnableClientSideMediaProcessing() { return true; } +/** + * Checks if HEIC-only canvas processing should be enabled. + * + * Returns true when: + * 1. Full client-side processing is NOT available (otherwise it handles HEIC already) + * 2. The server has set the __heicUploadSupport flag + * 3. The browser supports createImageBitmap + OffscreenCanvas (e.g. Safari) + * + * @return {boolean} Whether HEIC-only canvas processing should be enabled. + */ +function shouldEnableHeicCanvasProcessing() { + if ( isHeicCanvasEnabledCache !== null ) { + return isHeicCanvasEnabledCache; + } + + // If full client-side processing is enabled, it already handles HEIC. + if ( shouldEnableClientSideMediaProcessing() ) { + isHeicCanvasEnabledCache = false; + return false; + } + + if ( ! window.__heicUploadSupport ) { + isHeicCanvasEnabledCache = false; + return false; + } + + if ( + typeof isHeicCanvasSupported !== 'function' || + ! isHeicCanvasSupported() + ) { + isHeicCanvasEnabledCache = false; + return false; + } + + isHeicCanvasEnabledCache = true; + return true; +} + /** * Upload a media file when the file upload button is activated * or when adding a file to the editor via drag & drop. @@ -128,6 +178,92 @@ function mediaUpload( } ); } +/** + * Upload interceptor for HEIC-only mode. + * + * Routes HEIC files through the upload-media pipeline for canvas-based + * conversion, while passing non-HEIC files to the original mediaUpload + * function for standard server-side processing. + * + * @param {WPDataRegistry} registry + * @param {Object} settings Block editor settings. + * @param {Object} $3 Parameters object passed to the function. + * @param {Array} $3.allowedTypes Array with the types of media that can be uploaded, if unset all types are allowed. + * @param {Object} $3.additionalData Additional data to include in the request. + * @param {Array} $3.filesList List of files. + * @param {Function} $3.onError Function called when an error happens. + * @param {Function} $3.onFileChange Function called each time a file or a temporary representation of the file is available. + * @param {Function} $3.onSuccess Function called once a file has completely finished uploading, including thumbnails. + * @param {Function} $3.onBatchSuccess Function called once all files in a group have completely finished uploading, including thumbnails. + */ +function heicMediaUpload( + registry, + settings, + { + allowedTypes, + additionalData = {}, + filesList, + onError = noop, + onFileChange, + onSuccess, + onBatchSuccess, + } +) { + const files = Array.from( filesList ); + const heicFiles = files.filter( ( file ) => + HEIC_MIME_TYPES.includes( file.type ) + ); + const otherFiles = files.filter( + ( file ) => ! HEIC_MIME_TYPES.includes( file.type ) + ); + + // When the batch contains both HEIC and non-HEIC files, coordinate + // onBatchSuccess so it fires only after *both* paths have completed. + const hasBothPaths = + heicFiles.length > 0 && otherFiles.length > 0 && settings?.mediaUpload; + let pathsRemaining = hasBothPaths ? 2 : 1; + const coordinatedBatchSuccess = hasBothPaths + ? () => { + pathsRemaining--; + if ( pathsRemaining <= 0 ) { + onBatchSuccess?.(); + } + } + : onBatchSuccess; + + // Route HEIC files through the upload-media pipeline. + if ( heicFiles.length > 0 ) { + void registry.dispatch( uploadStore ).addItems( { + files: heicFiles, + onChange: onFileChange, + onSuccess: ( attachments ) => { + settings?.[ mediaUploadOnSuccessKey ]?.( attachments ); + onSuccess?.( attachments ); + }, + onBatchSuccess: coordinatedBatchSuccess, + onError: ( error ) => + onError( + typeof error === 'string' ? error : error?.message ?? '' + ), + additionalData, + allowedTypes, + } ); + } + + // Pass non-HEIC files to the original server-side upload function. + if ( otherFiles.length > 0 && settings?.mediaUpload ) { + settings.mediaUpload( { + allowedTypes, + additionalData, + filesList: otherFiles, + onError, + onFileChange, + onSuccess, + onBatchSuccess: coordinatedBatchSuccess, + } ); + } +} + /** * Calls useBlockSync as a child of SelectionContext.Provider so that the * hook can read selection state from the context provided by this tree @@ -152,6 +288,9 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( const isClientSideMediaEnabled = shouldEnableClientSideMediaProcessing(); + const isHeicCanvasEnabled = shouldEnableHeicCanvasProcessing(); + const useUploadMediaPipeline = + isClientSideMediaEnabled || isHeicCanvasEnabled; // Nested providers (e.g. from useBlockPreview) inherit settings // where mediaUpload has already been replaced with the @@ -162,16 +301,17 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( const settings = useMemo( () => { if ( - isClientSideMediaEnabled && + useUploadMediaPipeline && _settings?.mediaUpload && ! isMediaUploadIntercepted ) { - // Create a new object so that the original props.settings.mediaUpload is not modified. - const interceptor = mediaUpload.bind( - null, - registry, - _settings - ); + // Choose the right interceptor: + // - Full mode: all uploads go through upload-media pipeline. + // - HEIC-only mode: only HEIC files go through, rest use legacy path. + const uploadFn = isClientSideMediaEnabled + ? mediaUpload + : heicMediaUpload; + const interceptor = uploadFn.bind( null, registry, _settings ); interceptor.__isMediaUploadInterceptor = true; return { ..._settings, @@ -182,6 +322,7 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( }, [ _settings, registry, + useUploadMediaPipeline, isClientSideMediaEnabled, isMediaUploadIntercepted, ] ); @@ -258,7 +399,7 @@ export const ExperimentalBlockEditorProvider = withRegistryProvider( // overwrite the store's server-side function with the // interceptor, causing uploads to loop instead of reaching // the server. - if ( isClientSideMediaEnabled && ! isMediaUploadIntercepted ) { + if ( useUploadMediaPipeline && ! isMediaUploadIntercepted ) { return ( { - if ( ! window.__clientSideMediaProcessing || ! id ) { + if ( + ( ! window.__clientSideMediaProcessing && + ! window.__heicUploadSupport ) || + ! id + ) { return false; } return select( uploadStore ).isUploadingById( id ); diff --git a/packages/editor/src/components/provider/use-upload-save-lock.js b/packages/editor/src/components/provider/use-upload-save-lock.js index 8c0473e061774f..c4803d478db025 100644 --- a/packages/editor/src/components/provider/use-upload-save-lock.js +++ b/packages/editor/src/components/provider/use-upload-save-lock.js @@ -20,7 +20,7 @@ const LOCK_NAME = 'upload-in-progress'; */ export default function useUploadSaveLock() { const isClientSideMediaProcessingEnabled = - window.__clientSideMediaProcessing; + window.__clientSideMediaProcessing || window.__heicUploadSupport; const isUploading = useSelect( ( select ) => { diff --git a/packages/media-utils/src/utils/types.ts b/packages/media-utils/src/utils/types.ts index 2be7ed5f17e0ef..1243bce329eb97 100644 --- a/packages/media-utils/src/utils/types.ts +++ b/packages/media-utils/src/utils/types.ts @@ -208,6 +208,8 @@ export type AdditionalData = BetterOmit< CreateRestAttachment, 'meta' >; export interface CreateSideloadFile { image_size?: string; upload_request?: string; + generate_sub_sizes?: boolean; + convert_format?: boolean; } export interface SideloadAdditionalData { diff --git a/packages/media-utils/src/utils/upload-media.ts b/packages/media-utils/src/utils/upload-media.ts index 278b5c4ababf7a..79c6fe0697101a 100644 --- a/packages/media-utils/src/utils/upload-media.ts +++ b/packages/media-utils/src/utils/upload-media.ts @@ -22,6 +22,7 @@ import { UploadError } from './upload-error'; declare global { interface Window { __clientSideMediaProcessing?: boolean; + __heicUploadSupport?: boolean; } } diff --git a/packages/upload-media/src/canvas-utils.ts b/packages/upload-media/src/canvas-utils.ts new file mode 100644 index 00000000000000..909e7072df72d5 --- /dev/null +++ b/packages/upload-media/src/canvas-utils.ts @@ -0,0 +1,255 @@ +/** + * Internal dependencies + */ +import { getFileBasename } from './utils'; +import { parseHeic } from './heic-parser'; + +/** + * Converts an image file to JPEG using the browser's native decoder and canvas. + * + * Tries three decoding strategies: + * 1. createImageBitmap() + OffscreenCanvas (works in Safari, future Chrome). + * 2. WebCodecs ImageDecoder API (uses platform codecs; may work in future + * Chrome if HEIC is added to its image decoder pipeline). + * 3. HEIC container parsing + WebCodecs VideoDecoder (Chrome 107+ on macOS). + * Parses the HEIC/ISOBMFF container to extract the HEVC bitstream, then + * decodes it using Chrome's platform HEVC video decoder (hardware- + * accelerated via macOS VideoToolbox). + * + * This avoids shipping our own HEVC decoder, sidestepping patent/licensing concerns. + * + * @param file Source image file (e.g., HEIC/HEIF). + * @param quality JPEG quality (0-1). Default 0.82. + * @return JPEG File object. + */ +export async function canvasConvertToJpeg( + file: File, + quality = 0.82 +): Promise< File > { + const baseName = getFileBasename( file.name ); + + // Strategy 1: createImageBitmap + OffscreenCanvas. + try { + const bitmap = await createImageBitmap( file ); + try { + const canvas = new OffscreenCanvas( bitmap.width, bitmap.height ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } + + ctx.drawImage( bitmap, 0, 0 ); + + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } finally { + bitmap.close(); + } + } catch { + // createImageBitmap doesn't support HEIC in this browser. + // Fall through to strategy 2. + } + + // Strategy 2: WebCodecs ImageDecoder API. + // Uses platform codecs (e.g., macOS HEIC support) that may not be + // exposed through createImageBitmap or elements. + if ( typeof ImageDecoder !== 'undefined' ) { + const supported = await ImageDecoder.isTypeSupported( file.type ); + if ( supported ) { + const decoder = new ImageDecoder( { + type: file.type, + data: file.stream(), + } ); + try { + const { image: videoFrame } = await decoder.decode(); + try { + const canvas = new OffscreenCanvas( + videoFrame.displayWidth, + videoFrame.displayHeight + ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } + + ctx.drawImage( videoFrame, 0, 0 ); + + const jpegBlob = await canvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } finally { + videoFrame.close(); + } + } finally { + decoder.close(); + } + } + } + + // Strategy 3: HEIC container parsing + WebCodecs VideoDecoder. + // Chrome 107+ on macOS supports HEVC *video* decoding via platform codecs + // (macOS VideoToolbox), even though it doesn't support HEIC through image + // APIs. A HEIC file is an ISOBMFF container with HEVC-encoded tiles — + // we parse the container and decode each tile via VideoDecoder. + if ( typeof VideoDecoder !== 'undefined' ) { + try { + const heicData = parseHeic( await file.arrayBuffer() ); + + const support = await VideoDecoder.isConfigSupported( { + codec: heicData.codecString, + } ); + + if ( support.supported ) { + const canvas = new OffscreenCanvas( + heicData.outputWidth, + heicData.outputHeight + ); + const ctx = canvas.getContext( '2d' ); + + if ( ! ctx ) { + throw new Error( 'Could not get canvas 2d context' ); + } + + // Decode each tile and draw it at its grid position. + for ( const tile of heicData.tiles ) { + const frame = await decodeHevcFrame( + heicData.codecString, + heicData.description, + heicData.tileWidth, + heicData.tileHeight, + tile.data + ); + try { + ctx.drawImage( frame, tile.x, tile.y ); + } finally { + frame.close(); + } + } + + // Apply ISOBMFF irot rotation if present. + const outputCanvas = applyRotation( canvas, heicData.rotation ); + + const jpegBlob = await outputCanvas.convertToBlob( { + type: 'image/jpeg', + quality, + } ); + + return new File( [ jpegBlob ], `${ baseName }.jpeg`, { + type: 'image/jpeg', + } ); + } + } catch { + // VideoDecoder HEVC not available or HEIC parsing failed. + // Fall through to error. + } + } + + throw new Error( + 'This browser cannot decode HEIC images. Please use Safari or convert to JPEG before uploading.' + ); +} + +/** + * Apply ISOBMFF irot rotation to a canvas. + * + * Returns the original canvas if no rotation is needed, or a new + * OffscreenCanvas with the rotation applied. + * + * @param source Source canvas with the decoded image. + * @param rotation Rotation angle in degrees counter-clockwise (0, 90, 180, 270). + * @return Canvas with rotation applied. + */ +function applyRotation( + source: OffscreenCanvas, + rotation: number +): OffscreenCanvas { + if ( rotation === 0 ) { + return source; + } + + const swap = rotation === 90 || rotation === 270; + const w = swap ? source.height : source.width; + const h = swap ? source.width : source.height; + + const rotated = new OffscreenCanvas( w, h ); + const ctx = rotated.getContext( '2d' ); + + if ( ! ctx ) { + return source; + } + + ctx.translate( w / 2, h / 2 ); + // irot angle is CCW; canvas rotate() is CW, so negate. + ctx.rotate( ( -rotation * Math.PI ) / 180 ); + ctx.drawImage( source, -source.width / 2, -source.height / 2 ); + + return rotated; +} + +/** + * Decode a single HEVC key frame using the WebCodecs VideoDecoder API. + * + * @param codec HEVC codec string (e.g. 'hvc1.1.6.L93.B0'). + * @param description HEVCDecoderConfigurationRecord bytes. + * @param width Coded width of the frame. + * @param height Coded height of the frame. + * @param data Raw HEVC bitstream (IDR frame). + * @return Decoded VideoFrame. Caller must call frame.close(). + */ +function decodeHevcFrame( + codec: string, + description: Uint8Array, + width: number, + height: number, + data: Uint8Array +): Promise< VideoFrame > { + return new Promise< VideoFrame >( ( resolve, reject ) => { + const decoder = new VideoDecoder( { + output: ( frame ) => { + decoder.close(); + resolve( frame ); + }, + error: ( e ) => { + if ( decoder.state !== 'closed' ) { + decoder.close(); + } + reject( e ); + }, + } ); + + decoder.configure( { + codec, + codedWidth: width, + codedHeight: height, + description, + } ); + + decoder.decode( + new EncodedVideoChunk( { + type: 'key', + timestamp: 0, + data, + } ) + ); + + decoder.flush().catch( ( e ) => { + if ( decoder.state !== 'closed' ) { + decoder.close(); + } + reject( e ); + } ); + } ); +} diff --git a/packages/upload-media/src/feature-detection.ts b/packages/upload-media/src/feature-detection.ts index a69d7cfaad8d17..1b05187594effd 100644 --- a/packages/upload-media/src/feature-detection.ts +++ b/packages/upload-media/src/feature-detection.ts @@ -165,6 +165,23 @@ export function isClientSideMediaSupported(): boolean { return detectClientSideMediaSupport().supported; } +/** + * Detects whether the browser can decode HEIC images via canvas APIs. + * + * This checks for createImageBitmap and OffscreenCanvas support, + * which are sufficient to convert HEIC to JPEG without VIPS/WASM. + * Safari supports both APIs and can natively decode HEIC via + * createImageBitmap(), leveraging macOS platform codecs. + * + * @return Whether HEIC canvas-based processing is supported. + */ +export function isHeicCanvasSupported(): boolean { + return ( + typeof createImageBitmap !== 'undefined' && + typeof OffscreenCanvas !== 'undefined' + ); +} + /** * Clears the cached feature detection result. * diff --git a/packages/upload-media/src/heic-parser.ts b/packages/upload-media/src/heic-parser.ts new file mode 100644 index 00000000000000..4625263486dc61 --- /dev/null +++ b/packages/upload-media/src/heic-parser.ts @@ -0,0 +1,839 @@ +/* eslint-disable no-bitwise */ + +/** + * Lightweight HEIC (ISOBMFF) container parser. + * + * Extracts the HEVC decoder configuration and compressed image data from + * HEIC files, including grid/tiled images (the common iPhone format). + * This enables decoding via the WebCodecs VideoDecoder API without + * shipping our own HEVC codec. + * + * Only the container format is parsed here (no patented codec involved). + * Actual HEVC decoding is delegated to the browser's platform decoder. + */ + +/** A single tile to decode and draw. */ +export interface HeicTile { + /** Raw HEVC bitstream for this tile. */ + data: Uint8Array; + /** X position on the output canvas. */ + x: number; + /** Y position on the output canvas. */ + y: number; +} + +export interface HeicImageData { + /** HEVC codec string for VideoDecoder (e.g. 'hvc1.1.6.L93.B0'). */ + codecString: string; + /** Raw HEVCDecoderConfigurationRecord bytes for VideoDecoder description. */ + description: Uint8Array; + /** Tiles to decode (1 for single-image, many for grid). */ + tiles: HeicTile[]; + /** Width of each HEVC tile in pixels. */ + tileWidth: number; + /** Height of each HEVC tile in pixels. */ + tileHeight: number; + /** Final output width in pixels. */ + outputWidth: number; + /** Final output height in pixels. */ + outputHeight: number; + /** Rotation angle in degrees counter-clockwise (0, 90, 180, 270). */ + rotation: number; +} + +// --------------------------------------------------------------------------- +// Binary reader +// --------------------------------------------------------------------------- + +class Reader { + readonly view: DataView; + readonly buffer: ArrayBuffer; + pos: number; + + constructor( buffer: ArrayBuffer, offset = 0 ) { + this.buffer = buffer; + this.view = new DataView( buffer ); + this.pos = offset; + } + + u8(): number { + const v = this.view.getUint8( this.pos ); + this.pos += 1; + return v; + } + + u16(): number { + const v = this.view.getUint16( this.pos ); + this.pos += 2; + return v; + } + + u32(): number { + const v = this.view.getUint32( this.pos ); + this.pos += 4; + return v; + } + + u64(): number { + const hi = this.view.getUint32( this.pos ); + const lo = this.view.getUint32( this.pos + 4 ); + this.pos += 8; + return hi * 0x100000000 + lo; + } + + /** + * Read a variable-width unsigned integer (0, 4 or 8 bytes). + * + * @param size Byte width to read (0, 4, or 8). + */ + uN( size: number ): number { + if ( size === 0 ) { + return 0; + } + if ( size === 4 ) { + return this.u32(); + } + if ( size === 8 ) { + return this.u64(); + } + throw new Error( `Unsupported uint size: ${ size }` ); + } + + str( len: number ): string { + let s = ''; + for ( let i = 0; i < len; i++ ) { + s += String.fromCharCode( this.view.getUint8( this.pos + i ) ); + } + this.pos += len; + return s; + } + + bytes( len: number ): Uint8Array { + const b = new Uint8Array( this.buffer, this.pos, len ); + this.pos += len; + return new Uint8Array( b ); // copy to avoid detach issues + } +} + +// --------------------------------------------------------------------------- +// ISOBMFF box helpers +// --------------------------------------------------------------------------- + +interface BoxInfo { + type: string; + offset: number; + size: number; + headerSize: number; +} + +function readBox( r: Reader ): BoxInfo | null { + if ( r.pos + 8 > r.view.byteLength ) { + return null; + } + const offset = r.pos; + let size: number = r.u32(); + const type = r.str( 4 ); + let headerSize = 8; + + if ( size === 1 ) { + size = r.u64(); + headerSize = 16; + } else if ( size === 0 ) { + size = r.view.byteLength - offset; + } + + return { type, offset, size, headerSize }; +} + +function findBoxes( r: Reader, start: number, end: number ): BoxInfo[] { + const boxes: BoxInfo[] = []; + r.pos = start; + while ( r.pos < end ) { + const box = readBox( r ); + if ( ! box || box.size < 8 ) { + break; + } + boxes.push( box ); + r.pos = box.offset + box.size; + } + return boxes; +} + +function findBox( + r: Reader, + start: number, + end: number, + type: string +): BoxInfo | undefined { + r.pos = start; + while ( r.pos < end ) { + const box = readBox( r ); + if ( ! box || box.size < 8 ) { + break; + } + if ( box.type === type ) { + return box; + } + r.pos = box.offset + box.size; + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Specific box parsers +// --------------------------------------------------------------------------- + +/** + * Parse Primary Item Box → primary item ID. + * + * @param r Binary reader. + * @param box BoxInfo for the pitm box. + */ +function parsePitm( r: Reader, box: BoxInfo ): number { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + return version === 0 ? r.u16() : r.u32(); +} + +interface ItemExtent { + offset: number; + length: number; +} + +interface ItemLocation { + /** 0 = file offset (mdat), 1 = idat offset. */ + constructionMethod: number; + extents: ItemExtent[]; +} + +/** + * Parse Item Location Box → map of item ID to data extents. + * + * @param r Binary reader. + * @param box BoxInfo for the iloc box. + */ +function parseIloc( r: Reader, box: BoxInfo ): Map< number, ItemLocation > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const byte1 = r.u8(); + const offsetSize = ( byte1 >> 4 ) & 0xf; + const lengthSize = byte1 & 0xf; + + const byte2 = r.u8(); + const baseOffsetSize = ( byte2 >> 4 ) & 0xf; + const indexSize = version >= 1 ? byte2 & 0xf : 0; + + const itemCount = version < 2 ? r.u16() : r.u32(); + const items = new Map< number, ItemLocation >(); + + for ( let i = 0; i < itemCount; i++ ) { + const itemId = version < 2 ? r.u16() : r.u32(); + + let constructionMethod = 0; + if ( version === 1 || version === 2 ) { + const cm = r.u16(); + constructionMethod = cm & 0xf; // lower 4 bits + } + + r.u16(); // data_reference_index + const baseOffset = r.uN( baseOffsetSize ); + const extentCount = r.u16(); + const extents: ItemExtent[] = []; + + for ( let j = 0; j < extentCount; j++ ) { + if ( version >= 1 ) { + r.uN( indexSize ); // extent_index — skip + } + const extOffset = r.uN( offsetSize ); + const extLength = r.uN( lengthSize ); + extents.push( { + offset: baseOffset + extOffset, + length: extLength, + } ); + } + + items.set( itemId, { constructionMethod, extents } ); + } + + return items; +} + +/** + * Parse Item Property Association Box → map of item ID to 1-based property indices. + * + * @param r Binary reader. + * @param box BoxInfo for the ipma box. + */ +function parseIpma( r: Reader, box: BoxInfo ): Map< number, number[] > { + r.pos = box.offset + box.headerSize; + const vf = r.u32(); // version (8 bits) + flags (24 bits) + const version = vf >>> 24; + const flags = vf & 0xffffff; + const largeIndex = ( flags & 1 ) !== 0; + + const entryCount = r.u32(); + const associations = new Map< number, number[] >(); + + for ( let i = 0; i < entryCount; i++ ) { + const itemId = version < 1 ? r.u16() : r.u32(); + const assocCount = r.u8(); + const indices: number[] = []; + + for ( let j = 0; j < assocCount; j++ ) { + if ( largeIndex ) { + indices.push( r.u16() & 0x7fff ); // strip essential bit + } else { + indices.push( r.u8() & 0x7f ); // strip essential bit + } + } + + associations.set( itemId, indices ); + } + + return associations; +} + +/** + * Parse Image Spatial Extents → width & height. + * + * @param r Binary reader. + * @param box BoxInfo for the ispe box. + */ +function parseIspe( + r: Reader, + box: BoxInfo +): { width: number; height: number } { + // ispe is a FullBox: skip version (1) + flags (3). + r.pos = box.offset + box.headerSize + 4; + return { width: r.u32(), height: r.u32() }; +} + +/** + * Parse Image Rotation box → rotation angle in degrees CCW. + * + * Format: 1 byte with reserved (6 bits) + angle (2 bits). + * angle * 90 = rotation in degrees counter-clockwise. + * + * @param r Binary reader. + * @param box BoxInfo for the irot box. + */ +function parseIrot( r: Reader, box: BoxInfo ): number { + r.pos = box.offset + box.headerSize; + return ( r.u8() & 0x3 ) * 90; +} + +/** + * Parse Item Info Box → map of item ID to item type (4-char code). + * + * @param r Binary reader. + * @param box BoxInfo for the iinf box. + */ +function parseIinf( r: Reader, box: BoxInfo ): Map< number, string > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const entryCount = version === 0 ? r.u16() : r.u32(); + const itemTypes = new Map< number, string >(); + + // Parse infe (ItemInfoEntry) sub-boxes. + const entriesStart = r.pos; + const boxEnd = box.offset + box.size; + + const infeBoxes = findBoxes( r, entriesStart, boxEnd ); + for ( let i = 0; i < Math.min( entryCount, infeBoxes.length ); i++ ) { + const infe = infeBoxes[ i ]; + if ( infe.type !== 'infe' ) { + continue; + } + r.pos = infe.offset + infe.headerSize; + const infeVersion = r.u8(); + r.pos += 3; // flags + + if ( infeVersion >= 2 ) { + const itemId = infeVersion === 2 ? r.u16() : r.u32(); + r.u16(); // item_protection_index + const itemType = r.str( 4 ); + itemTypes.set( itemId, itemType ); + } + } + + return itemTypes; +} + +/** + * Parse Item Reference Box → map of (fromItemId) to array of referenced item IDs, + * filtered by reference type. + * + * @param r Binary reader. + * @param box BoxInfo for the iref box. + * @param refType Reference type to filter (e.g. 'dimg'). + */ +function parseIref( + r: Reader, + box: BoxInfo, + refType: string +): Map< number, number[] > { + r.pos = box.offset + box.headerSize; + const version = r.u8(); + r.pos += 3; // flags + + const refs = new Map< number, number[] >(); + const boxEnd = box.offset + box.size; + + while ( r.pos < boxEnd ) { + const refBox = readBox( r ); + if ( ! refBox || refBox.size < 8 ) { + break; + } + + r.pos = refBox.offset + refBox.headerSize; + const fromId = version === 0 ? r.u16() : r.u32(); + const refCount = r.u16(); + const toIds: number[] = []; + + for ( let i = 0; i < refCount; i++ ) { + toIds.push( version === 0 ? r.u16() : r.u32() ); + } + + if ( refBox.type === refType ) { + refs.set( fromId, toIds ); + } + + r.pos = refBox.offset + refBox.size; + } + + return refs; +} + +// --------------------------------------------------------------------------- +// HEVC codec string construction +// --------------------------------------------------------------------------- + +/** + * Reverse all 32 bits of a number. + * + * @param n 32-bit unsigned integer. + */ +export function reverseBits32( n: number ): number { + n = ( ( n >>> 1 ) & 0x55555555 ) | ( ( n & 0x55555555 ) << 1 ); + n = ( ( n >>> 2 ) & 0x33333333 ) | ( ( n & 0x33333333 ) << 2 ); + n = ( ( n >>> 4 ) & 0x0f0f0f0f ) | ( ( n & 0x0f0f0f0f ) << 4 ); + n = ( ( n >>> 8 ) & 0x00ff00ff ) | ( ( n & 0x00ff00ff ) << 8 ); + n = ( n >>> 16 ) | ( n << 16 ); + return n >>> 0; +} + +/** + * Build an HEVC codec string from an HEVCDecoderConfigurationRecord. + * + * Format: hvc1.{profile}.{compat}.{tier}{level}[.{constraints}] + * See ISO 14496-15 Annex E and W3C WebCodecs HEVC Codec Registration. + * + * @param r Binary reader. + * @param recordOffset Byte offset of the HEVCDecoderConfigurationRecord. + */ +function buildCodecString( r: Reader, recordOffset: number ): string { + r.pos = recordOffset; + r.u8(); // configurationVersion + + const byte1 = r.u8(); + const profileSpace = ( byte1 >> 6 ) & 0x3; + const tierFlag = ( byte1 >> 5 ) & 0x1; + const profileIdc = byte1 & 0x1f; + + const compatFlags = r.u32(); + const constraintBytes = r.bytes( 6 ); + const levelIdc = r.u8(); + + // Profile: optional space prefix (A/B/C) + profile_idc. + const spacePrefix = + profileSpace > 0 ? String.fromCharCode( 64 + profileSpace ) : ''; + + // Compatibility flags: bit-reversed, as hex. + const compatHex = reverseBits32( compatFlags ).toString( 16 ).toUpperCase(); + + // Tier: 'L' (Main) or 'H' (High). + const tierChar = tierFlag ? 'H' : 'L'; + + // Constraint indicator flags: each byte as hex, trailing zeros removed. + let lastNonZero = -1; + for ( let i = 5; i >= 0; i-- ) { + if ( constraintBytes[ i ] !== 0 ) { + lastNonZero = i; + break; + } + } + let constraintStr = ''; + if ( lastNonZero >= 0 ) { + const parts: string[] = []; + for ( let i = 0; i <= lastNonZero; i++ ) { + parts.push( constraintBytes[ i ].toString( 16 ).toUpperCase() ); + } + constraintStr = '.' + parts.join( '.' ); + } + + return `hvc1.${ spacePrefix }${ profileIdc }.${ compatHex }.${ tierChar }${ levelIdc }${ constraintStr }`; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Read item data by concatenating all extents for a given item location. + * + * For construction_method 0, offsets are absolute file positions. + * For construction_method 1, offsets are relative to the idat box data. + * + * @param buffer File ArrayBuffer. + * @param loc Item location with extents. + * @param idatOffset Byte offset of the idat box's data within the file. + */ +function readItemData( + buffer: ArrayBuffer, + loc: ItemLocation, + idatOffset: number +): Uint8Array { + const baseOffset = loc.constructionMethod === 1 ? idatOffset : 0; + + if ( loc.extents.length === 1 ) { + const ext = loc.extents[ 0 ]; + const start = baseOffset + ext.offset; + return new Uint8Array( buffer.slice( start, start + ext.length ) ); + } + let totalLength = 0; + for ( const ext of loc.extents ) { + totalLength += ext.length; + } + const data = new Uint8Array( totalLength ); + let pos = 0; + for ( const ext of loc.extents ) { + const start = baseOffset + ext.offset; + data.set( + new Uint8Array( buffer.slice( start, start + ext.length ) ), + pos + ); + pos += ext.length; + } + return data; +} + +/** + * Find hvcC, ispe, and irot property boxes for a given item. + * + * @param propIndices 1-based property indices from ipma. + * @param properties All property boxes from ipco. + */ +function findHvcProperties( + propIndices: number[], + properties: BoxInfo[] +): { hvcCBox: BoxInfo; ispeBox: BoxInfo; irotBox?: BoxInfo } { + let hvcCBox: BoxInfo | undefined; + let ispeBox: BoxInfo | undefined; + let irotBox: BoxInfo | undefined; + + for ( const idx of propIndices ) { + if ( idx < 1 || idx > properties.length ) { + continue; + } + const prop = properties[ idx - 1 ]; + if ( prop.type === 'hvcC' && ! hvcCBox ) { + hvcCBox = prop; + } + if ( prop.type === 'ispe' && ! ispeBox ) { + ispeBox = prop; + } + if ( prop.type === 'irot' && ! irotBox ) { + irotBox = prop; + } + } + + if ( ! hvcCBox ) { + throw new Error( 'No HEVC configuration (hvcC) found' ); + } + if ( ! ispeBox ) { + throw new Error( 'No image dimensions (ispe) found' ); + } + + return { hvcCBox, ispeBox, irotBox }; +} + +// --------------------------------------------------------------------------- +// Main entry point +// --------------------------------------------------------------------------- + +/** + * Parse a HEIC file and extract the data needed for VideoDecoder. + * + * Handles both single-image HEIC files and grid/tiled images (the common + * iPhone format where the full image is split into multiple HEVC tiles). + * + * @param buffer Raw HEIC file contents. + * @return Parsed image data including codec config and HEVC tile data. + * @throws If the file is not a valid HEIC or lacks required boxes. + */ +export function parseHeic( buffer: ArrayBuffer ): HeicImageData { + const r = new Reader( buffer ); + const fileEnd = buffer.byteLength; + + // Find top-level 'meta' box. + const metaBox = findBox( r, 0, fileEnd, 'meta' ); + if ( ! metaBox ) { + throw new Error( 'No meta box found in HEIC file' ); + } + + // meta is a FullBox: children start after version (1) + flags (3). + const metaChildStart = metaBox.offset + metaBox.headerSize + 4; + const metaEnd = metaBox.offset + metaBox.size; + + // Locate required child boxes within meta. + const children = findBoxes( r, metaChildStart, metaEnd ); + const pitmBox = children.find( ( b ) => b.type === 'pitm' ); + const ilocBox = children.find( ( b ) => b.type === 'iloc' ); + const iprpBox = children.find( ( b ) => b.type === 'iprp' ); + const iinfBox = children.find( ( b ) => b.type === 'iinf' ); + const irefBox = children.find( ( b ) => b.type === 'iref' ); + const idatBox = children.find( ( b ) => b.type === 'idat' ); + + // idat data offset (for construction_method 1 items). + const idatOffset = idatBox ? idatBox.offset + idatBox.headerSize : 0; + + if ( ! pitmBox || ! ilocBox || ! iprpBox ) { + throw new Error( 'Missing required boxes (pitm, iloc, iprp) in HEIC' ); + } + + // Primary item ID. + const primaryId = parsePitm( r, pitmBox ); + + // Item locations. + const locations = parseIloc( r, ilocBox ); + + // Item properties: iprp contains ipco (properties) + ipma (associations). + const iprpStart = iprpBox.offset + iprpBox.headerSize; + const iprpEnd = iprpBox.offset + iprpBox.size; + const iprpChildren = findBoxes( r, iprpStart, iprpEnd ); + const ipcoBox = iprpChildren.find( ( b ) => b.type === 'ipco' ); + const ipmaBox = iprpChildren.find( ( b ) => b.type === 'ipma' ); + + if ( ! ipcoBox || ! ipmaBox ) { + throw new Error( 'Missing ipco or ipma in HEIC properties' ); + } + + const allAssoc = parseIpma( r, ipmaBox ); + + // Enumerate ipco children (properties are 1-indexed). + const ipcoStart = ipcoBox.offset + ipcoBox.headerSize; + const ipcoEnd = ipcoBox.offset + ipcoBox.size; + const properties = findBoxes( r, ipcoStart, ipcoEnd ); + + // Determine if the primary item is a grid or a direct HEVC image. + let primaryItemType = 'hvc1'; + if ( iinfBox ) { + const itemTypes = parseIinf( r, iinfBox ); + const t = itemTypes.get( primaryId ); + if ( t ) { + primaryItemType = t; + } + } + + if ( primaryItemType === 'grid' ) { + // --- Grid/tiled image (common iPhone format) --- + return parseGridImage( + r, + buffer, + primaryId, + locations, + allAssoc, + properties, + irefBox, + idatOffset + ); + } + + // --- Single HEVC image --- + const primaryLoc = locations.get( primaryId ); + if ( ! primaryLoc || primaryLoc.extents.length === 0 ) { + throw new Error( `No location data for primary item ${ primaryId }` ); + } + + const primaryPropIndices = allAssoc.get( primaryId ); + if ( ! primaryPropIndices || primaryPropIndices.length === 0 ) { + throw new Error( 'No property associations for primary item' ); + } + + const { hvcCBox, ispeBox, irotBox } = findHvcProperties( + primaryPropIndices, + properties + ); + + const hvcCDataStart = hvcCBox.offset + hvcCBox.headerSize; + const hvcCDataSize = hvcCBox.size - hvcCBox.headerSize; + const description = new Uint8Array( + buffer.slice( hvcCDataStart, hvcCDataStart + hvcCDataSize ) + ); + const codecString = buildCodecString( r, hvcCDataStart ); + const { width, height } = parseIspe( r, ispeBox ); + const rotation = irotBox ? parseIrot( r, irotBox ) : 0; + + return { + codecString, + description, + tiles: [ + { + data: readItemData( buffer, primaryLoc, idatOffset ), + x: 0, + y: 0, + }, + ], + tileWidth: width, + tileHeight: height, + outputWidth: width, + outputHeight: height, + rotation, + }; +} + +/** + * Parse a grid/tiled HEIC image. + * + * @param r Binary reader. + * @param buffer File ArrayBuffer. + * @param gridItemId The grid item's ID. + * @param locations Parsed iloc data. + * @param allAssoc Parsed ipma data. + * @param properties ipco property boxes. + * @param irefBox iref box (required for grid). + * @param idatOffset Byte offset of idat box data (for construction_method 1). + */ +function parseGridImage( + r: Reader, + buffer: ArrayBuffer, + gridItemId: number, + locations: Map< number, ItemLocation >, + allAssoc: Map< number, number[] >, + properties: BoxInfo[], + irefBox: BoxInfo | undefined, + idatOffset: number +): HeicImageData { + // Parse grid descriptor from the grid item's data. + const gridLoc = locations.get( gridItemId ); + if ( ! gridLoc || gridLoc.extents.length === 0 ) { + throw new Error( 'No location data for grid item' ); + } + const gridData = readItemData( buffer, gridLoc, idatOffset ); + + // Grid descriptor format: + // version (1 byte), flags (1 byte), + // rows_minus_one (1 byte), columns_minus_one (1 byte), + // output_width (2 or 4 bytes), output_height (2 or 4 bytes) + const largeFields = gridData.length > 1 && ( gridData[ 1 ] & 1 ) !== 0; + const minGridSize = largeFields ? 12 : 8; + if ( gridData.length < minGridSize ) { + throw new Error( + `Grid descriptor too short: ${ gridData.length } bytes` + ); + } + + const rows = gridData[ 2 ] + 1; + const columns = gridData[ 3 ] + 1; + + const gv = new DataView( gridData.buffer, gridData.byteOffset ); + let outputWidth: number; + let outputHeight: number; + if ( largeFields ) { + outputWidth = gv.getUint32( 4 ); + outputHeight = gv.getUint32( 8 ); + } else { + outputWidth = gv.getUint16( 4 ); + outputHeight = gv.getUint16( 6 ); + } + + // Find tile item IDs from iref 'dimg' references. + if ( ! irefBox ) { + throw new Error( 'Grid image requires iref box' ); + } + const dimgRefs = parseIref( r, irefBox, 'dimg' ); + const tileItemIds = dimgRefs.get( gridItemId ); + if ( ! tileItemIds || tileItemIds.length === 0 ) { + throw new Error( 'No tile references found for grid item' ); + } + + // The iref may include extra references (alpha planes, thumbnails). + // Use at least rows * columns tiles; ignore any surplus. + const expectedTiles = rows * columns; + if ( tileItemIds.length < expectedTiles ) { + throw new Error( + `Grid expects ${ expectedTiles } tiles but found ${ tileItemIds.length }` + ); + } + + // Get hvcC and ispe from the first tile item's properties. + // All tiles in a grid share the same HEVC configuration. + const firstTileProps = allAssoc.get( tileItemIds[ 0 ] ); + if ( ! firstTileProps || firstTileProps.length === 0 ) { + throw new Error( 'No property associations for tile item' ); + } + + const { hvcCBox, ispeBox } = findHvcProperties( + firstTileProps, + properties + ); + + // irot is associated with the grid item, not the tiles. + const gridProps = allAssoc.get( gridItemId ) || []; + let irotBox: BoxInfo | undefined; + for ( const idx of gridProps ) { + if ( idx >= 1 && idx <= properties.length ) { + const prop = properties[ idx - 1 ]; + if ( prop.type === 'irot' ) { + irotBox = prop; + break; + } + } + } + + const hvcCDataStart = hvcCBox.offset + hvcCBox.headerSize; + const hvcCDataSize = hvcCBox.size - hvcCBox.headerSize; + const description = new Uint8Array( + buffer.slice( hvcCDataStart, hvcCDataStart + hvcCDataSize ) + ); + const codecString = buildCodecString( r, hvcCDataStart ); + const { width: tileWidth, height: tileHeight } = parseIspe( r, ispeBox ); + + // Extract tile data in raster scan order (left→right, top→bottom). + const tiles: HeicTile[] = []; + for ( let row = 0; row < rows; row++ ) { + for ( let col = 0; col < columns; col++ ) { + const tileIdx = row * columns + col; + const tileId = tileItemIds[ tileIdx ]; + const tileLoc = locations.get( tileId ); + if ( ! tileLoc || tileLoc.extents.length === 0 ) { + throw new Error( `No location data for tile item ${ tileId }` ); + } + tiles.push( { + data: readItemData( buffer, tileLoc, idatOffset ), + x: col * tileWidth, + y: row * tileHeight, + } ); + } + } + + const rotation = irotBox ? parseIrot( r, irotBox ) : 0; + + return { + codecString, + description, + tiles, + tileWidth, + tileHeight, + outputWidth, + outputHeight, + rotation, + }; +} + +/* eslint-enable no-bitwise */ diff --git a/packages/upload-media/src/index.ts b/packages/upload-media/src/index.ts index ec4225139c4b1b..032a99367c2c94 100644 --- a/packages/upload-media/src/index.ts +++ b/packages/upload-media/src/index.ts @@ -10,6 +10,7 @@ export { UploadError } from './upload-error'; export { detectClientSideMediaSupport, isClientSideMediaSupported, + isHeicCanvasSupported, clearFeatureDetectionCache, } from './feature-detection'; diff --git a/packages/upload-media/src/store/constants.ts b/packages/upload-media/src/store/constants.ts index d0b6f5591387ae..d71b6c1ed62499 100644 --- a/packages/upload-media/src/store/constants.ts +++ b/packages/upload-media/src/store/constants.ts @@ -28,3 +28,15 @@ export const CLIENT_SIDE_SUPPORTED_MIME_TYPES: readonly string[] = [ 'image/webp', 'image/avif', ] as const; + +/** + * HEIC/HEIF MIME types. + * + * These formats use the HEVC codec which has patent/licensing restrictions. + * Instead of shipping our own decoder, the client falls back to the browser's + * native createImageBitmap() which leverages OS/browser-licensed HEVC codecs. + */ +export const HEIC_MIME_TYPES: readonly string[] = [ + 'image/heic', + 'image/heif', +] as const; diff --git a/packages/upload-media/src/store/private-actions.ts b/packages/upload-media/src/store/private-actions.ts index a43c5e7acd6b29..75e3c91731e90c 100644 --- a/packages/upload-media/src/store/private-actions.ts +++ b/packages/upload-media/src/store/private-actions.ts @@ -14,7 +14,9 @@ type WPDataRegistry = ReturnType< typeof createRegistry >; * Internal dependencies */ import { cloneFile, convertBlobToFile, renameFile } from '../utils'; -import { CLIENT_SIDE_SUPPORTED_MIME_TYPES } from './constants'; +import { canvasConvertToJpeg } from '../canvas-utils'; +import { isClientSideMediaSupported } from '../feature-detection'; +import { CLIENT_SIDE_SUPPORTED_MIME_TYPES, HEIC_MIME_TYPES } from './constants'; import { StubFile } from '../stub-file'; import { UploadError } from '../upload-error'; import { @@ -63,6 +65,7 @@ type ActionCreators = { addItem: typeof addItem; addSideloadItem: typeof addSideloadItem; removeItem: typeof removeItem; + pauseItem: typeof pauseItem; prepareItem: typeof prepareItem; processItem: typeof processItem; finishOperation: typeof finishOperation; @@ -310,7 +313,14 @@ export function processItem( id: QueueItemId ) { } if ( attachment ) { - onChange?.( [ attachment ] ); + // Don't update the block with a HEIC URL — the browser can't + // display it. The scaled JPEG sideload will call onChange + // with a usable URL once the client-side conversion completes. + const isHeicUrl = + attachment.url && /\.hei[cf]$/i.test( attachment.url ); + if ( ! isHeicUrl ) { + onChange?.( [ attachment ] ); + } } /* @@ -671,11 +681,13 @@ export function prepareItem( id: QueueItemId ) { const operations: Operation[] = []; const settings = select.getSettings(); + let heicJpeg: File | null = null; const isImage = file.type.startsWith( 'image/' ); const isVipsSupported = CLIENT_SIDE_SUPPORTED_MIME_TYPES.includes( file.type ); + const isHeic = HEIC_MIME_TYPES.includes( file.type ); // For images that can be processed by vips, check if we need to scale down based on threshold. if ( isImage && isVipsSupported ) { @@ -695,6 +707,36 @@ export function prepareItem( id: QueueItemId ) { } } + operations.push( + OperationType.Upload, + OperationType.ThumbnailGeneration, + OperationType.Finalize + ); + } else if ( isImage && isHeic ) { + // HEIC/HEIF: convert to JPEG client-side before upload. + // The server may not support HEIC, so decode it using the + // browser's native HEVC codec (createImageBitmap or VideoDecoder) + // and upload the resulting JPEG. The server then handles it like + // any normal JPEG (threshold scaling, sub-sizes, etc.). + // This matches iOS behavior where HEIC is converted on the fly. + try { + heicJpeg = await canvasConvertToJpeg( + file, + settings.imageQuality ?? DEFAULT_OUTPUT_QUALITY + ); + } catch { + dispatch.cancelItem( + id, + new UploadError( { + code: 'HEIC_DECODE_ERROR', + message: + 'This browser cannot decode HEIC images and the server does not support them either. Please convert to JPEG before uploading.', + file, + } ) + ); + return; + } + operations.push( OperationType.Upload, OperationType.ThumbnailGeneration, @@ -712,16 +754,34 @@ export function prepareItem( id: QueueItemId ) { // If the file is not processed by vips, tell the server to // generate sub-sizes since they won't be created client-side. - const updates = - ! isVipsSupported || ! isImage - ? { - additionalData: { - ...item.additionalData, - generate_sub_sizes: true, - convert_format: true, - }, - } - : {}; + let updates: Partial< QueueItem > = {}; + if ( isHeic && heicJpeg ) { + // HEIC was converted to JPEG client-side. Upload the JPEG + // and let the server handle it normally (threshold scaling, + // sub-sizes, format conversion). Keep the original HEIC in + // a separate field so it can be sideloaded as the "original" + // after upload, preserving the user's file without leaking it + // into paths that expect an editor-supported image. + const vipsAvailable = isClientSideMediaSupported(); + updates = { + file: heicJpeg, + sourceFile: heicJpeg, + originalHeicFile: item.file, + additionalData: { + ...item.additionalData, + generate_sub_sizes: ! vipsAvailable, + convert_format: true, + }, + }; + } else if ( ! isVipsSupported || ! isImage ) { + updates = { + additionalData: { + ...item.additionalData, + generate_sub_sizes: true, + convert_format: true, + }, + }; + } dispatch.finishOperation( id, updates ); }; @@ -1029,46 +1089,69 @@ export function generateThumbnails( id: QueueItemId ) { return; } const attachment = item.attachment; + const settings = select.getSettings(); + + // HEIC/HEIF: preserve the original file under a dedicated metadata + // key so it never collides with `original_image`, which the scaled + // sideload flow owns. The HEIC was kept on item.originalHeicFile; + // the uploaded file is a JPEG conversion. parentId guarantees + // processItem routes this to the sideload endpoint, never the main + // create endpoint. + if ( item.originalHeicFile && attachment.id ) { + dispatch.addSideloadItem( { + file: item.originalHeicFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original-heic', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } // Check if image needs rotation. // If exif_orientation is not 1, the image needs rotation. // Images that were scaled (bigImageSizeThreshold) are already rotated by vips. - const needsRotation = - attachment.exif_orientation && - attachment.exif_orientation !== 1 && - ! item.file.name.includes( '-scaled' ); - - // If rotation is needed for a non-scaled image, sideload the rotated version. - // This matches WordPress core's behavior of creating a -rotated version. - if ( needsRotation && attachment.id ) { - try { - const rotatedFile = await vipsRotateImage( - item.id, - item.sourceFile, - attachment.exif_orientation as number, - item.abortController?.signal - ); + { + const needsRotation = + attachment.exif_orientation && + attachment.exif_orientation !== 1 && + ! item.file.name.includes( '-scaled' ); + + // If rotation is needed for a non-scaled image, sideload the rotated version. + // This matches WordPress core's behavior of creating a -rotated version. + if ( needsRotation && attachment.id ) { + try { + const rotatedFile = await vipsRotateImage( + item.id, + item.sourceFile, + attachment.exif_orientation as number, + item.abortController?.signal + ); - // Sideload the rotated file as the "original" to set original_image metadata. - // The server will store this in $metadata['original_image']. - dispatch.addSideloadItem( { - file: rotatedFile, - batchId: uuidv4(), - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'original', - convert_format: false, - }, - operations: [ OperationType.Upload ], - } ); - } catch { - // If rotation fails, continue with thumbnail generation. - // Thumbnails will still be rotated correctly by vips. - // eslint-disable-next-line no-console - console.warn( - 'Failed to rotate image, continuing with thumbnails' - ); + // Sideload the rotated file as the "original" to set original_image metadata. + // The server will store this in $metadata['original_image']. + dispatch.addSideloadItem( { + file: rotatedFile, + batchId: uuidv4(), + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'original', + convert_format: false, + }, + operations: [ OperationType.Upload ], + } ); + } catch { + // If rotation fails, continue with thumbnail generation. + // Thumbnails will still be rotated correctly by vips. + // eslint-disable-next-line no-console + console.warn( + 'Failed to rotate image, continuing with thumbnails' + ); + } } } @@ -1078,17 +1161,14 @@ export function generateThumbnails( id: QueueItemId ) { attachment.missing_image_sizes && attachment.missing_image_sizes.length > 0 ) { - const settings = select.getSettings(); const allImageSizes = settings.allImageSizes || {}; const sizesToGenerate: string[] = attachment.missing_image_sizes as string[]; - // Use sourceFile for thumbnail generation to preserve quality. - // WordPress core generates thumbnails from the original (unscaled) image. - // Vips will auto-rotate based on EXIF orientation during thumbnail generation. + const thumbnailSource = item.sourceFile; const file = attachment.filename - ? renameFile( item.sourceFile, attachment.filename ) - : item.sourceFile; + ? renameFile( thumbnailSource, attachment.filename ) + : thumbnailSource; const batchId = uuidv4(); const { imageOutputFormats } = settings; @@ -1096,7 +1176,7 @@ export function generateThumbnails( id: QueueItemId ) { // Check if thumbnails should be transcoded to a different format. // Uses the same transparency-aware logic as the main image // to avoid converting transparent PNGs to JPEG. - const sourceType = item.sourceFile.type; + const sourceType = thumbnailSource.type; const outputMimeType = imageOutputFormats?.[ sourceType ]; let thumbnailTranscodeOperation: @@ -1108,7 +1188,7 @@ export function generateThumbnails( id: QueueItemId ) { if ( outputMimeType && outputMimeType !== sourceType ) { thumbnailTranscodeOperation = await getTranscodeImageOperation( - item.sourceFile, + thumbnailSource, outputMimeType, settings ); @@ -1152,56 +1232,60 @@ export function generateThumbnails( id: QueueItemId ) { } ); } - // Create and sideload the scaled version. - const { bigImageSizeThreshold } = settings; - if ( bigImageSizeThreshold && attachment.id ) { - // Check if the image actually exceeds the threshold. - // Only create a scaled version for images larger than the threshold, - // matching WordPress core's wp_create_image_subsizes() behavior. - const bitmap = await createImageBitmap( item.sourceFile ); - const needsScaling = - bitmap.width > bigImageSizeThreshold || - bitmap.height > bigImageSizeThreshold; - bitmap.close(); - - if ( needsScaling ) { - // Rename sourceFile to match the server attachment filename. - const sourceForScaled = attachment.filename - ? renameFile( item.sourceFile, attachment.filename ) - : item.sourceFile; - - // Add scaling to queue. - const scaledOperations: Operation[] = [ - [ - OperationType.ResizeCrop, - { - resize: { - width: bigImageSizeThreshold, - height: bigImageSizeThreshold, + // Create and sideload the scaled version if it exceeds the threshold. + { + const { bigImageSizeThreshold } = settings; + if ( bigImageSizeThreshold && attachment.id ) { + // Check if the image actually exceeds the threshold. + // Only create a scaled version for images larger than the threshold, + // matching WordPress core's wp_create_image_subsizes() behavior. + const bitmap = await createImageBitmap( thumbnailSource ); + const needsScaling = + bitmap.width > bigImageSizeThreshold || + bitmap.height > bigImageSizeThreshold; + bitmap.close(); + + if ( needsScaling ) { + // Rename sourceFile to match the server attachment filename. + const sourceForScaled = attachment.filename + ? renameFile( thumbnailSource, attachment.filename ) + : thumbnailSource; + + // Add scaling to queue. + const scaledOperations: Operation[] = [ + [ + OperationType.ResizeCrop, + { + resize: { + width: bigImageSizeThreshold, + height: bigImageSizeThreshold, + }, + isThresholdResize: true, }, - isThresholdResize: true, + ], + ]; + + // Add transcoding if format conversion is configured. + if ( thumbnailTranscodeOperation ) { + scaledOperations.push( + thumbnailTranscodeOperation + ); + } + + scaledOperations.push( OperationType.Upload ); + + dispatch.addSideloadItem( { + file: sourceForScaled, + batchId, + parentId: item.id, + additionalData: { + post: attachment.id, + image_size: 'scaled', + convert_format: false, }, - ], - ]; - - // Add transcoding if format conversion is configured. - if ( thumbnailTranscodeOperation ) { - scaledOperations.push( thumbnailTranscodeOperation ); + operations: scaledOperations, + } ); } - - scaledOperations.push( OperationType.Upload ); - - dispatch.addSideloadItem( { - file: sourceForScaled, - batchId, - parentId: item.id, - additionalData: { - post: attachment.id, - image_size: 'scaled', - convert_format: false, - }, - operations: scaledOperations, - } ); } } } diff --git a/packages/upload-media/src/store/types.ts b/packages/upload-media/src/store/types.ts index 3c977e13a9ba93..d87ed02b91a30f 100644 --- a/packages/upload-media/src/store/types.ts +++ b/packages/upload-media/src/store/types.ts @@ -24,6 +24,10 @@ export interface QueueItem { id: QueueItemId; sourceFile: File; file: File; + // Original HEIC/HEIF file, kept separately so it can be sideloaded + // as the attachment's "original_image" after the converted JPEG is + // uploaded. Not set for non-HEIC items. + originalHeicFile?: File; poster?: File; attachment?: Partial< Attachment >; status: ItemStatus; diff --git a/packages/upload-media/src/test/canvas-utils.ts b/packages/upload-media/src/test/canvas-utils.ts new file mode 100644 index 00000000000000..8eb3c5a64d49d5 --- /dev/null +++ b/packages/upload-media/src/test/canvas-utils.ts @@ -0,0 +1,198 @@ +/** + * Internal dependencies + */ +import { canvasConvertToJpeg } from '../canvas-utils'; + +describe( 'canvasConvertToJpeg', () => { + const originalCreateImageBitmap = global.createImageBitmap; + const originalOffscreenCanvas = global.OffscreenCanvas; + const originalImageDecoder = ( global as any ).ImageDecoder; + const originalVideoDecoder = ( global as any ).VideoDecoder; + + afterEach( () => { + // Restore all globals. + if ( originalCreateImageBitmap ) { + global.createImageBitmap = originalCreateImageBitmap; + } else { + // @ts-ignore + delete global.createImageBitmap; + } + if ( originalOffscreenCanvas ) { + global.OffscreenCanvas = originalOffscreenCanvas; + } else { + // @ts-ignore + delete global.OffscreenCanvas; + } + if ( originalImageDecoder ) { + ( global as any ).ImageDecoder = originalImageDecoder; + } else { + delete ( global as any ).ImageDecoder; + } + if ( originalVideoDecoder ) { + ( global as any ).VideoDecoder = originalVideoDecoder; + } else { + delete ( global as any ).VideoDecoder; + } + } ); + + describe( 'Strategy 1: createImageBitmap + OffscreenCanvas', () => { + it( 'should convert via createImageBitmap when available', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + + const mockBitmap = { + width: 200, + height: 150, + close: jest.fn(), + }; + + const mockCtx = { + drawImage: jest.fn(), + }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest.fn().mockReturnValue( mockCtx ), + convertToBlob: jest.fn().mockResolvedValue( jpegBlob ), + } ) ); + + const file = new File( [ 'heic-data' ], 'photo.heic', { + type: 'image/heic', + } ); + const result = await canvasConvertToJpeg( file ); + + expect( result ).toBeInstanceOf( File ); + expect( result.name ).toBe( 'photo.jpeg' ); + expect( result.type ).toBe( 'image/jpeg' ); + expect( mockBitmap.close ).toHaveBeenCalled(); + expect( global.createImageBitmap ).toHaveBeenCalledWith( file ); + } ); + + it( 'should use the specified quality', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + + const mockConvertToBlob = jest.fn().mockResolvedValue( jpegBlob ); + const mockBitmap = { width: 100, height: 100, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest + .fn() + .mockReturnValue( { drawImage: jest.fn() } ), + convertToBlob: mockConvertToBlob, + } ) ); + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + await canvasConvertToJpeg( file, 0.5 ); + + expect( mockConvertToBlob ).toHaveBeenCalledWith( { + type: 'image/jpeg', + quality: 0.5, + } ); + } ); + + it( 'should strip the extension and use .jpeg', async () => { + const jpegBlob = new Blob( [ 'jpeg-data' ], { + type: 'image/jpeg', + } ); + const mockBitmap = { width: 10, height: 10, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest + .fn() + .mockReturnValue( { drawImage: jest.fn() } ), + convertToBlob: jest.fn().mockResolvedValue( jpegBlob ), + } ) ); + + const file = new File( [ 'data' ], 'my-photo.HEIC', { + type: 'image/heic', + } ); + const result = await canvasConvertToJpeg( file ); + expect( result.name ).toBe( 'my-photo.jpeg' ); + } ); + + it( 'should close the bitmap even if canvas context fails', async () => { + const mockBitmap = { width: 10, height: 10, close: jest.fn() }; + + global.createImageBitmap = jest + .fn() + .mockResolvedValue( mockBitmap ); + global.OffscreenCanvas = jest.fn().mockImplementation( () => ( { + getContext: jest.fn().mockReturnValue( null ), + convertToBlob: jest.fn(), + } ) ); + + // Remove other decoders so it falls through to the final error. + delete ( global as any ).ImageDecoder; + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + expect( mockBitmap.close ).toHaveBeenCalled(); + } ); + } ); + + describe( 'fallback behavior', () => { + it( 'should throw when no strategy is available', async () => { + // createImageBitmap throws (doesn't support HEIC). + global.createImageBitmap = jest + .fn() + .mockRejectedValue( new Error( 'Unsupported format' ) ); + // No ImageDecoder or VideoDecoder. + delete ( global as any ).ImageDecoder; + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + } ); + + it( 'should fall through Strategy 1 failure to subsequent strategies', async () => { + // Strategy 1 fails. + global.createImageBitmap = jest + .fn() + .mockRejectedValue( new Error( 'Unsupported' ) ); + + // Strategy 2: ImageDecoder not supported for this type. + ( global as any ).ImageDecoder = { + isTypeSupported: jest.fn().mockResolvedValue( false ), + }; + + // No VideoDecoder. + delete ( global as any ).VideoDecoder; + + const file = new File( [ 'data' ], 'photo.heic', { + type: 'image/heic', + } ); + + await expect( canvasConvertToJpeg( file ) ).rejects.toThrow( + 'cannot decode HEIC' + ); + + expect( + ( global as any ).ImageDecoder.isTypeSupported + ).toHaveBeenCalledWith( 'image/heic' ); + } ); + } ); +} ); diff --git a/packages/upload-media/src/test/feature-detection.ts b/packages/upload-media/src/test/feature-detection.ts index b96ced8a50df7c..3c4572892747df 100644 --- a/packages/upload-media/src/test/feature-detection.ts +++ b/packages/upload-media/src/test/feature-detection.ts @@ -4,6 +4,7 @@ import { detectClientSideMediaSupport, isClientSideMediaSupported, + isHeicCanvasSupported, clearFeatureDetectionCache, } from '../feature-detection'; @@ -289,6 +290,67 @@ describe( 'feature-detection', () => { } ); } ); + describe( 'isHeicCanvasSupported', () => { + const originalCreateImageBitmap = + global.createImageBitmap as typeof createImageBitmap; + const originalOffscreenCanvas = + global.OffscreenCanvas as typeof OffscreenCanvas; + + afterEach( () => { + // Restore globals after each test. + if ( originalCreateImageBitmap !== undefined ) { + global.createImageBitmap = + originalCreateImageBitmap as typeof createImageBitmap; + } else { + // @ts-ignore + delete global.createImageBitmap; + } + if ( originalOffscreenCanvas !== undefined ) { + global.OffscreenCanvas = + originalOffscreenCanvas as typeof OffscreenCanvas; + } else { + // @ts-ignore + delete global.OffscreenCanvas; + } + } ); + + it( 'returns true when both createImageBitmap and OffscreenCanvas are available', () => { + global.createImageBitmap = + jest.fn() as unknown as typeof createImageBitmap; + global.OffscreenCanvas = + jest.fn() as unknown as typeof OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( true ); + } ); + + it( 'returns false when createImageBitmap is unavailable', () => { + // @ts-ignore + delete global.createImageBitmap; + global.OffscreenCanvas = + jest.fn() as unknown as typeof OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + + it( 'returns false when OffscreenCanvas is unavailable', () => { + global.createImageBitmap = + jest.fn() as unknown as typeof createImageBitmap; + // @ts-ignore + delete global.OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + + it( 'returns false when both are unavailable', () => { + // @ts-ignore + delete global.createImageBitmap; + // @ts-ignore + delete global.OffscreenCanvas; + + expect( isHeicCanvasSupported() ).toBe( false ); + } ); + } ); + describe( 'clearFeatureDetectionCache', () => { it( 'clears the cached result', () => { const result1 = detectClientSideMediaSupport(); diff --git a/packages/upload-media/src/test/heic-parser.ts b/packages/upload-media/src/test/heic-parser.ts new file mode 100644 index 00000000000000..c7c523d80d61d9 --- /dev/null +++ b/packages/upload-media/src/test/heic-parser.ts @@ -0,0 +1,733 @@ +/* eslint-disable no-bitwise, jsdoc/require-param */ + +/** + * Internal dependencies + */ +import { parseHeic, reverseBits32 } from '../heic-parser'; + +// --------------------------------------------------------------------------- +// Helpers for constructing synthetic ISOBMFF structures +// --------------------------------------------------------------------------- + +/** Write a big-endian uint32 into a DataView. */ +function writeU32( view: DataView, offset: number, value: number ) { + view.setUint32( offset, value ); +} + +/** Write a big-endian uint16 into a DataView. */ +function writeU16( view: DataView, offset: number, value: number ) { + view.setUint16( offset, value ); +} + +/** Build an ISOBMFF box (size + fourcc + data). */ +function buildBox( type: string, data: Uint8Array ): Uint8Array { + const size = 8 + data.length; + const buf = new Uint8Array( size ); + const view = new DataView( buf.buffer ); + writeU32( view, 0, size ); + for ( let i = 0; i < 4; i++ ) { + buf[ 4 + i ] = type.charCodeAt( i ); + } + buf.set( data, 8 ); + return buf; +} + +/** Build a FullBox (size + fourcc + version + flags + data). */ +function buildFullBox( + type: string, + version: number, + flags: number, + data: Uint8Array +): Uint8Array { + const inner = new Uint8Array( 4 + data.length ); + inner[ 0 ] = version; + inner[ 1 ] = ( flags >> 16 ) & 0xff; + inner[ 2 ] = ( flags >> 8 ) & 0xff; + inner[ 3 ] = flags & 0xff; + inner.set( data, 4 ); + return buildBox( type, inner ); +} + +/** Concatenate multiple Uint8Arrays. */ +function concat( ...arrays: Uint8Array[] ): Uint8Array { + const total = arrays.reduce( ( sum, a ) => sum + a.length, 0 ); + const result = new Uint8Array( total ); + let offset = 0; + for ( const a of arrays ) { + result.set( a, offset ); + offset += a.length; + } + return result; +} + +/** Build a pitm box (Primary Item). */ +function buildPitm( primaryItemId: number ): Uint8Array { + const data = new Uint8Array( 2 ); + const view = new DataView( data.buffer ); + writeU16( view, 0, primaryItemId ); + return buildFullBox( 'pitm', 0, 0, data ); +} + +/** + * Build a minimal hvcC box (HEVCDecoderConfigurationRecord). + * + * Fields: configVersion=1, profileSpace=0, tier=0, profileIdc=1, + * compatFlags=0x60000000, constraintBytes=[0xB0,0,0,0,0,0], + * levelIdc=93, then zeros for remaining fields + 0 NAL arrays. + */ +function buildHvcC(): Uint8Array { + // HEVCDecoderConfigurationRecord (23 bytes minimum with 0 arrays) + const record = new Uint8Array( 23 ); + record[ 0 ] = 1; // configurationVersion + // byte1: profileSpace=0 (bits 6-7), tier=0 (bit 5), profileIdc=1 (bits 0-4) + record[ 1 ] = 0x01; + // general_profile_compatibility_flags = 0x60000000 + record[ 2 ] = 0x60; + record[ 3 ] = 0x00; + record[ 4 ] = 0x00; + record[ 5 ] = 0x00; + // general_constraint_indicator_flags (6 bytes) = [0xB0, 0, 0, 0, 0, 0] + record[ 6 ] = 0xb0; + // bytes 7-11 are zero (remaining constraint bytes) + // general_level_idc = 93 + record[ 12 ] = 93; + // remaining fields: min_spatial_segmentation_idc, parallelismType, + // chromaFormat, bitDepthLuma, bitDepthChroma, avgFrameRate, misc, numOfArrays + // All zero is valid for our test purposes. + return buildBox( 'hvcC', record ); +} + +/** Build an ispe (Image Spatial Extents) box. */ +function buildIspe( width: number, height: number ): Uint8Array { + const data = new Uint8Array( 8 ); + const view = new DataView( data.buffer ); + writeU32( view, 0, width ); + writeU32( view, 4, height ); + return buildFullBox( 'ispe', 0, 0, data ); +} + +/** Build an irot (Image Rotation) box. angle is 0-3 (multiplied by 90°). */ +function buildIrot( angle: number ): Uint8Array { + const data = new Uint8Array( [ angle & 0x3 ] ); + return buildBox( 'irot', data ); +} + +/** Build an ipco (Item Property Container) with the given property boxes. */ +function buildIpco( ...properties: Uint8Array[] ): Uint8Array { + return buildBox( 'ipco', concat( ...properties ) ); +} + +/** + * Build an ipma (Item Property Association) box. + * + * @param associations Array of [itemId, propertyIndices[]] + */ +function buildIpma( associations: Array< [ number, number[] ] > ): Uint8Array { + // Calculate data size: 4 (entry_count) + per entry: 2 (itemId) + 1 (assocCount) + N (indices) + let dataSize = 4; + for ( const [ , indices ] of associations ) { + dataSize += 2 + 1 + indices.length; + } + const data = new Uint8Array( dataSize ); + const view = new DataView( data.buffer ); + writeU32( view, 0, associations.length ); + let pos = 4; + for ( const [ itemId, indices ] of associations ) { + writeU16( view, pos, itemId ); + pos += 2; + data[ pos ] = indices.length; + pos += 1; + for ( const idx of indices ) { + data[ pos ] = idx & 0x7f; // 7-bit index, essential=0 + pos += 1; + } + } + return buildFullBox( 'ipma', 0, 0, data ); +} + +/** Build an iprp box containing ipco + ipma. */ +function buildIprp( ipco: Uint8Array, ipma: Uint8Array ): Uint8Array { + return buildBox( 'iprp', concat( ipco, ipma ) ); +} + +/** + * Build an iloc box for version 0, with 4-byte offsets and 4-byte lengths. + * + * @param items Array of [itemId, [[offset, length], ...]] + */ +function buildIloc( + items: Array< [ number, Array< [ number, number ] > ] > +): Uint8Array { + // version 0: offsetSize=4, lengthSize=4, baseOffsetSize=0 + // per item: 2 (itemId) + 2 (data_reference_index) + 0 (base_offset) + 2 (extent_count) + N*(4+4) + let dataSize = 2 + 2; // sizes byte + item_count + for ( const [ , extents ] of items ) { + dataSize += 2 + 2 + 2 + extents.length * 8; + } + const data = new Uint8Array( dataSize ); + const view = new DataView( data.buffer ); + // offset_size=4 (upper nibble), length_size=4 (lower nibble) + data[ 0 ] = 0x44; + // base_offset_size=0 (upper nibble), reserved=0 + data[ 1 ] = 0x00; + // item_count + writeU16( view, 2, items.length ); + let pos = 4; + for ( const [ itemId, extents ] of items ) { + writeU16( view, pos, itemId ); + pos += 2; + // data_reference_index = 0 + writeU16( view, pos, 0 ); + pos += 2; + // no base_offset (size=0) + // extent_count + writeU16( view, pos, extents.length ); + pos += 2; + for ( const [ offset, length ] of extents ) { + writeU32( view, pos, offset ); + pos += 4; + writeU32( view, pos, length ); + pos += 4; + } + } + return buildFullBox( 'iloc', 0, 0, data ); +} + +/** Build an hdlr (Handler) box with handler_type='pict'. */ +function buildHdlr(): Uint8Array { + // Minimal hdlr: 4 bytes pre_defined + 4 bytes handler_type + 12 bytes reserved + 1 byte name (null) + const data = new Uint8Array( 21 ); + // handler_type = 'pict' at offset 4 + data[ 4 ] = 0x70; // p + data[ 5 ] = 0x69; // i + data[ 6 ] = 0x63; // c + data[ 7 ] = 0x74; // t + return buildFullBox( 'hdlr', 0, 0, data ); +} + +/** + * Build a minimal single-image HEIC file as an ArrayBuffer. + * + * The image data is fake (not decodable) but the container structure + * is valid for testing the parser. + */ +function buildSingleImageHeic( { + width = 100, + height = 80, + imageData = new Uint8Array( [ 0xde, 0xad, 0xbe, 0xef ] ), + rotation, +}: { + width?: number; + height?: number; + imageData?: Uint8Array; + rotation?: number; +} = {} ): ArrayBuffer { + const primaryItemId = 1; + + // Build property boxes (1-indexed: 1=ispe, 2=hvcC, optionally 3=irot) + const ispe = buildIspe( width, height ); + const hvcC = buildHvcC(); + const propBoxes: Uint8Array[] = [ ispe, hvcC ]; + const propIndices = [ 1, 2 ]; + if ( rotation !== undefined ) { + propBoxes.push( buildIrot( rotation / 90 ) ); + propIndices.push( 3 ); + } + const ipco = buildIpco( ...propBoxes ); + const ipma = buildIpma( [ [ primaryItemId, propIndices ] ] ); + const iprp = buildIprp( ipco, ipma ); + + // We need to know where mdat data will be placed. + // Build everything except mdat first to calculate the offset. + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); // brand='heic' + const pitm = buildPitm( primaryItemId ); + // iloc will reference the image data at an absolute file offset. + // We'll calculate the actual offset after constructing the meta box. + // Use a placeholder first, then fix it up. + + // Build meta children (without iloc - we'll add it after calculating offset) + const hdlr = buildHdlr(); + const metaChildrenWithoutIloc = concat( hdlr, pitm, iprp ); + + // Calculate sizes to determine mdat data offset: + // ftyp + meta box header (8 + 4 fullbox) + metaChildren + iloc + mdat header (8) + // iloc size depends on items, so build a placeholder iloc to get its size. + const placeholderIloc = buildIloc( [ + [ primaryItemId, [ [ 0, imageData.length ] ] ], + ] ); + const metaSize = + 8 + 4 + metaChildrenWithoutIloc.length + placeholderIloc.length; + const mdatDataOffset = ftyp.length + metaSize + 8; // +8 for mdat box header + + // Now build the real iloc with the correct offset + const iloc = buildIloc( [ + [ primaryItemId, [ [ mdatDataOffset, imageData.length ] ] ], + ] ); + + // Build meta box (FullBox) + const metaChildren = concat( hdlr, pitm, iloc, iprp ); + const meta = buildFullBox( 'meta', 0, 0, metaChildren ); + + // Build mdat + const mdat = buildBox( 'mdat', imageData ); + + // Assemble full file + const file = concat( ftyp, meta, mdat ); + return file.buffer; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe( 'heic-parser', () => { + describe( 'reverseBits32', () => { + it( 'should reverse bits of 0x60000000 to 0x00000006', () => { + expect( reverseBits32( 0x60000000 ) ).toBe( 0x00000006 ); + } ); + + it( 'should reverse bits of 0 to 0', () => { + expect( reverseBits32( 0 ) ).toBe( 0 ); + } ); + + it( 'should reverse bits of 0xFFFFFFFF to 0xFFFFFFFF', () => { + expect( reverseBits32( 0xffffffff ) ).toBe( 0xffffffff ); + } ); + + it( 'should reverse bits of 1 to 0x80000000', () => { + expect( reverseBits32( 1 ) ).toBe( 0x80000000 ); + } ); + + it( 'should reverse bits of 0x80000000 to 1', () => { + expect( reverseBits32( 0x80000000 ) ).toBe( 1 ); + } ); + } ); + + describe( 'parseHeic – single image', () => { + it( 'should parse a minimal single-image HEIC', () => { + const imageData = new Uint8Array( [ 1, 2, 3, 4, 5, 6 ] ); + const buffer = buildSingleImageHeic( { + width: 200, + height: 150, + imageData, + } ); + + const result = parseHeic( buffer ); + + expect( result.outputWidth ).toBe( 200 ); + expect( result.outputHeight ).toBe( 150 ); + expect( result.tileWidth ).toBe( 200 ); + expect( result.tileHeight ).toBe( 150 ); + expect( result.tiles ).toHaveLength( 1 ); + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 0 ].y ).toBe( 0 ); + expect( result.tiles[ 0 ].data ).toEqual( imageData ); + } ); + + it( 'should build correct codec string for Main Profile L3.1', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + // profileIdc=1, compatFlags=0x60000000→reversed=6, + // tier=L, level=93, constraints=B0 + expect( result.codecString ).toBe( 'hvc1.1.6.L93.B0' ); + } ); + + it( 'should extract the HEVCDecoderConfigurationRecord', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + expect( result.description ).toBeInstanceOf( Uint8Array ); + expect( result.description.length ).toBe( 23 ); // minimal hvcC record + expect( result.description[ 0 ] ).toBe( 1 ); // configurationVersion + } ); + + it( 'should return rotation 0 when no irot box present', () => { + const buffer = buildSingleImageHeic(); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 0 ); + } ); + + it( 'should parse 90° CCW rotation from irot box', () => { + const buffer = buildSingleImageHeic( { rotation: 90 } ); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 90 ); + } ); + + it( 'should parse 270° CCW rotation from irot box', () => { + const buffer = buildSingleImageHeic( { rotation: 270 } ); + const result = parseHeic( buffer ); + + expect( result.rotation ).toBe( 270 ); + } ); + } ); + + describe( 'parseHeic – grid/tiled image', () => { + /** + * Build a grid HEIC file with multiple tiles (like iPhone photos). + * + * The structure is: ftyp, meta (hdlr, pitm, iinf, iref, iloc, iprp), mdat. + * The primary item is type 'grid' referencing tile items of type 'hvc1'. + */ + function buildGridHeic( { + rows = 2, + columns = 2, + tileWidth = 512, + tileHeight = 512, + outputWidth = 1024, + outputHeight = 1024, + tileData = new Uint8Array( [ 0xca, 0xfe ] ), + }: { + rows?: number; + columns?: number; + tileWidth?: number; + tileHeight?: number; + outputWidth?: number; + outputHeight?: number; + tileData?: Uint8Array; + } = {} ): ArrayBuffer { + const gridItemId = 1; + const tileCount = rows * columns; + // Tile item IDs start at 2. + const tileItemIds = Array.from( + { length: tileCount }, + ( _, i ) => i + 2 + ); + + // Grid descriptor: version(1), flags(1), rows_minus_one(1), columns_minus_one(1), + // output_width(2), output_height(2) = 8 bytes (small fields). + const gridDescriptor = new Uint8Array( 8 ); + gridDescriptor[ 0 ] = 0; // version + gridDescriptor[ 1 ] = 0; // flags (no large fields) + gridDescriptor[ 2 ] = rows - 1; + gridDescriptor[ 3 ] = columns - 1; + const gdv = new DataView( gridDescriptor.buffer ); + gdv.setUint16( 4, outputWidth ); + gdv.setUint16( 6, outputHeight ); + + // Build property boxes (1=hvcC, 2=ispe) + const hvcC = buildHvcC(); + const ispe = buildIspe( tileWidth, tileHeight ); + const ipco = buildIpco( hvcC, ispe ); + + // ipma: each tile item → [1, 2] (hvcC at index 1, ispe at index 2) + const associations: Array< [ number, number[] ] > = tileItemIds.map( + ( id ) => [ id, [ 1, 2 ] ] as [ number, number[] ] + ); + const ipma = buildIpma( associations ); + const iprp = buildIprp( ipco, ipma ); + + // iinf: grid item type 'grid', tile items type 'hvc1'. + const iinf = buildIinf( [ + [ gridItemId, 'grid' ], + ...tileItemIds.map( + ( id ) => [ id, 'hvc1' ] as [ number, string ] + ), + ] ); + + // iref: dimg reference from grid to tiles. + const iref = buildIref( gridItemId, tileItemIds ); + + // Build everything except iloc and mdat first to calculate offsets. + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); + const hdlr = buildHdlr(); + const pitm = buildPitm( gridItemId ); + + // We need iloc for grid item + all tile items. + // First, build a placeholder iloc to calculate meta size. + const placeholderItems: Array< + [ number, Array< [ number, number ] > ] + > = [ + [ gridItemId, [ [ 0, gridDescriptor.length ] ] ], + ...tileItemIds.map( + ( id ) => + [ id, [ [ 0, tileData.length ] ] ] as [ + number, + Array< [ number, number ] >, + ] + ), + ]; + const placeholderIloc = buildIloc( placeholderItems ); + + const metaChildren = concat( + hdlr, + pitm, + iinf, + iref, + placeholderIloc, + iprp + ); + const metaSize = 8 + 4 + metaChildren.length; // box header + fullbox + const mdatHeaderSize = 8; + const mdatDataStart = ftyp.length + metaSize + mdatHeaderSize; + + // Grid descriptor is at the start of mdat, tiles follow. + const gridOffset = mdatDataStart; + let currentOffset = gridOffset + gridDescriptor.length; + const tileOffsets: number[] = []; + for ( let i = 0; i < tileCount; i++ ) { + tileOffsets.push( currentOffset ); + currentOffset += tileData.length; + } + + // Build real iloc. + const realItems: Array< [ number, Array< [ number, number ] > ] > = + [ + [ gridItemId, [ [ gridOffset, gridDescriptor.length ] ] ], + ...tileItemIds.map( + ( id, i ) => + [ + id, + [ [ tileOffsets[ i ], tileData.length ] ], + ] as [ number, Array< [ number, number ] > ] + ), + ]; + const iloc = buildIloc( realItems ); + + const realMetaChildren = concat( + hdlr, + pitm, + iinf, + iref, + iloc, + iprp + ); + const meta = buildFullBox( 'meta', 0, 0, realMetaChildren ); + + // Build mdat: grid descriptor + tile data. + const mdatPayload = concat( + gridDescriptor, + ...Array( tileCount ).fill( tileData ) + ); + const mdat = buildBox( 'mdat', mdatPayload ); + + return concat( ftyp, meta, mdat ).buffer; + } + + /** Build an iinf box with infe entries. */ + function buildIinf( items: Array< [ number, string ] > ): Uint8Array { + // Each infe: version=2, flags=0, itemId(u16), protection_index(u16), item_type(4 bytes) + const infeBoxes = items.map( ( [ itemId, itemType ] ) => { + const infeData = new Uint8Array( 8 ); + const idv = new DataView( infeData.buffer ); + idv.setUint16( 0, itemId ); + idv.setUint16( 2, 0 ); // protection index + for ( let k = 0; k < 4; k++ ) { + infeData[ 4 + k ] = itemType.charCodeAt( k ); + } + return buildFullBox( 'infe', 2, 0, infeData ); + } ); + + // iinf: version=0, entry_count(u16), then infe boxes. + const countData = new Uint8Array( 2 ); + new DataView( countData.buffer ).setUint16( 0, items.length ); + return buildFullBox( + 'iinf', + 0, + 0, + concat( countData, ...infeBoxes ) + ); + } + + /** Build an iref box with a single dimg reference. */ + function buildIref( fromId: number, toIds: number[] ): Uint8Array { + // dimg reference box: fromId(u16), refCount(u16), toIds(u16 each) + const dimgData = new Uint8Array( 4 + toIds.length * 2 ); + const dv = new DataView( dimgData.buffer ); + dv.setUint16( 0, fromId ); + dv.setUint16( 2, toIds.length ); + for ( let i = 0; i < toIds.length; i++ ) { + dv.setUint16( 4 + i * 2, toIds[ i ] ); + } + const dimgBox = buildBox( 'dimg', dimgData ); + + // iref is a FullBox (version=0). + return buildFullBox( 'iref', 0, 0, dimgBox ); + } + + it( 'should parse a 2x2 grid HEIC', () => { + const tileData = new Uint8Array( [ 0x11, 0x22, 0x33 ] ); + const buffer = buildGridHeic( { + rows: 2, + columns: 2, + tileWidth: 512, + tileHeight: 512, + outputWidth: 1024, + outputHeight: 1024, + tileData, + } ); + + const result = parseHeic( buffer ); + + expect( result.outputWidth ).toBe( 1024 ); + expect( result.outputHeight ).toBe( 1024 ); + expect( result.tileWidth ).toBe( 512 ); + expect( result.tileHeight ).toBe( 512 ); + expect( result.tiles ).toHaveLength( 4 ); + } ); + + it( 'should place tiles in correct grid positions', () => { + const buffer = buildGridHeic( { + rows: 2, + columns: 2, + tileWidth: 256, + tileHeight: 256, + } ); + + const result = parseHeic( buffer ); + + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 0 ].y ).toBe( 0 ); + expect( result.tiles[ 1 ].x ).toBe( 256 ); + expect( result.tiles[ 1 ].y ).toBe( 0 ); + expect( result.tiles[ 2 ].x ).toBe( 0 ); + expect( result.tiles[ 2 ].y ).toBe( 256 ); + expect( result.tiles[ 3 ].x ).toBe( 256 ); + expect( result.tiles[ 3 ].y ).toBe( 256 ); + } ); + + it( 'should extract tile data for each tile', () => { + const tileData = new Uint8Array( [ 0xaa, 0xbb ] ); + const buffer = buildGridHeic( { tileData } ); + + const result = parseHeic( buffer ); + + for ( const tile of result.tiles ) { + expect( tile.data ).toEqual( tileData ); + } + } ); + + it( 'should extract codec string from grid tiles', () => { + const buffer = buildGridHeic(); + const result = parseHeic( buffer ); + + // Same hvcC as single image tests. + expect( result.codecString ).toBe( 'hvc1.1.6.L93.B0' ); + } ); + + it( 'should parse a 1x3 grid', () => { + const buffer = buildGridHeic( { + rows: 1, + columns: 3, + tileWidth: 100, + tileHeight: 300, + outputWidth: 300, + outputHeight: 300, + } ); + + const result = parseHeic( buffer ); + + expect( result.tiles ).toHaveLength( 3 ); + expect( result.tiles[ 0 ].x ).toBe( 0 ); + expect( result.tiles[ 1 ].x ).toBe( 100 ); + expect( result.tiles[ 2 ].x ).toBe( 200 ); + expect( result.outputWidth ).toBe( 300 ); + expect( result.outputHeight ).toBe( 300 ); + } ); + + it( 'should extract the HEVCDecoderConfigurationRecord from grid', () => { + const buffer = buildGridHeic(); + const result = parseHeic( buffer ); + + expect( result.description ).toBeInstanceOf( Uint8Array ); + expect( result.description.length ).toBe( 23 ); + expect( result.description[ 0 ] ).toBe( 1 ); // configurationVersion + } ); + } ); + + describe( 'parseHeic – error cases', () => { + it( 'should throw for empty buffer', () => { + expect( () => parseHeic( new ArrayBuffer( 0 ) ) ).toThrow( + 'No meta box found' + ); + } ); + + it( 'should throw for buffer without meta box', () => { + const ftyp = buildBox( + 'ftyp', + new Uint8Array( [ 0x68, 0x65, 0x69, 0x63 ] ) + ); + expect( () => parseHeic( ftyp.buffer ) ).toThrow( + 'No meta box found' + ); + } ); + + it( 'should throw when required boxes are missing', () => { + // meta box with only hdlr (no pitm, iloc, iprp) + const hdlr = buildHdlr(); + const meta = buildFullBox( 'meta', 0, 0, hdlr ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'Missing required boxes' + ); + } ); + + it( 'should throw for missing ipco or ipma inside iprp', () => { + // meta with pitm, iloc, iprp but iprp is empty (no ipco/ipma) + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 1, [ [ 0, 4 ] ] ] ] ); + const emptyIprp = buildBox( 'iprp', new Uint8Array( 0 ) ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, emptyIprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'Missing ipco or ipma' + ); + } ); + + it( 'should throw when primary item has no location data', () => { + // iloc with item ID 99, but pitm says primary is 1 + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 99, [ [ 0, 4 ] ] ] ] ); + const hvcC = buildHvcC(); + const ispe = buildIspe( 100, 100 ); + const ipco = buildIpco( hvcC, ispe ); + const ipma = buildIpma( [ [ 1, [ 1, 2 ] ] ] ); + const iprp = buildIprp( ipco, ipma ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, iprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'No location data for primary item' + ); + } ); + + it( 'should throw when primary item has no property associations', () => { + const hdlr = buildHdlr(); + const pitm = buildPitm( 1 ); + const iloc = buildIloc( [ [ 1, [ [ 0, 4 ] ] ] ] ); + const hvcC = buildHvcC(); + const ispe = buildIspe( 100, 100 ); + const ipco = buildIpco( hvcC, ispe ); + // ipma associates item 99, not item 1 + const ipma = buildIpma( [ [ 99, [ 1, 2 ] ] ] ); + const iprp = buildIprp( ipco, ipma ); + const meta = buildFullBox( + 'meta', + 0, + 0, + concat( hdlr, pitm, iloc, iprp ) + ); + expect( () => parseHeic( meta.buffer ) ).toThrow( + 'No property associations' + ); + } ); + } ); +} ); + +/* eslint-enable no-bitwise, jsdoc/require-param */ diff --git a/packages/upload-media/src/test/rename-and-clone-file.ts b/packages/upload-media/src/test/rename-and-clone-file.ts new file mode 100644 index 00000000000000..65e86e40d67765 --- /dev/null +++ b/packages/upload-media/src/test/rename-and-clone-file.ts @@ -0,0 +1,87 @@ +/** + * Internal dependencies + */ +import { renameFile, cloneFile } from '../utils'; + +describe( 'renameFile', () => { + it( 'should return a new File with the given name', () => { + const original = new File( [ 'content' ], 'old-name.jpg', { + type: 'image/jpeg', + lastModified: 1700000000000, + } ); + + const result = renameFile( original, 'new-name.png' ); + + expect( result ).toBeInstanceOf( File ); + expect( result.name ).toBe( 'new-name.png' ); + expect( result ).not.toBe( original ); + } ); + + it( 'should preserve the original file type', () => { + const original = new File( [ 'content' ], 'photo.heic', { + type: 'image/heic', + } ); + + const result = renameFile( original, 'photo.jpeg' ); + + expect( result.type ).toBe( 'image/heic' ); + } ); + + it( 'should preserve lastModified timestamp', () => { + const timestamp = 1600000000000; + const original = new File( [ 'content' ], 'file.txt', { + type: 'text/plain', + lastModified: timestamp, + } ); + + const result = renameFile( original, 'renamed.txt' ); + + expect( result.lastModified ).toBe( timestamp ); + } ); + + it( 'should preserve file size', () => { + const original = new File( [ 'hello world' ], 'file.txt', { + type: 'text/plain', + } ); + + const result = renameFile( original, 'new.txt' ); + + expect( result.size ).toBe( original.size ); + } ); +} ); + +describe( 'cloneFile', () => { + it( 'should return a new File with the same name', () => { + const original = new File( [ 'content' ], 'photo.jpg', { + type: 'image/jpeg', + } ); + + const result = cloneFile( original ); + + expect( result ).toBeInstanceOf( File ); + expect( result ).not.toBe( original ); + expect( result.name ).toBe( 'photo.jpg' ); + } ); + + it( 'should preserve type and lastModified', () => { + const original = new File( [ 'content' ], 'doc.pdf', { + type: 'application/pdf', + lastModified: 1500000000000, + } ); + + const result = cloneFile( original ); + + expect( result.type ).toBe( 'application/pdf' ); + expect( result.lastModified ).toBe( 1500000000000 ); + } ); + + it( 'should preserve file size', () => { + const original = new File( [ 'test data' ], 'test.bin', { + type: 'application/octet-stream', + } ); + + const result = cloneFile( original ); + + expect( result.size ).toBe( original.size ); + } ); +} );