feat(video): add removeBackground prop for chroma key compositing

src/ai-sdk/providers/editly/layers.ts

@@ -72,6 +72,13 @@ export function getVideoFilter(
     filters.push(
       `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`,
     );
+    if (layer.chromaKey) {
+      const color = layer.chromaKey.color ?? "0x00FF00";
+      const similarity = layer.chromaKey.similarity ?? 0.1;
+      const blend = layer.chromaKey.blend ?? 0.05;
+      filters.push(`colorkey=${color}:${similarity}:${blend}`);
+      filters.push("format=yuva420p");
+    }
     filters.push("setsar=1");
     filters.push("fps=30");
     filters.push("settb=1/30");
@@ -124,6 +131,13 @@ export function getVideoFilter(
 
   filters.push(scaleFilter);
   filters.push(`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`);
+  if (layer.chromaKey) {
+    const color = layer.chromaKey.color ?? "0x00FF00";
+    const similarity = layer.chromaKey.similarity ?? 0.1;
+    const blend = layer.chromaKey.blend ?? 0.05;
+    filters.push(`colorkey=${color}:${similarity}:${blend}`);
+    filters.push("format=yuva420p");
+  }
   filters.push("setsar=1");
   filters.push("fps=30");
   filters.push("settb=1/30");
@@ -164,6 +178,13 @@ export function getVideoFilterWithTrim(
     filters.push(
       `scale=${layerWidth}:${layerHeight}:force_original_aspect_ratio=decrease`,
     );
+    if (layer.chromaKey) {
+      const color = layer.chromaKey.color ?? "0x00FF00";
+      const similarity = layer.chromaKey.similarity ?? 0.1;
+      const blend = layer.chromaKey.blend ?? 0.05;
+      filters.push(`colorkey=${color}:${similarity}:${blend}`);
+      filters.push("format=yuva420p");
+    }
     filters.push("setsar=1");
     filters.push("fps=30");
     filters.push("settb=1/30");
@@ -177,6 +198,13 @@ export function getVideoFilterWithTrim(
 
     filters.push(scaleFilter);
     filters.push(`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2`);
+    if (layer.chromaKey) {
+      const color = layer.chromaKey.color ?? "0x00FF00";
+      const similarity = layer.chromaKey.similarity ?? 0.1;
+      const blend = layer.chromaKey.blend ?? 0.05;
+      filters.push(`colorkey=${color}:${similarity}:${blend}`);
+      filters.push("format=yuva420p");
+    }
     filters.push("setsar=1");
     filters.push("fps=30");
     filters.push("settb=1/30");

src/ai-sdk/providers/editly/types.ts

@@ -91,6 +91,12 @@ export interface TextLayer extends BaseLayer {
  * For video layers, if parent `clip.duration` is specified, the video will be slowed/sped-up to match `clip.duration`.
  * If `cutFrom`/`cutTo` is set, the resulting segment (`cutTo`-`cutFrom`) will be slowed/sped-up to fit `clip.duration`.
  */
+export interface ChromaKeyOptions {
+  color?: string;
+  similarity?: number;
+  blend?: number;
+}
+
 export interface VideoLayer extends BaseLayer {
   type: "video";
   path: string;
@@ -104,6 +110,7 @@ export interface VideoLayer extends BaseLayer {
   originX?: OriginX;
   originY?: OriginY;
   mixVolume?: number | string;
+  chromaKey?: ChromaKeyOptions;
 }
 
 /**

src/react/examples/remove-background-demo.tsx

@@ -0,0 +1,76 @@
+import { fal } from "../../ai-sdk/providers/fal";
+import { Clip, Image, Overlay, Render, render, Video } from "..";
+
+// background: construction worker
+const constructionWorker = Image({
+  prompt: "construction worker in hard hat and orange vest, working on building site, holding tools, industrial background, realistic photo",
+  model: fal.imageModel("flux-schnell"),
+  aspectRatio: "9:16",
+});
+
+// foreground: influencer doing makeup (will be on green screen)
+const influencer = Image({
+  prompt: "young woman instagram influencer doing makeup tutorial, holding makeup brush, looking at camera, beauty vlogger style, portrait. IMPORTANT: replace background with solid #00FF00 green screen background for chroma key compositing",
+  model: fal.imageModel("flux-schnell"),
+  aspectRatio: "9:16",
+});
+
+const pipDemo = (
+  <Render width={1080} height={1920}>
+    {/* background video */}
+    <Clip duration={5}>
+      <Video
+        prompt={{
+          text: "construction worker hammering, building, hard work, sweat on forehead",
+          images: [constructionWorker],
+        }}
+        model={fal.videoModel("kling-v2.5")}
+      />
+    </Clip>
+    {/* foreground overlay with green screen removal */}
+    <Overlay left="55%" top="55%" width="40%" height="40%">
+      <Video
+        prompt={{
+          text: "woman applying lipstick, looking in mirror, makeup tutorial, beauty influencer",
+          images: [influencer],
+        }}
+        model={fal.videoModel("wan-2.5")}
+        removeBackground={{ color: "#00FF00", tolerance: 0.15 }}
+      />
+    </Overlay>
+  </Render>
+);
+
+export default pipDemo;
+
+async function main() {
+  console.log("=== PiP Green Screen Demo ===\n");
+  console.log("background: construction worker");
+  console.log("foreground: influencer doing makeup (green screen removed)\n");
+
+  if (!process.env.FAL_API_KEY && !process.env.FAL_KEY) {
+    console.error("ERROR: FAL_API_KEY or FAL_KEY not found");
+    process.exit(1);
+  }
+
+  try {
+    const buffer = await render(pipDemo, {
+      output: "output/pip-greenscreen-demo.mp4",
+      cache: ".cache/ai",
+      verbose: true,
+    });
+
+    console.log("\n=== SUCCESS ===");
+    console.log(
+      `output: output/pip-greenscreen-demo.mp4 (${(buffer.byteLength / 1024 / 1024).toFixed(2)} MB)`,
+    );
+  } catch (error) {
+    console.error("\n=== FAILED ===");
+    console.error("error:", error instanceof Error ? error.message : error);
+    process.exit(1);
+  }
+}
+
+if (import.meta.main) {
+  main();
+}

src/react/index.ts

@@ -22,6 +22,7 @@ export { render, renderStream } from "./render";
 export type {
   CaptionsProps,
   ClipProps,
+  RemoveBackgroundOptions,
   ImageProps,
   MusicProps,
   OverlayProps,

src/react/renderers/render.ts

@@ -10,6 +10,7 @@ import {
 import { editly } from "../../ai-sdk/providers/editly";
 import type {
   AudioTrack,
+  ChromaKeyOptions,
   Clip,
   Layer,
   VideoLayer,
@@ -19,11 +20,13 @@ import type {
   ClipProps,
   MusicProps,
   OverlayProps,
+  RemoveBackgroundOptions,
   RenderMode,
   RenderOptions,
   RenderProps,
   SpeechProps,
   VargElement,
+  VideoProps,
 } from "../types";
 import { renderCaptions } from "./captions";
 import { renderClip } from "./clip";
@@ -44,6 +47,11 @@ interface RenderedOverlay {
   path: string;
   props: OverlayProps;
   isVideo: boolean;
+  chromaKey?: ChromaKeyOptions;
+}
+
+function hexToFFmpegColor(hex: `#${string}`): `0x${string}` {
+  return `0x${hex.slice(1)}` as `0x${string}`;
 }
 
 export async function renderRoot(
@@ -170,16 +178,28 @@ export async function renderRoot(
       const childElement = child as VargElement;
 
       let path: string | undefined;
+      let chromaKey: ChromaKeyOptions | undefined;
       const isVideo = childElement.type === "video";
 
       if (childElement.type === "video") {
         path = await renderVideo(childElement as VargElement<"video">, ctx);
+        const videoProps = childElement.props as VideoProps;
+        if (videoProps.removeBackground) {
+          const rbOpts = videoProps.removeBackground === true 
+            ? {} 
+            : videoProps.removeBackground;
+          chromaKey = {
+            color: rbOpts.color ? hexToFFmpegColor(rbOpts.color) : "0x00FF00",
+            similarity: rbOpts.tolerance ?? 0.1,
+            blend: rbOpts.blend ?? 0.05,
+          };
+        }
       } else if (childElement.type === "image") {
         path = await renderImage(childElement as VargElement<"image">, ctx);
       }
 
       if (path) {
-        renderedOverlays.push({ path, props: overlayProps, isVideo });
+        renderedOverlays.push({ path, props: overlayProps, isVideo, chromaKey });
 
         if (isVideo && overlayProps.keepAudio) {
           audioTracks.push({
@@ -218,6 +238,7 @@ export async function renderRoot(
         top: overlay.props.top,
         width: overlay.props.width,
         height: overlay.props.height,
+        chromaKey: overlay.chromaKey,
       };
       clip.layers.push(overlayLayer as Layer);
     }

src/react/renderers/video.ts

@@ -1,6 +1,7 @@
 import { File } from "../../ai-sdk/file";
 import type { generateVideo } from "../../ai-sdk/generate-video";
 import type {
+  RemoveBackgroundOptions,
   ImageInput,
   VargElement,
   VideoPrompt,
@@ -70,6 +71,34 @@ async function resolveVideoInput(
   );
 }
 
+function parseRemoveBackgroundOptions(
+  removeBackground: boolean | RemoveBackgroundOptions | undefined,
+): RemoveBackgroundOptions | null {
+  if (!removeBackground) return null;
+  if (removeBackground === true) {
+    return { color: "#00FF00", tolerance: 0.1, blend: 0.05 };
+  }
+  return {
+    color: removeBackground.color ?? "#00FF00",
+    tolerance: removeBackground.tolerance ?? 0.1,
+    blend: removeBackground.blend ?? 0.05,
+  };
+}
+
+function appendGreenScreenInstruction(
+  prompt: VideoPrompt,
+  color: string,
+): VideoPrompt {
+  const instruction = `IMPORTANT: replace video background with solid ${color} background for chroma key compositing.`;
+  if (typeof prompt === "string") {
+    return `${prompt}. ${instruction}`;
+  }
+  return {
+    ...prompt,
+    text: prompt.text ? `${prompt.text}. ${instruction}` : instruction,
+  };
+}
+
 async function resolvePrompt(
   prompt: VideoPrompt,
   ctx: RenderContext,
@@ -107,7 +136,7 @@ export async function renderVideo(
   const props = element.props as VideoProps;
 
   if (props.src && !props.prompt) {
-    return props.src;
+return props.src;
   }
 
   const prompt = props.prompt;
@@ -122,19 +151,22 @@ export async function renderVideo(
     );
   }
 
-  // Compute cache key for deduplication
+  const removeBackgroundConfig = parseRemoveBackgroundOptions(props.removeBackground);
+
   const cacheKey = computeCacheKey(element);
   const cacheKeyStr = JSON.stringify(cacheKey);
 
-  // Check if this element is already being rendered (deduplication)
   const pendingRender = ctx.pending.get(cacheKeyStr);
   if (pendingRender) {
     return pendingRender;
   }
 
-  // Create the render promise and store it for deduplication
   const renderPromise = (async () => {
-    const resolvedPrompt = await resolvePrompt(prompt, ctx);
+    const promptToUse = removeBackgroundConfig
+      ? appendGreenScreenInstruction(prompt, removeBackgroundConfig.color ?? "#00FF00")
+      : prompt;
+
+    const resolvedPrompt = await resolvePrompt(promptToUse, ctx);
 
     const modelId = typeof model === "string" ? model : model.modelId;
     const taskId = ctx.progress

src/react/types.ts

@@ -119,6 +119,7 @@ export type VideoProps = BaseProps &
     model?: VideoModelV3;
     resize?: ResizeMode;
     aspectRatio?: `${number}:${number}`;
+    removeBackground?: boolean | RemoveBackgroundOptions;
   };
 
 export interface SpeechProps extends BaseProps, VolumeProps {
@@ -154,6 +155,15 @@ export interface SubtitleProps extends BaseProps {
   children?: string;
 }
 
+export interface RemoveBackgroundOptions {
+  /** Chroma key color to remove (default: '#00FF00' green) */
+  color?: string;
+  /** Tolerance for color matching 0-1 (default: 0.1) */
+  tolerance?: number;
+  /** Edge blend amount 0-1 (default: 0.05) */
+  blend?: number;
+}
+
 export type MusicProps = BaseProps &
   VolumeProps &
   TrimProps & {