Mastering Sora 2 Prompting: A Technical Guide with Code Examples

import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

const job = await openai.videos.create({
  model: 'sora-2',           // or 'sora-2-pro'
  size: '1280x720',          // Resolution: portrait or landscape
  seconds: '8',              // Duration: '4', '8', or '12'
  prompt: 'Your creative prompt here'
});

interface BasicPrompt {
  style: string;
  scene: string;
  cinematography: {
    camera: string;
    mood: string;
  };
  actions: string[];
}

function buildBasicPrompt(config: BasicPrompt): string {
  return `
Style: ${config.style}

${config.scene}

Cinematography:
Camera: ${config.cinematography.camera}
Mood: ${config.cinematography.mood}

Actions:
${config.actions.map(action => `- ${action}`).join('\n')}
  `.trim();
}

const prompt = buildBasicPrompt({
  style: '90s documentary-style interview',
  scene: 'An old Swedish man sits in a study.',
  cinematography: {
    camera: 'medium close-up, eye level',
    mood: 'intimate and reflective'
  },
  actions: [
    'The man looks directly at camera',
    'He says: "I still remember when I was young."',
    'He smiles softly and looks away'
  ]
});

const job = await openai.videos.create({
  model: 'sora-2',
  size: '1280x720',
  seconds: '4',
  prompt
});

const loosePrompt = `
A chef tosses vegetables in a wok. 
The kitchen is busy, steam rises, and flames flare up.
`;

const tightPrompt = `
Style: Cinematic food documentary, shot on 35mm film

A middle-aged Asian chef in a white uniform stands at a commercial wok station. 
She tosses bok choy, bell peppers, and snap peas in a carbon steel wok. 
Flames leap up as she adds oil. Steam rises into warm overhead lighting.

Cinematography:
Camera: medium shot from chef's right side, slight dutch angle
Lens: 40mm spherical, shallow depth of field
Lighting: warm overhead tungsten key, cool natural light from window left
Mood: energetic, professional, tactile

Actions:
- Chef flips wok with confident motion
- Flames burst upward, illuminating her face
- She smiles slightly, focused on the cooking
- Steam drifts across the frame
`;

interface SoraPromptConfig {
  // Core content
  style?: string;
  scene: string;
  
  // Camera and lighting
  cinematography?: {
    camera?: string;
    lens?: string;
    lighting?: string;
    mood?: string;
  };
  
  // Story beats
  actions?: string[];
  dialogue?: Array<{ speaker: string; line: string }>;
  
  // Audio cues
  backgroundSound?: string;
  
  // Advanced: production details
  advanced?: {
    format?: string;
    grade?: string;
    lenses?: string;
    atmosphere?: string;
  };
}

function buildSoraPrompt(config: SoraPromptConfig): string {
  const sections: string[] = [];
  
  // Style and format
  if (config.style || config.advanced?.format) {
    sections.push(`Style: ${config.style || ''}`);
    if (config.advanced?.format) {
      sections.push(config.advanced.format);
    }
  }
  
  // Scene description
  sections.push(config.scene);
  
  // Cinematography block
  if (config.cinematography) {
    const cine = config.cinematography;
    sections.push('\nCinematography:');
    if (cine.camera) sections.push(`Camera: ${cine.camera}`);
    if (cine.lens) sections.push(`Lens: ${cine.lens}`);
    if (cine.lighting) sections.push(`Lighting: ${cine.lighting}`);
    if (cine.mood) sections.push(`Mood: ${cine.mood}`);
  }
  
  // Advanced production details
  if (config.advanced) {
    const adv = config.advanced;
    if (adv.grade) sections.push(`\nGrade: ${adv.grade}`);
    if (adv.atmosphere) sections.push(`Atmosphere: ${adv.atmosphere}`);
  }
  
  // Actions
  if (config.actions?.length) {
    sections.push('\nActions:');
    config.actions.forEach(action => sections.push(`- ${action}`));
  }
  
  // Dialogue
  if (config.dialogue?.length) {
    sections.push('\nDialogue:');
    config.dialogue.forEach(d => sections.push(`- ${d.speaker}: "${d.line}"`));
  }
  
  // Background sound
  if (config.backgroundSound) {
    sections.push(`\nBackground Sound:\n${config.backgroundSound}`);
  }
  
  return sections.join('\n').trim();
}

const config: SoraPromptConfig = {
  style: '1970s romantic drama, shot on 35mm film',
  scene: `At golden hour, a brick tenement rooftop transforms into a small stage. 
Laundry lines strung with white sheets sway in the wind, catching the last rays of sunlight. 
Strings of mismatched fairy bulbs hum faintly overhead. A young woman in a flowing red silk 
dress dances barefoot, curls glowing in the fading light.`,
  
  cinematography: {
    camera: 'medium-wide shot, slow dolly-in from eye level',
    lens: '40mm spherical; shallow focus to isolate couple from skyline',
    lighting: 'golden natural key with tungsten bounce; edge from fairy bulbs',
    mood: 'nostalgic, tender, cinematic'
  },
  
  actions: [
    'She spins; her dress flares, catching sunlight',
    'He steps in, catches her hand, and dips her into shadow',
    'Sheets drift across frame, briefly veiling the skyline before parting again'
  ],
  
  dialogue: [
    { speaker: 'Woman', line: 'See? Even the city dances with us tonight.' },
    { speaker: 'Man', line: 'Only because you lead.' }
  ],
  
  backgroundSound: 'Natural ambience only: faint wind, fabric flutter, street noise. No added score.'
};

const prompt = buildSoraPrompt(config);

const filmStockExamples = [
  'shot on 35mm Kodak film with natural grain',
  '16mm documentary footage, vintage 1980s',
  'digital capture emulating 65mm photochemical contrast',
  'Super 8 home video, faded colors, slight gate weave'
];

const lensExamples = [
  '40mm spherical prime, shallow depth of field',
  '24mm wide-angle, deep focus',
  '85mm portrait lens with smooth bokeh',
  'IMAX 70mm for epic scale'
];

const lightingExamples = [
  'Natural window light from camera left, soft shadows',
  'Hard tungsten key light with cool fill from right',
  'Golden hour sunlight, warm and directional',
  'Overcast diffused light, flat and even'
];

interface StylePreset {
  name: string;
  filmStock: string;
  lens: string;
  lighting: string;
  colorPalette: string;
}

const styleLibrary: Record<string, StylePreset> = {
  documentary: {
    name: 'Modern Documentary',
    filmStock: 'Digital capture, natural grain',
    lens: '35mm handheld, slight shake',
    lighting: 'Available light, mixed color temperature',
    colorPalette: 'Desaturated, neutral tones with teal shadows'
  },
  
  cinematic: {
    name: 'Cinematic Drama',
    filmStock: '35mm film, fine grain',
    lens: '50mm spherical prime, shallow depth of field',
    lighting: 'Motivated three-point lighting, warm key',
    colorPalette: 'Rich colors, deep blacks, warm-cool contrast'
  },
  
  retro: {
    name: 'Vintage 1970s',
    filmStock: '16mm film with visible grain and halation',
    lens: '28mm spherical, slight vignette',
    lighting: 'Practical sources, mixed warm tungsten',
    colorPalette: 'Faded earth tones, amber lift in highlights'
  }
};

function applyStylePreset(
  baseConfig: SoraPromptConfig, 
  presetName: keyof typeof styleLibrary
): SoraPromptConfig {
  const preset = styleLibrary[presetName];
  
  return {
    ...baseConfig,
    style: preset.filmStock,
    cinematography: {
      ...baseConfig.cinematography,
      lens: preset.lens,
      lighting: preset.lighting
    },
    advanced: {
      ...baseConfig.advanced,
      grade: preset.colorPalette
    }
  };
}

let config: SoraPromptConfig = {
  scene: 'A woman walks through a rainy city street at night',
  actions: ['She looks up at neon signs', 'Rain drips from her umbrella']
};

config = applyStylePreset(config, 'cinematic');
const prompt = buildSoraPrompt(config);

async function remixVideo(
  originalJobId: string,
  changes: string
): Promise<string> {
  const job = await openai.videos.create({
    model: 'sora-2',
    size: '1280x720',
    seconds: '4',
    prompt: changes,
    // Reference the original video
    input_reference: originalJobId
  });
  
  // Poll until complete
  let status = await openai.videos.retrieve(job.id);
  while (status.status === 'in_progress' || status.status === 'queued') {
    await new Promise(resolve => setTimeout(resolve, 2000));
    status = await openai.videos.retrieve(job.id);
  }
  
  return job.id;
}

async function iterativeRefinement(
  initialPrompt: string,
  refinements: string[]
): Promise<string[]> {
  const jobIds: string[] = [];
  
  // Generate initial video
  let job = await openai.videos.create({
    model: 'sora-2',
    size: '1280x720',
    seconds: '4',
    prompt: initialPrompt
  });
  
  // Wait for completion
  while (job.status === 'in_progress' || job.status === 'queued') {
    await new Promise(r => setTimeout(r, 2000));
    job = await openai.videos.retrieve(job.id);
  }
  
  jobIds.push(job.id);
  let currentJobId = job.id;
  
  // Apply refinements sequentially
  for (const refinement of refinements) {
    currentJobId = await remixVideo(currentJobId, refinement);
    jobIds.push(currentJobId);
  }
  
  return jobIds;
}

const jobIds = await iterativeRefinement(
  'A chef cooking in a busy kitchen',
  [
    'Same scene, but change camera to over-shoulder angle',
    'Same framing, add more steam and smoke in the air',
    'Same everything, but change lighting to golden hour through window'
  ]
);

// Download all versions to compare
for (const id of jobIds) {
  const response = await openai.videos.downloadContent(id);
  // Save to disk or display
}

async function extractLastFrame(videoBlob: Blob): Promise<Blob> {
  const video = document.createElement('video');
  video.src = URL.createObjectURL(videoBlob);
  video.muted = true;
  
  await new Promise(resolve => {
    video.onloadedmetadata = resolve;
  });
  
  // Seek to the end
  video.currentTime = video.duration;
  await new Promise(resolve => {
    video.onseeked = resolve;
  });
  
  // Draw to canvas
  const canvas = document.createElement('canvas');
  canvas.width = video.videoWidth;
  canvas.height = video.videoHeight;
  
  const ctx = canvas.getContext('2d');
  if (!ctx) throw new Error('Canvas context unavailable');
  ctx.drawImage(video, 0, 0);
  
  // Export as JPEG
  return new Promise((resolve, reject) => {
    canvas.toBlob(
      blob => blob ? resolve(blob) : reject(new Error('Failed to extract frame')),
      'image/jpeg',
      0.92
    );
  });
}

import { toFile } from 'openai';

async function generateSegmentWithImage(
  prompt: string,
  inputImage?: Blob
): Promise<Blob> {
  const body: any = {
    model: 'sora-2',
    prompt,
    seconds: '8',
    size: '1280x720'
  };
  
  if (inputImage) {
    body.input_reference = await toFile(
      inputImage,
      'reference.jpg',
      { type: 'image/jpeg' }
    );
  }
  
  let job = await openai.videos.create(body);
  
  while (job.status === 'in_progress' || job.status === 'queued') {
    await new Promise(resolve => setTimeout(resolve, 2000));
    job = await openai.videos.retrieve(job.id);
  }
  
  if (job.status === 'failed') {
    throw new Error(job.error?.message || 'Generation failed');
  }
  
  const response = await openai.videos.downloadContent(job.id);
  const arrayBuffer = await response.arrayBuffer();
  return new Blob([arrayBuffer], { type: 'video/mp4' });
}

interface Shot {
  prompt: string;
  duration: 4 | 8 | 12;
}

async function generateSequence(shots: Shot[]): Promise<Blob[]> {
  const segments: Blob[] = [];
  let inputReference: Blob | undefined;
  
  for (const shot of shots) {
    const segment = await generateSegmentWithImage(
      shot.prompt,
      inputReference
    );
    
    segments.push(segment);
    inputReference = await extractLastFrame(segment);
  }
  
  return segments;
}

const sequence: Shot[] = [
  {
    duration: 8,
    prompt: buildSoraPrompt({
      style: 'Cinematic drama, 35mm film',
      scene: 'A woman stands at a window, looking out at the city at dusk',
      cinematography: {
        camera: 'medium shot from behind, over shoulder',
        lighting: 'warm interior light, cool blue twilight through window',
        mood: 'contemplative, quiet'
      },
      actions: [
        'She touches the window glass',
        'Her breath fogs it slightly',
        'She traces a shape with her finger'
      ]
    })
  },
  {
    duration: 8,
    prompt: buildSoraPrompt({
      style: 'Same cinematic style, maintaining continuity',
      scene: 'Close-up of her hand on the window glass. The city lights twinkle outside.',
      cinematography: {
        camera: 'tight close-up on hand and glass',
        lighting: 'same warm-cool contrast',
        mood: 'intimate, reflective'
      },
      actions: [
        'She traces a heart shape',
        'Camera slowly pulls back to reveal her face in reflection',
        'She smiles softly'
      ]
    })
  },
  {
    duration: 4,
    prompt: buildSoraPrompt({
      style: 'Same style, final beat',
      scene: 'She turns away from the window and walks into the warmly lit room',
      cinematography: {
        camera: 'wide shot, static',
        lighting: 'warm practical lights in room, fading twilight from window',
        mood: 'resolution, peace'
      },
      actions: [
        'She walks slowly across the room',
        'She picks up a book from the table',
        'She sits down in an armchair'
      ]
    })
  }
];

const segments = await generateSequence(sequence);

import {
  ALL_FORMATS,
  BlobSource,
  BufferTarget,
  EncodedAudioPacketSource,
  EncodedPacket,
  EncodedPacketSink,
  EncodedVideoPacketSource,
  Input,
  Mp4OutputFormat,
  Output
} from 'mediabunny';

async function stitchSegments(segments: Blob[]): Promise<Blob> {
  if (!segments.length) throw new Error('No segments to stitch');

  const firstInput = new Input({ 
    source: new BlobSource(segments[0]), 
    formats: ALL_FORMATS 
  });
  const firstVideoTrack = await firstInput.getPrimaryVideoTrack();
  const firstAudioTrack = await firstInput.getPrimaryAudioTrack();
  
  if (!firstVideoTrack) throw new Error('No video track');

  const videoCodec = (await firstVideoTrack.codec) || 'avc';
  const audioCodec = firstAudioTrack 
    ? (await firstAudioTrack.codec) || 'aac' 
    : null;

  const target = new BufferTarget();
  const output = new Output({ 
    format: new Mp4OutputFormat({ fastStart: 'in-memory' }), 
    target 
  });

  const outVideo = new EncodedVideoPacketSource(videoCodec as any);
  output.addVideoTrack(outVideo);

  let outAudio: EncodedAudioPacketSource | null = null;
  if (audioCodec) {
    outAudio = new EncodedAudioPacketSource(audioCodec as any);
    output.addAudioTrack(outAudio);
  }

  await output.start();

  let videoTimestampOffset = 0;
  let audioTimestampOffset = 0;

  for (const seg of segments) {
    const input = new Input({ 
      source: new BlobSource(seg), 
      formats: ALL_FORMATS 
    });
    const vTrack = await input.getPrimaryVideoTrack();
    const aTrack = await input.getPrimaryAudioTrack();
    if (!vTrack) continue;

    const vSink = new EncodedPacketSink(vTrack);
    for await (const packet of vSink.packets()) {
      const cloned: EncodedPacket = packet.clone({ 
        timestamp: packet.timestamp + videoTimestampOffset 
      });
      await outVideo.add(cloned);
    }
    videoTimestampOffset += await vTrack.computeDuration();

    if (aTrack && outAudio) {
      const aSink = new EncodedPacketSink(aTrack);
      for await (const packet of aSink.packets()) {
        const cloned: EncodedPacket = packet.clone({ 
          timestamp: packet.timestamp + audioTimestampOffset 
        });
        await outAudio.add(cloned);
      }
      audioTimestampOffset += await aTrack.computeDuration();
    }
  }

  outVideo.close();
  if (outAudio) outAudio.close();
  await output.finalize();

  const finalBuffer: ArrayBuffer | null = 
    (target as unknown as { buffer: ArrayBuffer | null }).buffer;
  if (!finalBuffer) throw new Error('Failed to finalize video');
  
  return new Blob([finalBuffer], { type: 'video/mp4' });
}

const segments = await generateSequence(sequence);
const finalVideo = await stitchSegments(segments);

// Save or display
const url = URL.createObjectURL(finalVideo);
const videoElement = document.createElement('video');
videoElement.src = url;
videoElement.controls = true;
document.body.appendChild(videoElement);

const proPrompt = buildSoraPrompt({
  style: `Format & Look: 180° shutter; digital capture emulating 65mm photochemical 
contrast; fine grain; subtle halation on speculars.

Lenses & Filtration: 32mm spherical prime; Black Pro-Mist 1/4; slight CPL rotation 
to manage glass reflections.`,

  scene: `Urban commuter platform, dawn. Foreground: yellow safety line, coffee cup on 
bench. Midground: waiting passengers silhouetted in haze. Background: arriving train 
braking to a stop.`,

  cinematography: {
    camera: '32mm shoulder-mounted slow dolly left, low and close to lens axis',
    lens: 'Spherical prime with minimal flare, preserve silhouette clarity',
    lighting: `Natural sunlight from camera left, low angle (07:30 AM).
Bounce: 4×4 ultrabounce silver from trackside.
Negative fill from opposite wall.
Practical: sodium platform lights on dim fade.`,
    mood: 'Anticipatory, realistic, tactile'
  },

  advanced: {
    grade: `Highlights: clean morning sunlight with amber lift.
Mids: balanced neutrals with slight teal cast in shadows.
Blacks: soft, neutral with mild lift for haze retention.`,
    atmosphere: 'Gentle mist; train exhaust drift through light beam.'
  },

  actions: [
    'Camera slides past platform edge; train headlights flare softly through mist',
    'Traveler mid-frame looks down tracks',
    'Morning light blooms across lens naturally'
  ],

  backgroundSound: `Diegetic only: faint rail screech, train brakes hiss, 
distant announcement muffled, low ambient hum. No score.`
});

interface PromptVersion {
  id: string;
  prompt: string;
  config: SoraPromptConfig;
  generatedAt: Date;
  jobId?: string;
  rating?: number;
  notes?: string;
}

class PromptVersionManager {
  private versions: PromptVersion[] = [];
  
  saveVersion(
    config: SoraPromptConfig, 
    jobId?: string
  ): string {
    const version: PromptVersion = {
      id: crypto.randomUUID(),
      prompt: buildSoraPrompt(config),
      config,
      generatedAt: new Date(),
      jobId
    };
    
    this.versions.push(version);
    return version.id;
  }
  
  rate(versionId: string, rating: number, notes?: string) {
    const version = this.versions.find(v => v.id === versionId);
    if (version) {
      version.rating = rating;
      version.notes = notes;
    }
  }
  
  getBestVersions(limit: number = 5): PromptVersion[] {
    return this.versions
      .filter(v => v.rating !== undefined)
      .sort((a, b) => (b.rating || 0) - (a.rating || 0))
      .slice(0, limit);
  }
  
  exportHistory(): string {
    return JSON.stringify(this.versions, null, 2);
  }
}

const manager = new PromptVersionManager();

// Try multiple variations
const variations: SoraPromptConfig[] = [
  { scene: 'A cat sits on a windowsill, version A...', actions: ['...'] },
  { scene: 'A cat sits on a windowsill, version B...', actions: ['...'] },
  { scene: 'A cat sits on a windowsill, version C...', actions: ['...'] }
];

for (const config of variations) {
  const prompt = buildSoraPrompt(config);
  const job = await openai.videos.create({
    model: 'sora-2',
    size: '1280x720',
    seconds: '4',
    prompt
  });
  
  manager.saveVersion(config, job.id);
}

// Later, rate them
manager.rate('version-id-1', 4, 'Good framing but lighting too dark');
manager.rate('version-id-2', 5, 'Perfect!');

// Export for review
console.log(manager.exportHistory());