Interpolate a Video from the First and Last Frames with Veo 3.1 and Nano Banana

IS_VEO31_USED="true"

// firebase-ai.config
{
  // ... other configuration values
  "is_veo31_used": true
}

// gemini provider

import firebaseConfig from '../../firebase-ai.json';
import { GEMINI_AI, IS_VEO31_USED } from '../constants/gemini.constant';

export const IS_VEO31_USED = new InjectionToken<boolean>('IS_VEO31_USED');

export function provideGemini() {
  return makeEnvironmentProviders([
    {
      provide: IS_VEO31_USED,
      useValue: firebaseConfig.is_veo31_used,
    }
  ])
}

export const appConfig: ApplicationConfig = {
  providers: [
    provideGemini(),
  ]
};

// Visual Story Service

@Injectable({
  providedIn: 'root'
})
export class VisualStoryService {
  buildStepPrompts(genArgs: VisualStoryGenerateArgs): string[] {
    const { userPrompt, args } = genArgs;
    const currentPrompt = userPrompt.trim();

    if (!currentPrompt) {
      return [];
    }

    const stepPrompts: string[] = [];

    for (let i = 0; i < args.numberOfImages; i++) {
      const storyPrompt = this.buildStoryPrompt({ userPrompt: currentPrompt, args }, i + 1);
      stepPrompts.push(storyPrompt);
    }

    return stepPrompts;
  }

  private buildStoryPrompt(genArgs: VisualStoryGenerateArgs, stepNumber: number): string {
    const { userPrompt, args } = genArgs;
    const { numberOfImages, style, transition, type } = args;
    let fullPrompt = `${userPrompt}, step ${stepNumber} of ${numberOfImages}`;

    // Add context based on type
    switch (type) {
      case 'story':
        fullPrompt += `, narrative sequence, ${style} art style`;
        break;
      case 'process':
        fullPrompt += `, procedural step, instructional illustration`;
        break;
      // ... other type of visual story
    }

    if (stepNumber > 1) {
      fullPrompt += `, ${transition} transition from previous step`;
    }

    return fullPrompt;
  }
}

// Example: userPrompt="A wizard making a potion", args.numberOfImages=3, args.type='story'

Step 1 Prompt: "A wizard making a potion, step 1 of 3, narrative sequence, cinematic art style"
Step 2 Prompt: "A wizard making a potion, step 2 of 3, narrative sequence, cinematic art style, liquid dissolving transition from previous step"
Step 3 Prompt: "A wizard making a potion, step 3 of 3, narrative sequence, cinematic art style, liquid dissolving transition from previous step"

@Injectable({
  providedIn: 'root'
})
export class GenMediaService {
  private readonly geminiService = inject(GeminiService);

  async generateVideoFromFrames(imageParams: GenerateVideoFromFramesRequest) {
    const isVeo31Used = imageParams.isVeo31Used || false;

    const loadVideoPromise = isVeo31Used ?
       this.geminiService.generateVideo({
          prompt: imageParams.prompt,
          imageBytes: imageParams.imageBytes,
          mimeType: imageParams.mimeType,
          config: {
            aspectRatio: '16:9',
            resolution: "720p",
            lastFrame: {
              imageBytes: imageParams.lastFrameImageBytes,
              mimeType: imageParams.lastFrameMimeType
            }
          }
       }) : this.getFallbackVideoUrl(imageParams);

      return await loadVideoPromise;
    }
  }

  private async getFallbackVideoUrl(imageParams: GenerateVideoRequestImageParams) {
    return this.geminiService.generateVideo({
      prompt: imageParams.prompt,
      imageBytes: imageParams.imageBytes,
      mimeType: imageParams.mimeType,
      config: {
        aspectRatio: '16:9',
      }
    });
  }
}

// Visual Story Service

interpolateVideo(request: GenerateVideoFromFramesRequest): Promise<VideoResponse> {
    return this.genMediaService.generateVideoFromFrames(request);
}

<app-visual-story-video
    [userPrompt]="this.promptArgs().userPrompt"
    [images]="this.genmedia()?.images()"
/>

@Component({
  selector: 'app-visual-story-video',
  imports: [...import components...],
  template: `
    @if (canGenerateVideoFromFirstLastFrames()) {
      <button type="button (click)="generateVideoFromFrames()">
          Interpolate video
      </button>

      @let videoUrl = videoResponse()?.videoUrl;
      @if (isLoading()) {
        <app-loader />
      } @else if (videoUrl) {
        <app-video-player class="block" [videoUrl]="videoUrl" />
      }
    }
  `,
  changeDetection: ChangeDetectionStrategy.OnPush,
})
export default class VisualStoryVideoComponent {
  private readonly visualStoryService = inject(VisualStoryService);
  private readonly isVeo31Used = inject(IS_VEO31_USED);

  images = input<ImageResponse[] | undefined>(undefined);
  userPrompt = input.required<string>();

  isLoading = signal(false);
  videoResponse = signal<VideoResponse | undefined>(undefined);

  firstImage = computed(() => this.images()?.[0]);
  lastImage = computed(() => {
    const numImages = this.images()?.length || 0;
    return numImages < 2 ? undefined : this.images()?.[numImages - 1];
  });

  canGenerateVideoFromFirstLastFrames = computed(() => {
    const hasFirstImage = !!this.firstImage()?.data && !!this.firstImage()?.mimeType;
    const hasLastImage = !!this.lastImage()?.data && !!this.lastImage()?.mimeType;
    return this.isVeo31Used && hasFirstImage && hasLastImage;
  });

  async generateVideoFromFrames(): Promise<void> {
    try {
      this.isLoading.set(true);
      this.videoResponse.set(undefined);

      if (!this.canGenerateVideoFromFirstLastFrames()) {
        return;
      }

      const { data: firstImageData, mimeType: firstImageMimeType } = this.firstImage() || { data: '', mimeType: '' };
      const { data: lastImageData, mimeType: lastImageMimeType } = this.lastImage() || { data: '', mimeType: '' };
      const result = await this.visualStoryService.interpolateVideo({
        prompt: this.userPrompt(),
        imageBytes: firstImageData,
        mimeType: firstImageMimeType,
        lastFrameImageBytes: lastImageData,
        lastFrameMimeType: lastImageMimeType,
        isVeo31Used: this.isVeo31Used
      });
      this.videoResponse.set(result);
    } finally {
      this.isLoading.set(false);
    }
  }
}