Building an AI Conversation Practice App: Part 2 – Backend Speech-to-Text Processing with OpenAI Whisper

// Disable Next.js body parsing
export const config = { api: { bodyParser: false } };

// Custom form parsing with formidable
const form = new IncomingForm({
  keepExtensions: true,
});

const formData: [Fields, Files] = await new Promise((resolve, reject) => {
  form.parse(req, (err, fields, files) => {
    if (err) return reject(err);
    resolve([fields, files]);
  });
});

// Create readable stream from uploaded file
const audioPath = audioFile.filepath;
const audioStream = createReadStream(audioPath);

// Stream directly to OpenAI (no memory buffering)
const transcription = await openai.audio.transcriptions.create({
  file: audioStream,
  model: "whisper-1",
  language: "en",
  prompt: "This is a conversation in Canadian English.",
});

// Frontend validation before API call
const recordingDuration = Date.now() - recordingStartTimeRef.current;

if (recordingDuration < 300) {
  const clarificationText = getRandomClarification();

  const assistantMessage: Message = {
    role: 'assistant',
    content: '',
    isStreaming: true
  };

  setMessages(prevMessages => [...prevMessages, assistantMessage]);
  streamText(clarificationText, messageIndex);
  return; // Don't call STT API
}

// Only send to backend if recording is long enough
const sttResponse = await fetch('/api/stt', {
  method: 'POST',
  body: formData,
});

const transcription = await openai.audio.transcriptions.create({
  file: audioStream,
  model: "whisper-1",
  language: "en",
  prompt: "This is a conversation in Canadian English.",
});

export default async function handler(
  req: NextApiRequest, 
  res: NextApiResponse<ApiResponse>
) {
  if (req.method !== 'POST') {
    return res.status(405).json({ success: false, error: 'Method not allowed' });
  }

  try {
    // Parse form data
    const form = new IncomingForm({ keepExtensions: true });
    const formData: [Fields, Files] = await new Promise((resolve, reject) => {
      form.parse(req, (err, fields, files) => {
        if (err) return reject(err);
        resolve([fields, files]);
      });
    });

    const [fields, files] = formData;

    // Validate audio file
    const audioFiles = files.audio;
    if (!audioFiles || !Array.isArray(audioFiles) || audioFiles.length === 0) {
      return res.status(400).json({ success: false, error: 'No audio file provided' });
    }

    const audioFile = audioFiles[0] as File;

    // Process with OpenAI
    const audioPath = audioFile.filepath;
    const audioStream = createReadStream(audioPath);

    const transcription = await openai.audio.transcriptions.create({
      file: audioStream,
      model: "whisper-1",
      language: "en",
      prompt: "This is a conversation in Canadian English.",
    });

    // Cleanup and respond
    fs.unlinkSync(audioPath);
    return res.status(200).json({
      success: true,
      transcript: transcription.text
    });

  } catch (error) {
    console.error('STT Error:', error);
    return res.status(500).json({
      success: false,
      error: error instanceof Error ? error.message : 'Failed to transcribe audio'
    });
  }
}

// Access the audio file with proper type checking
const audioFiles = files.audio;
if (!audioFiles || !Array.isArray(audioFiles) || audioFiles.length === 0) {
  return res.status(400).json({ 
    success: false, 
    error: 'No audio file provided' 
  });
}

const audioFile = audioFiles[0] as File;

// Additional validation
if (!audioFile.filepath || audioFile.size === 0) {
  return res.status(400).json({ 
    success: false, 
    error: 'Invalid audio file' 
  });
}

// Critical: Clean up temporary files
const audioPath = audioFile.filepath;
const audioStream = createReadStream(audioPath);

// Process audio...

// Always cleanup, even on error
try {
  fs.unlinkSync(audioPath);
} catch (cleanupError) {
  console.warn('Failed to cleanup temp file:', cleanupError);
}

// Load entire file into memory
const audioBuffer = fs.readFileSync(audioPath);
const transcription = await openai.audio.transcriptions.create({
  file: audioBuffer, // Large memory usage
});

// Stream file directly
const audioStream = createReadStream(audioPath);
const transcription = await openai.audio.transcriptions.create({
  file: audioStream, // Minimal memory usage
});

// Frontend STT call
const sttResponse = await fetch('/api/stt', {
  method: 'POST',
  body: formData,
});

const sttData = await sttResponse.json();

if (!sttData.success) {
  // Handle error gracefully
  const clarificationText = getRandomClarification();
  // Show clarification message to user
} else {
  // Continue with conversation
  const transcript = sttData.transcript;
  // Send to GPT for response generation
}

// If STT fails, don't break the conversation
if (!sttData.success) {
  const clarificationPhrases = [
    "Sorry, can you repeat that?",
    "Could you say that again please?",
    "I didn't quite get that. Could you repeat?",
  ];

  const randomClarification = clarificationPhrases[
    Math.floor(Math.random() * clarificationPhrases.length)
  ];

  // Continue conversation with clarification
}

// Comprehensive logging for debugging
console.log('STT Response:', {
  success: sttData.success,
  transcript: sttData.transcript?.substring(0, 50) + '...',
  processingTime: Date.now() - startTime,
  fileSize: audioFile.size
});

// Implement rate limiting for production
if (requestCount > 10) { // 10 requests per minute
  return res.status(429).json({
    success: false,
    error: 'You\'re speaking too fast! Please wait a moment before trying again.'
  });
}

if (response.status === 429) {
  showError("Please wait a moment before recording again");
}