Media Processing

import { Agent, openai } from '@runflow-ai/sdk';

export const agent = new Agent({
  name: 'product-helper',
  model: openai('gpt-4o'),                  // any vision-capable model
  instructions: 'Help the user evaluate products.',
  media: {
    processAttachments: true,                // ← opt-in
  },
});

import { Agent, openai } from '@runflow-ai/sdk';

export const whatsappAgent = new Agent({
  name: 'WhatsApp Support',
  instructions: `You are a customer support agent for WhatsApp.

## Behavior
- Respond in the customer's language
- Be concise — WhatsApp messages should be short
- When the customer sends a voice message, you'll receive the transcription — respond naturally
- When the customer sends a photo, analyze it and respond accordingly

## Tools
- Use create-ticket when the issue needs human follow-up
- If a customer sends a photo of a damaged product, create a ticket with priority 'high'`,
  model: openai('gpt-4o'),
  memory: { maxTurns: 30 },
  media: {
    transcribeAudio: true,
    processImages: true,
    audioProvider: 'openai',
    audioLanguage: 'pt',
  },
  tools: {
    createTicket: createTicketTool,
  },
  observability: 'full',
});

import { identify, track } from '@runflow-ai/sdk/observability';
import { whatsappAgent } from './agent';

function parseWhatsAppInput(input: any) {
  // Zenvia webhook format
  if (input.message?.from) {
    const content = input.message.contents?.[0];
    return {
      phone: input.message.from,
      message: content?.text || content?.caption || '',
      file: content?.fileUrl ? {
        url: content.fileUrl,
        contentType: content.fileMimeType,
        caption: content.caption,
      } : undefined,
      channel: 'zenvia',
    };
  }

  // Direct API
  return {
    phone: input.phone,
    message: input.message || '',
    file: input.file,
    channel: input.channel || 'api',
  };
}

export async function main(input: any) {
  const { phone, message, file, channel } = parseWhatsAppInput(input);

  if (!phone) {
    return { error: 'phone is required' };
  }

  if (!message && !file) {
    return { error: 'message or file is required' };
  }

  identify(phone);

  // Track media type for analytics
  const mediaType = file?.contentType?.startsWith('audio') ? 'audio'
    : file?.contentType?.startsWith('image') ? 'image'
    : 'text';

  track('whatsapp_message_received', { channel, mediaType });

  try {
    const result = await whatsappAgent.process({
      message,
      sessionId: `whatsapp_${phone}`,
      file,
    });

    return {
      message: result.message,
      phone,
    };
  } catch (error) {
    console.error('[whatsapp-support] Error:', error);
    return { error: 'An error occurred processing your message' };
  }
}

import { createTool } from '@runflow-ai/sdk';
import { transcribe } from '@runflow-ai/sdk';
import { track } from '@runflow-ai/sdk/observability';
import { z } from 'zod';

export const processVoiceTool = createTool({
  id: 'process-voice',
  description: 'Transcribe and analyze a voice message',
  inputSchema: z.object({
    audioUrl: z.string().describe('URL of the audio file'),
    language: z.string().optional().describe('Language code (default: pt)'),
  }),
  execute: async (params) => {
    try {
      const result = await transcribe({
        audioUrl: params.audioUrl,
        language: params.language || 'pt',
        provider: 'openai',
      });

      track('voice_transcribed', {
        language: params.language || 'pt',
        textLength: result.text.length,
      });

      return {
        success: true,
        text: result.text,
        language: params.language || 'pt',
      };
    } catch (error) {
      return {
        success: false,
        error: error instanceof Error ? error.message : 'Transcription failed',
      };
    }
  },
});