Auto-Reviewer Agent (Quality Control)

// main.ts of the reviewer agent
import { Executions } from '@runflow-ai/sdk/executions';
import { Reviews } from '@runflow-ai/sdk/reviews';
import { LLM } from '@runflow-ai/sdk/llm';

const JUDGE_PROMPT = `You are a strict QA reviewer for a customer-facing AI
agent. Read the user's question and the agent's answer, then output ONLY a
JSON object on a single line with exactly these fields:

  rating  — "good" | "bad" | "needs_improvement"
  reason  — short Portuguese explanation, at most 240 chars

Rubric:
  - "bad" if the answer is wrong, unsafe, contradicts the user, or hallucinates.
  - "needs_improvement" if correct but unclear, too short, missing context,
    or fails to ask a follow-up that the situation clearly required.
  - "good" if correct, clear, and well-targeted.
`;

export async function main() {
  return runCrossAgentReview({
    targetAgentId: process.env.TARGET_AGENT_ID ?? 'customer-support',
    windowHours:   24,
    limit:         200,
  });
}

async function runCrossAgentReview(opts: {
  targetAgentId: string;
  windowHours:   number;
  limit:         number;
}) {
  const executions = new Executions();
  const reviews    = new Reviews();
  const judge      = LLM.openai('gpt-4o-mini');

  // 1. Pull recent executions of the target agent
  const { data: list } = await executions.list({
    agentId: opts.targetAgentId,
    limit:   opts.limit,
  });

  const cutoff = Date.now() - opts.windowHours * 60 * 60 * 1000;
  let created = 0, skipped = 0, errors = 0;

  for (const exec of list) {
    try {
      // Skip outside the window
      const startedAtMs = exec.startedAt ? new Date(exec.startedAt).getTime() : Date.now();
      if (startedAtMs < cutoff) { skipped++; continue; }

      // Idempotency — never double-review
      const { exists } = await reviews.checkHasReview(exec.id);
      if (exists) { skipped++; continue; }

      // Drill into the execution to get the actual messages
      const detail  = await executions.get(exec.id);
      const userMsg = stringify(detail?.input?.message ?? detail?.input);
      const agentMsg = stringify(detail?.output?.message ?? detail?.output);
      if (!userMsg || !agentMsg) { skipped++; continue; }

      // Ask the LLM judge
      const verdictRaw = await judge.chat(
        `${JUDGE_PROMPT}\n\nUSER:\n${userMsg}\n\nAGENT:\n${agentMsg}`,
      );
      const verdict = parseVerdict(verdictRaw);
      if (!verdict) { errors++; continue; }

      // Persist the review
      await reviews.create({
        executionId: exec.id,
        agentId:     opts.targetAgentId,
        rating:      verdict.rating,
        comment:     verdict.reason,
        priority:    verdict.rating === 'bad' ? 'high' : 'medium',
        tags:        ['auto-judge', 'model:gpt-4o-mini'],
      });
      created++;
    } catch (err: any) {
      console.error(`error on exec=${exec.id}: ${err?.message}`);
      errors++;
    }
  }

  return { scanned: list.length, created, skipped, errors };
}

function stringify(v: unknown): string {
  if (v == null) return '';
  if (typeof v === 'string') return v;
  try { return JSON.stringify(v); } catch { return String(v); }
}

function parseVerdict(raw: string) {
  const match = raw.match(/\{[^}]*\}/);
  if (!match) return null;
  try {
    const parsed = JSON.parse(match[0]);
    const rating = parsed?.rating;
    const reason = String(parsed?.reason ?? '').slice(0, 240);
    if (!['good', 'bad', 'needs_improvement'].includes(rating)) return null;
    if (reason.length < 10) return null;
    return { rating, reason };
  } catch {
    return null;
  }
}

if (verdict.rating === 'good') {
  const { reviews: justCreated } = await reviews.list({
    agentId: opts.targetAgentId,
    status:  'pending_review',
    limit:   5,
  });
  const fresh = justCreated.find((r) => r.executionId === exec.id);
  if (fresh) {
    await reviews.dismiss(fresh.id, {
      resolutionNotes: 'auto-judge: good — no action needed.',
    });
  }
}