flare-agent Bài 3: LLM Router — Groq, Workers AI, Ollama

Series: Build Your Own AI Agent Framework trên Cloudflare Bài: 3 / 7 — @flare-agent/providers

---

Vấn đề cần giải quyết

Mỗi LLM provider có API khác nhau:

• Groq — OpenAI-compatible REST API
• Workers AI — env.AI.run() binding
• Ollama — REST API tương tự OpenAI nhưng response format khác một chút

Nếu code trực tiếp vào từng provider, khi muốn switch từ Groq sang Workers AI phải sửa code ở nhiều nơi. @flare-agent/providers giải quyết điều này bằng cách normalize tất cả về cùng 1 interface.

---

Cấu trúc package

packages/providers/
  src/
    base.ts           # Abstract BaseProvider
    groq.ts           # GroqProvider
    workersai.ts      # WorkersAIProvider
    ollama.ts         # OllamaProvider
    router.ts         # LLMRouter — factory + dispatch
    index.ts

---

Base Provider

// src/base.ts
import type { LLMProvider, Message, LLMResponse, ToolSchema } from '@flare-agent/types';

export abstract class BaseProvider implements LLMProvider {
  abstract chat(
    messages: Message[],
    tools?: ToolSchema[]
  ): Promise<LLMResponse>;

  // Default stream implementation — subclass override nếu có native streaming
  stream(
    messages: Message[],
    tools?: ToolSchema[]
  ): ReadableStream<string> {
    return new ReadableStream({
      start: async (controller) => {
        const response = await this.chat(messages, tools);
        if (response.content) {
          controller.enqueue(response.content);
        }
        controller.close();
      },
    });
  }

  // Helper: normalize tool_calls từ OpenAI format sang LLMResponse
  protected normalizeToolCall(toolCall: any): LLMResponse {
    return {
      type: 'tool_call',
      toolCall: {
        id: toolCall.id ?? crypto.randomUUID(),
        name: toolCall.function?.name ?? toolCall.name,
        args: typeof toolCall.function?.arguments === 'string'
          ? JSON.parse(toolCall.function.arguments)
          : toolCall.function?.arguments ?? toolCall.arguments,
      },
    };
  }
}

---

Groq Provider

// src/groq.ts
import type { Message, LLMResponse, ToolSchema } from '@flare-agent/types';
import { BaseProvider } from './base';

interface GroqConfig {
  apiKey: string;
  model: string;
  baseUrl?: string;
}

export class GroqProvider extends BaseProvider {
  private baseUrl: string;

  constructor(private config: GroqConfig) {
    super();
    this.baseUrl = config.baseUrl ?? 'https://api.groq.com/openai/v1';
  }

  async chat(
    messages: Message[],
    tools?: ToolSchema[]
  ): Promise<LLMResponse> {
    const res = await fetch(`${this.baseUrl}/chat/completions`, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${this.config.apiKey}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        model: this.config.model,
        messages,
        ...(tools?.length && {
          tools: tools.map((t) => ({ type: 'function', function: t })),
          tool_choice: 'auto',
        }),
      }),
    });

    if (!res.ok) {
      throw new Error(`Groq API error: ${res.status} ${await res.text()}`);
    }

    const data = await res.json<any>();
    const msg = data.choices[0].message;

    if (msg.tool_calls?.[0]) {
      return this.normalizeToolCall(msg.tool_calls[0]);
    }

    return { type: 'text', content: msg.content };
  }

  override stream(
    messages: Message[],
    tools?: ToolSchema[]
  ): ReadableStream<string> {
    const config = this.config;
    const baseUrl = this.baseUrl;

    return new ReadableStream({
      async start(controller) {
        const res = await fetch(`${baseUrl}/chat/completions`, {
          method: 'POST',
          headers: {
            Authorization: `Bearer ${config.apiKey}`,
            'Content-Type': 'application/json',
          },
          body: JSON.stringify({
            model: config.model,
            messages,
            stream: true,
          }),
        });

        const reader = res.body!.getReader();
        const decoder = new TextDecoder();

        while (true) {
          const { done, value } = await reader.read();
          if (done) break;

          const lines = decoder.decode(value).split('\n');
          for (const line of lines) {
            if (!line.startsWith('data: ')) continue;
            const json = line.slice(6).trim();
            if (json === '[DONE]') { controller.close(); return; }
            try {
              const delta = JSON.parse(json).choices[0].delta.content;
              if (delta) controller.enqueue(delta);
            } catch { /* skip malformed chunks */ }
          }
        }
      },
    });
  }
}

---

Workers AI Provider

// src/workersai.ts
import type { Message, LLMResponse, ToolSchema } from '@flare-agent/types';
import { BaseProvider } from './base';

interface WorkersAIConfig {
  ai: Ai; // Cloudflare AI binding
  model: string;
}

export class WorkersAIProvider extends BaseProvider {
  constructor(private config: WorkersAIConfig) {
    super();
  }

  async chat(
    messages: Message[],
    tools?: ToolSchema[]
  ): Promise<LLMResponse> {
    const response = await this.config.ai.run(
      this.config.model as any,
      {
        messages,
        ...(tools?.length && {
          tools: tools.map((t) => ({ type: 'function', function: t })),
        }),
      }
    ) as any;

    // Workers AI response format khác OpenAI một chút
    if (response.tool_calls?.[0]) {
      return this.normalizeToolCall(response.tool_calls[0]);
    }

    return { type: 'text', content: response.response };
  }

  override stream(
    messages: Message[]
  ): ReadableStream<string> {
    return this.config.ai.run(
      this.config.model as any,
      { messages, stream: true }
    ) as ReadableStream<string>;
  }
}

---

Ollama Provider

// src/ollama.ts
import type { Message, LLMResponse, ToolSchema } from '@flare-agent/types';
import { BaseProvider } from './base';

interface OllamaConfig {
  baseUrl: string; // e.g. http://localhost:11434
  model: string;
}

export class OllamaProvider extends BaseProvider {
  constructor(private config: OllamaConfig) {
    super();
  }

  async chat(
    messages: Message[],
    tools?: ToolSchema[]
  ): Promise<LLMResponse> {
    const res = await fetch(`${this.config.baseUrl}/api/chat`, {
      method: 'POST',
      body: JSON.stringify({
        model: this.config.model,
        messages,
        stream: false,
        ...(tools?.length && {
          tools: tools.map((t) => ({ type: 'function', function: t })),
        }),
      }),
    });

    const data = await res.json<any>();
    const msg = data.message;

    if (msg.tool_calls?.[0]) {
      return this.normalizeToolCall(msg.tool_calls[0]);
    }

    return { type: 'text', content: msg.content };
  }
}

---

LLM Router — factory + dispatch

// src/router.ts
import type { LLMProvider, ProviderName } from '@flare-agent/types';
import { GroqProvider } from './groq';
import { WorkersAIProvider } from './workersai';
import { OllamaProvider } from './ollama';

type ProviderConfig = {
  groq: { apiKey: string; model: string };
  workersai: { ai: Ai; model: string };
  ollama: { baseUrl: string; model: string };
};

export class LLMRouter {
  private providers = new Map<string, LLMProvider>();

  register<T extends ProviderName>(
    name: T,
    config: ProviderConfig[T]
  ): this {
    switch (name) {
      case 'groq':
        this.providers.set(name, new GroqProvider(config as any));
        break;
      case 'workersai':
        this.providers.set(name, new WorkersAIProvider(config as any));
        break;
      case 'ollama':
        this.providers.set(name, new OllamaProvider(config as any));
        break;
    }
    return this;
  }

  get(name: ProviderName): LLMProvider {
    const provider = this.providers.get(name);
    if (!provider) throw new Error(`Provider "${name}" not registered`);
    return provider;
  }
}

---

Sử dụng

const router = new LLMRouter()
  .register('groq', { apiKey: env.GROQ_API_KEY, model: 'llama-3.3-70b-versatile' })
  .register('workersai', { ai: env.AI, model: '@cf/meta/llama-3.3-70b-instruct-fp8-fast' })
  .register('ollama', { baseUrl: env.OLLAMA_URL, model: 'qwen3:8b' });

// Gọi Groq
const response = await router.get('groq').chat(messages, tools);

// Swap sang Workers AI — không đổi gì khác
const response = await router.get('workersai').chat(messages, tools);

---

Checklist

• Tạo packages/providers/
• Implement đủ 3 providers
• Unit test: mock fetch, verify response normalization
• Build thành công

---

Bài tiếp theo: Bài 4 — Memory: KV, D1 và Vectorize adapters