Streaming AI

Stream AI responses in real-time with AIEngine or directly from LLM providers.

Streaming AI

Stream AI responses token-by-token using the AsyncIterable API. All three providers support native streaming. Providers that don't override stream() automatically fall back to complete().

AIEngine Streaming

Use queryStream() for high-level streaming with automatic context building.

import { useState } from 'react';
import { GraphStore, AIEngine } from '@inferagraph/core';
import { AnthropicProvider } from '@inferagraph/anthropic-provider';

const store = new GraphStore();
const provider = new AnthropicProvider({ apiKey: 'sk-ant-...' });
const ai = new AIEngine(store, provider);

function StreamingChat() {
  const [response, setResponse] = useState('');
  const [streaming, setStreaming] = useState(false);

  const handleQuery = async () => {
    setResponse('');
    setStreaming(true);

    for await (const chunk of ai.queryStream('Tell me about Abraham's journey to Canaan')) {
      switch (chunk.type) {
        case 'text':
          setResponse(prev => prev + chunk.content);
          break;
        case 'done':
          setStreaming(false);
          break;
        case 'error':
          console.error('Error:', chunk.content);
          setStreaming(false);
          break;
      }
    }
  };

  return (
    <div>
      <button onClick={handleQuery} disabled={streaming}>Ask</button>
      <p>{response}</p>
    </div>
  );
}

Provider-Level Streaming

Access provider stream() directly for lower-level control.

import { useState } from 'react';
import { LLMProvider } from '@inferagraph/core';
import { AnthropicProvider } from '@inferagraph/anthropic-provider';

const provider = new AnthropicProvider({ apiKey: 'sk-ant-...' });

function ProviderStream() {
  const [result, setResult] = useState('');

  const handleStream = async () => {
    setResult('');

    for await (const chunk of provider.stream({
      messages: [{ role: 'user', content: 'Describe the Exodus' }],
    })) {
      if (chunk.type === 'text')
        setResult(prev => prev + chunk.content);
    }
  };

  return (
    <div>
      <button onClick={handleStream}>Stream</button>
      <p>{result}</p>
    </div>
  );
}

// Custom provider: override stream() or use default fallback
class MyProvider extends LLMProvider {
  async *stream(request) {
    yield { type: 'text', content: 'Jacob had twelve sons...' };
    yield { type: 'done', content: '' };
  }
}