Search + Scrape Pipeline

import { LLMLayerClient } from 'llmlayer';

const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY,
});

const search = await client.searchWeb({
  query: 'latest research on LLM evaluation',
  searchType: 'general',
  domainFilter: ['arxiv.org', 'openai.com', '-reddit.com'],
});

const pages = [];

for (const result of search.results.slice(0, 3)) {
  const url = result.link;
  if (typeof url !== 'string') continue;

  const page = await client.scrape(url, {
    formats: ['markdown'],
    mainContentOnly: true,
  });

  pages.push({
    title: page.title,
    url: page.url,
    markdown: page.markdown,
  });
}

const context = pages
  .map((page) => `Source: ${page.title}\nURL: ${page.url}\n${page.markdown}`)
  .join('\n\n---\n\n');

const answer = await client.answer({
  query: `Use this context to summarize the main LLM evaluation trends:\n\n${context}`,
  model: 'llmlayer-web',
  returnSources: true,
});

console.log(answer.answer);

from llmlayer import LLMLayerClient

client = LLMLayerClient(api_key="YOUR_LLMLAYER_API_KEY")

search = client.search_web(
    query="latest research on LLM evaluation",
    search_type="general",
    domain_filter=["arxiv.org", "openai.com", "-reddit.com"],
)

pages = []

for result in search.results[:3]:
    url = result.get("link")
    if not isinstance(url, str):
        continue

    page = client.scrape(
        url,
        formats=["markdown"],
        main_content_only=True,
    )

    pages.append(
        {
            "title": page.title,
            "url": page.url,
            "markdown": page.markdown,
        }
    )

context = "\n\n---\n\n".join(
    f"Source: {page['title']}\nURL: {page['url']}\n{page['markdown']}"
    for page in pages
)

answer = client.answer(
    query=f"Use this context to summarize the main LLM evaluation trends:\n\n{context}",
    model="llmlayer-web",
    return_sources=True,
)

print(answer.answer)