Crawl API

import { LLMLayerClient } from 'llmlayer';

const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

// Crawl with real-time streaming
for await (const frame of client.crawlStream({
  url: 'https://docs.example.com',
  maxPages: 5,         // Crawl up to 5 pages
})) {

  // Each frame has a type
  if (frame.type === 'page') {
    const page = frame.page;

    if (page.success) {
      console.log(`✅ ${page.title}`);
      console.log(`   URL: ${page.final_url}`);
      console.log(`   Content length: ${page.markdown?.length || 0} chars\n`);
    } else {
      console.log(`❌ Failed: ${page.final_url}`);
      console.log(`   Error: ${page.error}\n`);
    }
  }

  else if (frame.type === 'usage') {
    console.log(`\n💰 Billing:`);
    console.log(`   Successful pages: ${frame.billed_count}`);
    console.log(`   Cost: $${frame.cost}`);
  }

  else if (frame.type === 'done') {
    console.log(`\n⏱️  Completed in ${frame.response_time}s`);
  }
}

// Depth 1: Only crawl pages linked from the homepage
for await (const frame of client.crawlStream({
  url: 'https://example.com',
  maxPages: 10,
  maxDepth: 1,  // Only 1 level deep
})) {
  if (frame.type === 'page' && frame.page.success) {
    console.log(`Crawled: ${frame.page.title}`);
  }
}

// Depth 2: Crawl homepage + pages it links to + pages those link to
for await (const frame of client.crawlStream({
  url: 'https://example.com',
  maxPages: 20,
  maxDepth: 2,  // 2 levels deep
})) {
  if (frame.type === 'page' && frame.page.success) {
    console.log(`Crawled: ${frame.page.title}`);
  }
}

// Get clean content without navigation elements
for await (const frame of client.crawlStream({
  url: 'https://blog.example.com',
  maxPages: 20,
  mainContentOnly: true,  // Extract only main content
})) {

  if (frame.type === 'page' && frame.page.success) {
    const page = frame.page;
    console.log(`✅ ${page.title}`);
    console.log(`   Clean content: ${page.markdown?.length || 0} chars`);
    // Content without header, footer, sidebar, navigation
  }
}

// Crawl sites with bot protection
for await (const frame of client.crawlStream({
  url: 'https://protected-site.com',
  maxPages: 10,
  advancedProxy: true,  // Enable advanced proxy (+$0.004/page)
})) {

  if (frame.type === 'page' && frame.page.success) {
    console.log(`✅ Successfully scraped: ${frame.page.title}`);
  }

  else if (frame.type === 'usage') {
    console.log(`💰 Total cost: $${frame.cost}`);
    // Cost includes base ($0.001) + proxy ($0.004) per successful page
  }
}

// Best of both worlds: clean content from protected sites
for await (const frame of client.crawlStream({
  url: 'https://protected-news-site.com',
  maxPages: 30,
  mainContentOnly: true,    // Clean content
  advancedProxy: true,      // Better success rate
  maxDepth: 2
})) {

  if (frame.type === 'page' && frame.page.success) {
    const page = frame.page;
    console.log(`✅ ${page.title}`);
    console.log(`   Clean article: ${page.markdown?.length || 0} chars`);
  }

  else if (frame.type === 'usage') {
    console.log(`\n📊 Results:`);
    console.log(`   Successful: ${frame.billed_count} pages`);
    console.log(`   Cost: $${frame.cost} (includes proxy fees)`);
  }
}

for await (const frame of client.crawlStream({
  url: 'https://example.com',
  maxPages: 20,
  includeSubdomains: true,  // Crawl blog.*, docs.*, etc.
})) {

  if (frame.type === 'page' && frame.page.success) {
    const hostname = new URL(frame.page.final_url).hostname;
    console.log(`${hostname}: ${frame.page.title}`);
  }
}