Map API

import { LLMLayerClient } from 'llmlayer';

// 1. Create a client
const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

// 2. Map a website
const response = await client.map({
  url: 'https://example.com'
});

// 3. See all discovered URLs
console.log(`Found ${response.links.length} pages:\n`);

for (const link of response.links) {
  console.log(`📄 ${link.title}`);
  console.log(`   ${link.url}\n`);
}

console.log(`Cost: $${response.cost}`);  // $0.002

const response = await client.map({
  url: 'https://docs.example.com'
});

console.log(`Discovered ${response.links.length} pages`);

// Show first 5 pages
response.links.slice(0, 5).forEach(link => {
  console.log(`${link.title} → ${link.url}`);
});

// Only find blog posts
const response = await client.map({
  url: 'https://example.com',
  search: 'blog'  // Only URLs containing "blog"
});

console.log(`Found ${response.links.length} blog pages`);

for (const link of response.links) {
  console.log(`${link.title} → ${link.url}`);
}

const response = await client.map({
  url: 'https://example.com',
  includeSubdomains: true  // Include blog.*, docs.*, api.*, etc.
});

console.log(`Found ${response.links.length} pages across all subdomains`);

// Group by subdomain
const bySubdomain = new Map();

for (const link of response.links) {
  const hostname = new URL(link.url).hostname;
  if (!bySubdomain.has(hostname)) {
    bySubdomain.set(hostname, []);
  }
  bySubdomain.get(hostname).push(link);
}

// Show breakdown
for (const [subdomain, links] of bySubdomain) {
  console.log(`\n${subdomain}: ${links.length} pages`);
}

const response = await client.map({
  url: 'https://example.com',
  ignoreSitemap: true  // Don't use sitemap.xml, crawl instead
});

console.log(`Crawled and found ${response.links.length} pages`);

const response = await client.map({
  url: 'https://large-site.com',
  limit: 100  // Stop after finding 100 URLs
});

console.log(`Limited to ${response.links.length} pages`);

const response = await client.map({
  url: 'https://example.com',
  timeoutMs: 30000  // Wait up to 30 seconds
});

console.log(`Found ${response.links.length} pages`);

import { LLMLayerClient } from 'llmlayer';

const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

async function planScrape(baseUrl: string, keyword: string) {
  console.log(`🗺️  Mapping ${baseUrl}...\n`);

  // 1. Map the site
  const mapResponse = await client.map({
    url: baseUrl,
    search: keyword  // Filter by keyword
  });

  console.log(`✅ Found ${mapResponse.links.length} matching pages`);
  console.log(`💰 Map cost: $${mapResponse.cost}\n`);

  // 2. Review what you found
  console.log('Pages to scrape:');
  mapResponse.links.forEach((link, i) => {
    console.log(`${i + 1}. ${link.title}`);
    console.log(`   ${link.url}`);
  });

  // 3. Calculate scraping cost
  const scrapeCost = mapResponse.links.length * 0.002;
  const totalCost = mapResponse.cost + scrapeCost;

  console.log(`\n💰 Cost Estimate:`);
  console.log(`   Map: $${mapResponse.cost}`);
  console.log(`   Scrape ${mapResponse.links.length} pages: $${scrapeCost.toFixed(3)}`);
  console.log(`   Total: $${totalCost.toFixed(3)}`);

  return mapResponse.links;
}

// Use it
const pages = await planScrape('https://docs.example.com', 'api');

// Now scrape only the pages you want
console.log('\n📥 Scraping selected pages...');
for (const page of pages.slice(0, 5)) {  // Scrape first 5
  const content = await client.scrape({
    url: page.url,
    formats: ['markdown']
  });
  console.log(`✅ Scraped: ${page.title}`);
}

import fs from 'fs';

async function generateSitemap(url: string) {
  console.log(`🗺️  Mapping ${url}...\n`);

  const response = await client.map({ url });

  console.log(`✅ Found ${response.links.length} pages\n`);

  // Create XML sitemap
  const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${response.links.map(link => `  <url>
    <loc>${link.url}</loc>
    <changefreq>weekly</changefreq>
    <priority>0.8</priority>
  </url>`).join('\n')}
</urlset>`;

  // Save to file
  fs.writeFileSync('sitemap.xml', sitemap);

  console.log('✅ Saved sitemap.xml');
  console.log(`📊 ${response.links.length} URLs included`);
  console.log(`💰 Cost: $${response.cost}`);

  return sitemap;
}

await generateSitemap('https://example.com');

async function auditWebsite(url: string) {
  console.log(`🔍 Auditing ${url}...\n`);

  const response = await client.map({ url });

  console.log(`Total pages: ${response.links.length}\n`);

  // Categorize by path
  const categories = {
    blog: [],
    docs: [],
    api: [],
    products: [],
    other: []
  };

  for (const link of response.links) {
    const path = new URL(link.url).pathname.toLowerCase();

    if (path.includes('/blog')) categories.blog.push(link);
    else if (path.includes('/doc')) categories.docs.push(link);
    else if (path.includes('/api')) categories.api.push(link);
    else if (path.includes('/product')) categories.products.push(link);
    else categories.other.push(link);
  }

  // Report
  console.log('📊 Content Breakdown:');
  console.log(`   Blog posts: ${categories.blog.length}`);
  console.log(`   Documentation: ${categories.docs.length}`);
  console.log(`   API pages: ${categories.api.length}`);
  console.log(`   Products: ${categories.products.length}`);
  console.log(`   Other: ${categories.other.length}`);

  // Find pages without titles
  const noTitle = response.links.filter(l => !l.title || l.title.trim() === '');
  if (noTitle.length > 0) {
    console.log(`\n⚠️  ${noTitle.length} pages missing titles:`);
    noTitle.slice(0, 5).forEach(l => console.log(`   - ${l.url}`));
  }

  // Longest/shortest paths
  const sorted = [...response.links].sort((a, b) =>
    new URL(a.url).pathname.length - new URL(b.url).pathname.length
  );

  console.log('\n📏 URL Analysis:');
  console.log(`   Shortest: ${new URL(sorted[0].url).pathname}`);
  console.log(`   Longest: ${new URL(sorted[sorted.length - 1].url).pathname}`);

  console.log(`\n💰 Audit cost: $${response.cost}`);

  return categories;
}

const audit = await auditWebsite('https://example.com');

import {
  LLMLayerClient,
  AuthenticationError,
  InvalidRequest,
  InternalServerError
} from 'llmlayer';

const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

async function robustMap(url: string, maxRetries = 3) {
  for (let attempt = 0; attempt < maxRetries; attempt++) {
    try {
      return await client.map({ url });

    } catch (error) {
      // Don't retry auth errors
      if (error instanceof AuthenticationError) {
        console.error('❌ Fix your API key');
        throw error;
      }

      // Don't retry invalid URLs
      if (error instanceof InvalidRequest) {
        console.error('❌ Invalid URL:', url);
        throw error;
      }

      // Retry server errors
      if (error instanceof InternalServerError) {
        const waitTime = Math.pow(2, attempt) * 1000;
        console.log(`⏳ Map failed. Waiting ${waitTime}ms...`);
        await new Promise(resolve => setTimeout(resolve, waitTime));

        if (attempt === maxRetries - 1) {
          console.error('❌ Max retries exceeded');
          throw error;
        }
        continue;
      }

      throw error;
    }
  }
}

// Usage
try {
  const response = await robustMap('https://example.com');
  console.log(`✅ Mapped ${response.links.length} pages`);
} catch (error) {
  console.error('Map failed:', error);
}