Scraper API

import { LLMLayerClient } from 'llmlayer';

// 1. Create a client
const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

// 2. Scrape a website
const response = await client.scrape({
  url: 'https://example.com/article',
  formats: ['markdown']  // Get clean markdown
});

// 3. Use the content
console.log(response.markdown);
console.log(`\nTitle: ${response.title}`);
console.log(`Cost: $${response.cost}`);  // $0.001

const response = await client.scrape({
  url: 'https://techcrunch.com/category/startups/',
  formats: ['markdown'],
  includeImages: true,   // Keep images
  includeLinks: true     // Keep hyperlinks
});

console.log(response.markdown);
console.log(`Cost: $${response.cost}`);  // $0.001
// Output: Clean markdown with the article content

// Text only - no images, no links
const textOnly = await client.scrape({
  url: 'https://example.com',
  formats: ['markdown'],
  includeImages: false,  // Remove all images
  includeLinks: false    // Remove all links
});

console.log(textOnly.markdown);
// Output: Pure text content

// Get clean content without navigation elements
const response = await client.scrape({
  url: 'https://example.com/article',
  formats: ['markdown'],
  mainContentOnly: true,  // Extract only main content
});

console.log(response.markdown);
console.log(`Cost: $${response.cost}`);  // $0.001
// Content without header, footer, sidebar, navigation

const response = await client.scrape({
  url: 'https://example.com',
  formats: ['html']
});

console.log(response.html);
console.log(`Cost: $${response.cost}`);  // $0.001
// Output: Full HTML content

// Parse it yourself if needed
const cheerio = require('cheerio');
const $ = cheerio.load(response.html);
const title = $('h1').first().text();
console.log(`Main heading: ${title}`);

import fs from 'fs';

const response = await client.scrape({
  url: 'https://example.com',
  formats: ['screenshot']
});

// The screenshot is base64-encoded
console.log('Screenshot captured!');
console.log(`Cost: $${response.cost}`);  // $0.001

// Save to file
const buffer = Buffer.from(response.screenshot, 'base64');
fs.writeFileSync('page-screenshot.png', buffer);

console.log('Saved to page-screenshot.png');

// Scrape sites with bot protection
const response = await client.scrape({
  url: 'https://protected-site.com',
  formats: ['markdown'],
  advancedProxy: true,  // Enable advanced proxy (+$0.004/request)
});

console.log(`✅ Successfully scraped: ${response.title}`);
console.log(`💰 Total cost: $${response.cost}`);
// Cost includes base ($0.001) + proxy ($0.004) = $0.005

import fs from 'fs';

// Get all formats at once!
const response = await client.scrape({
  url: 'https://example.com/article',
  formats: ['markdown', 'html', 'screenshot'],
  includeImages: true,
  includeLinks: true
});

// Save markdown
fs.writeFileSync('article.md', response.markdown);

// Save HTML
fs.writeFileSync('article.html', response.html);

// Save screenshot
const screenshotBuffer = Buffer.from(response.screenshot, 'base64');
fs.writeFileSync('article.png', screenshotBuffer);

console.log('✅ Saved all formats!');
console.log(`📊 Title: ${response.title}`);
console.log(`💰 Total cost: $${response.cost}`);  // $0.003 (3 formats)

// Best of everything: clean content from protected sites, all formats
const response = await client.scrape({
  url: 'https://example.com/article',
  formats: ['markdown', 'html', 'screenshot'],
  mainContentOnly: true,    // Clean content
  advancedProxy: true,      // Better success rate
});

console.log(`✅ ${response.title}`);
console.log(`   Clean article: ${response.markdown?.length || 0} chars`);
console.log(`💰 Total cost: $${response.cost}`);
// Cost: (3 formats × $0.001) + $0.004 proxy = $0.007

import fs from 'fs';

async function archiveArticle(url: string, title: string) {
  console.log(`📥 Archiving: ${title}`);

  const response = await client.scrape({
    url,
    formats: ['markdown', 'screenshot'],  // Text + visual
    mainContentOnly: true  // Clean content
  });

  // Create safe filename
  const filename = title
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-|-$/g, '');

  // Save markdown for reading/searching
  fs.writeFileSync(`articles/${filename}.md`, response.markdown);

  // Save screenshot for visual archive
  const screenshotBuffer = Buffer.from(response.screenshot, 'base64');
  fs.writeFileSync(`articles/${filename}.png`, screenshotBuffer);

  console.log(`✅ Saved to articles/${filename}.*`);
  console.log(`   Cost: $${response.cost}\n`);  // $0.002 (2 formats)

  return response;
}

// Archive multiple articles
const articles = [
  { url: 'https://...', title: 'AI Breakthrough' },
  { url: 'https://...', title: 'Climate Report' },
  { url: 'https://...', title: 'Tech News' }
];

for (const article of articles) {
  await archiveArticle(article.url, article.title);
  await new Promise(resolve => setTimeout(resolve, 100)); // Rate limiting
}

console.log('📚 Archive complete!');

import fs from 'fs';
import crypto from 'crypto';

async function checkForChanges(url: string, previousHash: string | null) {
  // Get screenshot
  const response = await client.scrape({
    url,
    formats: ['screenshot']
  });

  // Calculate hash of screenshot
  const screenshotBuffer = Buffer.from(response.screenshot, 'base64');
  const currentHash = crypto
    .createHash('sha256')
    .update(screenshotBuffer)
    .digest('hex');

  // Compare with previous
  if (previousHash === null) {
    console.log('🆕 First capture - no previous screenshot');
    fs.writeFileSync(`screenshots/${currentHash}.png`, screenshotBuffer);
    return { changed: false, hash: currentHash };
  }

  if (currentHash !== previousHash) {
    console.log('⚠️  CHANGE DETECTED!');
    fs.writeFileSync(`screenshots/${currentHash}.png`, screenshotBuffer);
    return { changed: true, hash: currentHash };
  }

  console.log('✅ No changes detected');
  return { changed: false, hash: currentHash };
}

// Monitor a website
let lastHash: string | null = null;

// Check every hour
setInterval(async () => {
  const result = await checkForChanges('https://example.com/pricing', lastHash);
  lastHash = result.hash;

  if (result.changed) {
    // Send alert (email, Slack, etc.)
    console.log('🚨 Send alert: Pricing page changed!');
  }
}, 3600000); // 1 hour

async function scrapeProtectedProduct(url: string) {
  console.log(`🛒 Scraping product from ${url}\n`);

  try {
    const response = await client.scrape({
      url,
      formats: ['markdown', 'html'],
      advancedProxy: true,      // Bypass bot protection
      mainContentOnly: true,    // Clean product descriptions
    });

    console.log(`✅ ${response.title}`);
    console.log(`   Content: ${response.markdown?.length || 0} chars`);
    console.log(`💰 Cost: $${response.cost}`);
    // Cost: (2 formats × $0.001) + $0.004 proxy = $0.006

    return response;

  } catch (error) {
    console.error(`❌ Failed: ${error.message}`);
    return null;
  }
}

await scrapeProtectedProduct('https://shop.example.com/product/123');

import {
  LLMLayerClient,
  AuthenticationError,
  InvalidRequest,
  InternalServerError
} from 'llmlayer';

const client = new LLMLayerClient({
  apiKey: process.env.LLMLAYER_API_KEY
});

async function robustScrape(url: string, maxRetries = 3) {
  for (let attempt = 0; attempt < maxRetries; attempt++) {
    try {
      return await client.scrape({
        url,
        formats: ['markdown']
      });

    } catch (error) {
      // Don't retry authentication errors
      if (error instanceof AuthenticationError) {
        console.error('❌ Fix your API key');
        throw error;
      }

      // Don't retry invalid URLs
      if (error instanceof InvalidRequest) {
        console.error('❌ Invalid URL:', url);
        throw error;
      }

      // Retry server errors
      if (error instanceof InternalServerError) {
        const waitTime = Math.pow(2, attempt) * 1000;
        console.log(`⏳ Scraping failed. Waiting ${waitTime}ms...`);
        await new Promise(resolve => setTimeout(resolve, waitTime));

        // Last attempt
        if (attempt === maxRetries - 1) {
          console.error('❌ Max retries exceeded');
          throw error;
        }
        continue;
      }

      throw error;
    }
  }
}

// Usage
try {
  const response = await robustScrape('https://example.com');
  console.log(`✅ Scraped: ${response.title}`);
  console.log(response.markdown);
} catch (error) {
  console.error('Scraping failed:', error);
}