const API_KEY = process.env.CRAWLBRULEE_API_KEY
 
async function scrape(url) {
  const res = await fetch('https://api.crawlbrulee.com/api/scrape', {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({ url, extract: { markdown: true } }),
  })
 
  if (!res.ok) {
    const error = await res.json()
 
    switch (error.name) {
      case 'too_many_requests': {
        const retryMs = error.details?.retry_after_ms ?? 10_000
        console.log(`Rate limited. Retrying in ${retryMs}ms...`)
        await new Promise((r) => setTimeout(r, retryMs))
        return scrape(url) // retry once
      }
 
      case 'service_unavailable': {
        // Transient on our side — the key is fine, so retry it unchanged
        console.log('Service temporarily unavailable. Retrying in 2s...')
        await new Promise((r) => setTimeout(r, 2_000))
        return scrape(url) // retry once
      }
 
      case 'usage_allocation_error':
        console.error(`Usage limit hit: ${error.details?.reason}`)
        throw new Error('Usage limit reached — upgrade your plan')
 
      case 'invalid_credentials':
        throw new Error('Bad API key — check your token')
 
      case 'target_unreachable':
        // We could not reach the site at all. Not billed; try again later.
        throw new Error('Target site unreachable — try again later')
 
      default:
        throw new Error(`API error [${error.name}]: ${error.message}`)
    }
  }
 
  const page = await res.json()
 
  // A 200 from us can hold the site's error page. Check the page's own status.
  if (page.page_status_code >= 400) {
    console.warn(`The site answered ${page.page_status_code} for ${page.url}`)
  }
 
  return page
}

import os
import time
import requests
 
API_KEY = os.environ["CRAWLBRULEE_API_KEY"]
 
def scrape(url: str) -> dict:
    res = requests.post(
        "https://api.crawlbrulee.com/api/scrape",
        headers={
            "Authorization": f"Bearer {API_KEY}",
            "Content-Type": "application/json",
        },
        json={"url": url, "extract": {"markdown": True}},
    )
 
    if not res.ok:
        error = res.json()
        name = error.get("name")
        details = error.get("details", {})
 
        if name == "too_many_requests":
            retry_ms = details.get("retry_after_ms", 10_000)
            print(f"Rate limited. Retrying in {retry_ms}ms...")
            time.sleep(retry_ms / 1000)
            return scrape(url)  # retry once
 
        if name == "service_unavailable":
            # Transient on our side — the key is fine, so retry it unchanged
            print("Service temporarily unavailable. Retrying in 2s...")
            time.sleep(2)
            return scrape(url)  # retry once
 
        if name == "usage_allocation_error":
            reason = details.get("reason")
            raise Exception(f"Usage limit hit: {reason} — upgrade your plan")
 
        if name == "invalid_credentials":
            raise Exception("Bad API key — check your token")
 
        if name == "target_unreachable":
            # We could not reach the site at all. Not billed; try again later.
            raise Exception("Target site unreachable — try again later")
 
        raise Exception(f"API error [{name}]: {error.get('message')}")
 
    page = res.json()
 
    # A 200 from us can hold the site's error page. Check the page's own status.
    if page.get("page_status_code", 200) >= 400:
        print(f"The site answered {page['page_status_code']} for {page['url']}")
 
    return page