rate limits

// 429 = rate or usage limit, 503 = transient failure on our end. Both are safe
// to retry with the request unchanged.
const RETRYABLE_STATUSES = new Set([429, 503])
 
async function fetchWithRetry(url, options, maxRetries = 5) {
  let attempt = 0
 
  while (attempt < maxRetries) {
    const response = await fetch(url, options)
 
    if (!RETRYABLE_STATUSES.has(response.status)) {
      return response
    }
 
    attempt++
    if (attempt >= maxRetries) {
      throw new Error(`Still getting ${response.status} after ${maxRetries} attempts`)
    }
 
    const body = await response.json()
    const retryAfterMs = body?.details?.retry_after_ms
 
    let delay
    if (retryAfterMs) {
      // Server told us exactly how long to wait (429 only)
      delay = retryAfterMs
    } else {
      // Exponential backoff: 1s, 2s, 4s, 8s... capped at 30s. Every 503 lands here.
      delay = Math.min(1000 * 2 ** (attempt - 1), 30000)
    }
 
    // Add jitter (0-500ms) to avoid thundering herd
    delay += Math.random() * 500
 
    console.log(`Got ${response.status}. Retrying in ${Math.round(delay)}ms (attempt ${attempt}/${maxRetries})`)
    await new Promise((resolve) => setTimeout(resolve, delay))
  }
}
 
// Usage
const response = await fetchWithRetry('https://api.crawlbrulee.com/api/scrape', {
  method: 'POST',
  headers: {
    Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    url: 'https://example.com',
    extract: { markdown: true },
  }),
})
 
const data = await response.json()
console.log(data)

import os
import time
import random
import requests
 
 
# 429 = rate or usage limit, 503 = transient failure on our end. Both are safe
# to retry with the request unchanged.
RETRYABLE_STATUSES = {429, 503}
 
 
def fetch_with_retry(url, max_retries=5, **kwargs):
    attempt = 0
 
    while attempt < max_retries:
        response = requests.request(url=url, **kwargs)
 
        if response.status_code not in RETRYABLE_STATUSES:
            return response
 
        attempt += 1
        if attempt >= max_retries:
            raise Exception(f"Still getting {response.status_code} after {max_retries} attempts")
 
        body = response.json()
        retry_after_ms = body.get("details", {}).get("retry_after_ms")
 
        if retry_after_ms:
            # Server told us exactly how long to wait (429 only)
            delay = retry_after_ms / 1000
        else:
            # Exponential backoff: 1s, 2s, 4s, 8s... capped at 30s. Every 503 lands here.
            delay = min(2 ** (attempt - 1), 30)
 
        # Add jitter (0-500ms) to avoid thundering herd
        delay += random.uniform(0, 0.5)
 
        print(f"Got {response.status_code}. Retrying in {delay:.1f}s (attempt {attempt}/{max_retries})")
        time.sleep(delay)
 
 
# usage
response = fetch_with_retry(
    url="https://api.crawlbrulee.com/api/scrape",
    method="POST",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
        "Content-Type": "application/json",
    },
    json={
        "url": "https://example.com",
        "extract": {"markdown": True},
    },
)
 
data = response.json()
print(data)