quickstart

get up and running with crawlbrulee in under 5 minutes. you’ll scrape a page, map a site, and run an async job.

prerequisites

you need an api key from the crawlbrulee dashboard . see authentication for the full setup.

set it as an environment variable:


export CRAWLBRULEE_API_KEY="your-api-key"

every request uses this key via the Authorization: Bearer header. the examples below read it from the environment automatically.

scrape a page

hit POST /api/scrape with a url and tell the api what you want back. here we ask for the cleaned markdown and all links on the page.

curl


curl -X POST https://api.crawlbrulee.com/api/scrape \
  -H "Authorization: Bearer $CRAWLBRULEE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "extract": {
      "markdown": true,
      "links": true
    }
  }'

js/ts


const response = await fetch('https://api.crawlbrulee.com/api/scrape', {
  method: 'POST',
  headers: {
    Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    url: 'https://example.com',
    extract: {
      markdown: true,
      links: true,
    },
  }),
})
 
const data = await response.json()
console.log(data)

js/ts sdk


import { Crawlbrulee } from '@crawlbrulee/sdk'
 
// reads CRAWLBRULEE_API_KEY from the environment
const crawlbrulee = Crawlbrulee.fromEnv()
 
const page = await crawlbrulee.scrape({
  url: 'https://example.com',
  extract: { markdown: true, links: true },
})
 
console.log(page.markdown)

python


import os
import requests
 
response = requests.post(
    "https://api.crawlbrulee.com/api/scrape",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
        "Content-Type": "application/json",
    },
    json={
        "url": "https://example.com",
        "extract": {
            "markdown": True,
            "links": True,
        },
    },
)
 
data = response.json()
print(data)

python sdk


from crawlbrulee import Crawlbrulee, ScrapeExtract
 
# reads CRAWLBRULEE_API_KEY from the environment
client = Crawlbrulee.from_env()
 
page = client.scrape(
    url="https://example.com",
    extract=ScrapeExtract(markdown=True, links=True),
)
 
print(page.markdown)

response:


{
  "url": "https://example.com",
  "requested_url": "https://example.com",
  "content_type": "text/html",
  "markdown": "# Example Domain\n\nThis domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.\n\n[More information...](https://www.iana.org/domains/example)",
  "cleaned_html": "<h1>Example Domain</h1><p>This domain is for use in illustrative examples...</p>",
  "links": [
    {
      "text": "More information...",
      "href": "https://www.iana.org/domains/example",
      "internal": false
    }
  ],
  "metadata": {
    "title": "Example Domain",
    "og_title": "Example Domain",
    "favicon_url": "https://example.com/favicon.ico"
  },
  "response_meta": {
    "usage": {
      "credits": 1,
      "engine": "http",
      "proxy": "basic",
      "screenshot_slices": 0
    }
  }
}

a fully cached result is free. if crawlbrulee already has a fresh result for a url, you get it back instantly at zero base cost; a newly produced screenshot-slice variant adds one flat credit. control freshness with the cache.max_age parameter. check for response_meta.usage.engine: "cache" to see whether the base result came from cache.

map a site

POST /api/map discovers every url on a site by combining sitemap parsing with in-page link discovery. pass a root url and get back a full link inventory.

curl


curl -X POST https://api.crawlbrulee.com/api/map \
  -H "Authorization: Bearer $CRAWLBRULEE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com"
  }'

js/ts


const response = await fetch('https://api.crawlbrulee.com/api/map', {
  method: 'POST',
  headers: {
    Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    url: 'https://example.com',
  }),
})
 
const data = await response.json()
console.log(data.links.length, 'URLs found')

js/ts sdk


import { Crawlbrulee } from '@crawlbrulee/sdk'
 
// reads CRAWLBRULEE_API_KEY from the environment
const crawlbrulee = Crawlbrulee.fromEnv()
 
const data = await crawlbrulee.map({ url: 'https://example.com' })
 
console.log(data.links.length, 'URLs found')

python


import os
import requests
 
response = requests.post(
    "https://api.crawlbrulee.com/api/map",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
        "Content-Type": "application/json",
    },
    json={
        "url": "https://example.com",
    },
)
 
data = response.json()
print(f"{len(data['links'])} URLs found")

python sdk


from crawlbrulee import Crawlbrulee
 
# reads CRAWLBRULEE_API_KEY from the environment
client = Crawlbrulee.from_env()
 
data = client.map(url="https://example.com")
 
print(f"{len(data.links)} URLs found")

response:


{
  "links": [
    { "url": "https://example.com/about" },
    { "url": "https://example.com/blog" },
    { "url": "https://example.com/docs" },
    { "url": "https://example.com/pricing" },
    { "url": "https://example.com/blog/getting-started" }
  ],
  "response_meta": {
    "usage": {
      "credits": 1,
      "engine": "http",
      "proxy": "basic"
    },
    "pagination": {
      "page": 1,
      "limit": 5000,
      "total": 5,
      "total_pages": 1,
      "has_more": false
    },
    "truncation": {
      "storage_capped": false,
      "response_capped": false,
      "total_before_max_urls": 5,
      "total_detected_before_storage_cap": 5,
      "discovery_capped": false,
      "sitemaps_skipped": 0,
      "discovery_cap_reason": null
    }
  }
}

a map returns up to 5,000 urls by default — raise max_urls (up to 100,000) for a bigger site. you can also filter by link type (internal, internal_subdomains, external) and paginate through large sites. see the full map guide for all options.

async scraping

for batch pipelines or pages that take a while to render, use the async flow. you submit a job, poll for status, and fetch the result when it’s done.

don’t want to poll? pass a webhook: { url } object on submit and we POST a signed payload to that url once the job reaches a terminal state — see webhooks.

submit the job

POST /api/scrape/async takes the same body as the sync endpoint but returns immediately with a job_id.

curl


curl -X POST https://api.crawlbrulee.com/api/scrape/async \
  -H "Authorization: Bearer $CRAWLBRULEE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "extract": {
      "markdown": true,
      "links": true
    }
  }'

js/ts


const response = await fetch('https://api.crawlbrulee.com/api/scrape/async', {
  method: 'POST',
  headers: {
    Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    url: 'https://example.com',
    extract: {
      markdown: true,
      links: true,
    },
  }),
})
 
const { job_id } = await response.json()
console.log('Job submitted:', job_id)

js/ts sdk


import { Crawlbrulee } from '@crawlbrulee/sdk'
 
// reads CRAWLBRULEE_API_KEY from the environment
const crawlbrulee = Crawlbrulee.fromEnv()
 
const { job_id } = await crawlbrulee.scrapeAsync({
  url: 'https://example.com',
  extract: { markdown: true, links: true },
})
 
console.log('Job submitted:', job_id)

python


import os
import requests
 
response = requests.post(
    "https://api.crawlbrulee.com/api/scrape/async",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
        "Content-Type": "application/json",
    },
    json={
        "url": "https://example.com",
        "extract": {
            "markdown": True,
            "links": True,
        },
    },
)
 
data = response.json()
job_id = data["job_id"]
print(f"Job submitted: {job_id}")

python sdk


from crawlbrulee import Crawlbrulee, ScrapeExtract
 
# reads CRAWLBRULEE_API_KEY from the environment
client = Crawlbrulee.from_env()
 
job = client.scrape_async(
    url="https://example.com",
    extract=ScrapeExtract(markdown=True, links=True),
)
 
print(f"Job submitted: {job.job_id}")

response (http 202):


{
  "job_id": "683a1f2b4c5d6e7f8a9b0c1d"
}

poll for status

GET /api/scrape/status/{job_id} returns the current state: pending, running, done, or failed. keep polling until status is done or failed.

curl


curl https://api.crawlbrulee.com/api/scrape/status/683a1f2b4c5d6e7f8a9b0c1d \
  -H "Authorization: Bearer $CRAWLBRULEE_API_KEY"

js/ts


const statusRes = await fetch(
  `https://api.crawlbrulee.com/api/scrape/status/${job_id}`,
  {
    headers: {
      Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    },
  },
)
 
const status = await statusRes.json()
console.log('Status:', status.status)

js/ts sdk


import { Crawlbrulee } from '@crawlbrulee/sdk'
 
// reads CRAWLBRULEE_API_KEY from the environment
const crawlbrulee = Crawlbrulee.fromEnv()
 
const status = await crawlbrulee.getScrapeStatus(job_id)
 
console.log('Status:', status.status)

python


status_response = requests.get(
    f"https://api.crawlbrulee.com/api/scrape/status/{job_id}",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
    },
)
 
status = status_response.json()
print(f"Status: {status['status']}")

python sdk


from crawlbrulee import Crawlbrulee
 
# reads CRAWLBRULEE_API_KEY from the environment
client = Crawlbrulee.from_env()
 
status = client.get_scrape_status(job_id)
 
print(f"Status: {status.status}")

response:


{
  "job_id": "683a1f2b4c5d6e7f8a9b0c1d",
  "status": "running",
  "created_at": "2025-01-15T10:30:00.000Z"
}

get the result

once status is done, fetch the full scrape result from GET /api/scrape/result/{job_id}. the response shape is the same as the sync endpoint.

curl


curl https://api.crawlbrulee.com/api/scrape/result/683a1f2b4c5d6e7f8a9b0c1d \
  -H "Authorization: Bearer $CRAWLBRULEE_API_KEY"

js/ts


const resultRes = await fetch(
  `https://api.crawlbrulee.com/api/scrape/result/${job_id}`,
  {
    headers: {
      Authorization: `Bearer ${process.env.CRAWLBRULEE_API_KEY}`,
    },
  },
)
 
const result = await resultRes.json()
console.log(result)

js/ts sdk


import { Crawlbrulee } from '@crawlbrulee/sdk'
 
// reads CRAWLBRULEE_API_KEY from the environment
const crawlbrulee = Crawlbrulee.fromEnv()
 
const result = await crawlbrulee.getScrapeResult(job_id)
 
console.log(result)

python


result_response = requests.get(
    f"https://api.crawlbrulee.com/api/scrape/result/{job_id}",
    headers={
        "Authorization": f"Bearer {os.environ['CRAWLBRULEE_API_KEY']}",
    },
)
 
result = result_response.json()
print(result)

python sdk


from crawlbrulee import Crawlbrulee
 
# reads CRAWLBRULEE_API_KEY from the environment
client = Crawlbrulee.from_env()
 
result = client.get_scrape_result(job_id)
 
print(result)

response:


{
  "url": "https://example.com",
  "requested_url": "https://example.com",
  "content_type": "text/html",
  "markdown": "# Example Domain\n\nThis domain is for use in illustrative examples ...",
  "cleaned_html": "<h1>Example Domain</h1><p>This domain is for use in illustrative examples...</p>",
  "links": [
    {
      "href": "https://www.iana.org/domains/example",
      "text": "More information...",
      "internal": false
    }
  ],
  "metadata": {
    "title": "Example Domain"
  },
  "response_meta": {
    "usage": {
      "credits": 1,
      "engine": "http",
      "proxy": "basic",
      "screenshot_slices": 0
    }
  }
}

async is ideal for batch pipelines. submit hundreds of jobs, poll in parallel, and collect results as they finish. credits are reserved on submit, released if a job fails, and charged at the delivered engine on success. a cache hit has a 0-credit base.

what’s next

you’ve got the basics down. here’s where to go deeper:

core concepts — how scraping, caching, and credits work together.
extraction options — control exactly what data you pull from each page.
screenshots — capture full-page or viewport screenshots.
credits & pricing — understand what operations cost and how to optimize.
site mapping — discover every url on a domain before you scrape.
sdks & tools — official js/ts + Python sdks, a cli, and an mcp server.