Bulk jobs

# 1. submit -> job id
JOB=$(curl -s https://api.crawl4ai.com/scrape/jobs \
  -H "Authorization: Bearer sk_live_..." -H "Content-Type: application/json" \
  -d '{"urls": ["https://a.com", "https://b.com"]}' | jq -r .job_id)

# 2. poll until done
until [ "$(curl -s https://api.crawl4ai.com/scrape/jobs/$JOB \
       -H "Authorization: Bearer sk_live_..." | jq -r .status)" = "done" ]; \
  do sleep 2; done

# 3. fetch results (NDJSON, one line per URL)
curl -s https://api.crawl4ai.com/scrape/jobs/$JOB/results \
  -H "Authorization: Bearer sk_live_..."

import time, requests

H = {"Authorization": "Bearer sk_live_..."}
base = "https://api.crawl4ai.com"

# 1. submit
job = requests.post(f"{base}/scrape/jobs", headers=H,
                    json={"urls": ["https://a.com", "https://b.com"]}).json()["job_id"]

# 2. poll until done
while requests.get(f"{base}/scrape/jobs/{job}", headers=H).json()["status"] != "done":
    time.sleep(2)

# 3. page through the results
after = 0
while True:
    lines = requests.get(f"{base}/scrape/jobs/{job}/results?after={after}",
                         headers=H).text.splitlines()
    if not lines: break
    for line in lines: print(json.loads(line)["url"])
    after += len(lines)

const H = { "Authorization": "Bearer sk_live_...", "Content-Type": "application/json" };
const base = "https://api.crawl4ai.com";

// 1. submit
const { job_id } = await (await fetch(`${base}/scrape/jobs`, {
  method: "POST", headers: H,
  body: JSON.stringify({ urls: ["https://a.com", "https://b.com"] }),
})).json();

// 2. poll until done
let status;
do {
  await new Promise(r => setTimeout(r, 2000));
  ({ status } = await (await fetch(`${base}/scrape/jobs/${job_id}`, { headers: H })).json());
} while (status !== "done");

// 3. fetch results (NDJSON)
const out = await (await fetch(`${base}/scrape/jobs/${job_id}/results`, { headers: H })).text();
console.log(out);