Scrape many pages

# -N streams each result line as it lands
curl -N https://api.crawl4ai.com/scrape/batch \
  -H "Authorization: Bearer sk_live_..." \
  -H "Content-Type: application/json" \
  -d '{"urls": ["https://a.com", "https://b.com"], "format": "md"}'

import json, requests

with requests.post(
    "https://api.crawl4ai.com/scrape/batch",
    headers={"Authorization": "Bearer sk_live_..."},
    json={"urls": ["https://a.com", "https://b.com"], "format": "md"},
    stream=True,
) as r:
    for line in r.iter_lines():
        if line:
            row = json.loads(line)
            print(row["url"], "ok" if row["ok"] else row["error"])

const res = await fetch("https://api.crawl4ai.com/scrape/batch", {
  method: "POST",
  headers: { "Authorization": "Bearer sk_live_...", "Content-Type": "application/json" },
  body: JSON.stringify({ urls: ["https://a.com", "https://b.com"], format: "md" }),
});

// read the NDJSON stream line by line
const reader = res.body.getReader(), dec = new TextDecoder();
let buf = "";
for (;;) {
  const { value, done } = await reader.read();
  if (done) break;
  buf += dec.decode(value, { stream: true });
  let nl;
  while ((nl = buf.indexOf("\n")) >= 0) {
    const line = buf.slice(0, nl); buf = buf.slice(nl + 1);
    if (line.trim()) {
      const row = JSON.parse(line);
      console.log(row.url, row.ok ? "ok" : row.error);
    }
  }
}