const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
input: {
mode: 'scrape',
data: {
urls: [
{
url: 'https://example.com/products/anvil',
itemId: 'sku-1',
meta: {category: 'tools'}
},
{url: 'https://example.com/products/hammer', itemId: 'sku-2'}
],
formats: {markdown: true, html: true},
sharedParams: {mainContentOnly: true}
}
}
})
};
fetch('https://api.context.dev/v1/batch/submit', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"id": "batch_9f2c8a",
"status": "queued",
"mode": "scrape",
"tags": [
"docs"
],
"crawl": {
"source": {
"type": "start_url",
"url": "https://example.com/docs"
},
"max_pages": 500,
"max_depth": 0,
"follow_subdomains": true,
"url_pattern": "^https://example\\.com/docs/",
"exclude_url_pattern": "/tag/|/author/",
"languages": "same_as_start_url"
},
"input": {
"reserved": 24817,
"reserved_is_ceiling": false,
"submitted": 25000,
"duplicates": 183,
"invalid": 0
},
"credits": {
"reserved": 24817
},
"created_at": "<string>",
"invalid_urls": [
{
"url": "<string>",
"reason": "Must be a public http:// or https:// URL"
}
],
"cache_metadata": {
"status": "hit",
"age_ms": 1
},
"formats": {
"html": true,
"markdown": true
},
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"webhook_secret": "<string>",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"message": "<string>",
"error_code": "RATE_LIMITED",
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"error_data": {
"is_authenticated": true,
"limit_type": "concurrency",
"limit": 1,
"remaining": 1,
"retry_after_seconds": 1,
"credits_charged": 0,
"should_retry": false,
"retry_condition": "available_concurrency",
"recovery_options": [
"wait_for_in_flight_requests"
],
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1
},
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}Batches
Submit a batch
Scrape up to 100,000 URLs, or crawl a site, asynchronously. Poll the batch ID or receive a webhook when it finishes.
POST
/
batch
/
submit
const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
input: {
mode: 'scrape',
data: {
urls: [
{
url: 'https://example.com/products/anvil',
itemId: 'sku-1',
meta: {category: 'tools'}
},
{url: 'https://example.com/products/hammer', itemId: 'sku-2'}
],
formats: {markdown: true, html: true},
sharedParams: {mainContentOnly: true}
}
}
})
};
fetch('https://api.context.dev/v1/batch/submit', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"id": "batch_9f2c8a",
"status": "queued",
"mode": "scrape",
"tags": [
"docs"
],
"crawl": {
"source": {
"type": "start_url",
"url": "https://example.com/docs"
},
"max_pages": 500,
"max_depth": 0,
"follow_subdomains": true,
"url_pattern": "^https://example\\.com/docs/",
"exclude_url_pattern": "/tag/|/author/",
"languages": "same_as_start_url"
},
"input": {
"reserved": 24817,
"reserved_is_ceiling": false,
"submitted": 25000,
"duplicates": 183,
"invalid": 0
},
"credits": {
"reserved": 24817
},
"created_at": "<string>",
"invalid_urls": [
{
"url": "<string>",
"reason": "Must be a public http:// or https:// URL"
}
],
"cache_metadata": {
"status": "hit",
"age_ms": 1
},
"formats": {
"html": true,
"markdown": true
},
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"webhook_secret": "<string>",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"message": "<string>",
"error_code": "RATE_LIMITED",
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"error_data": {
"is_authenticated": true,
"limit_type": "concurrency",
"limit": 1,
"remaining": 1,
"retry_after_seconds": 1,
"credits_charged": 0,
"should_retry": false,
"retry_condition": "available_concurrency",
"recovery_options": [
"wait_for_in_flight_requests"
],
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1
},
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}{
"request_id": "3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91",
"message": "<string>",
"error_code": "PROFILE_NOT_FOUND",
"error_data": {
"is_authenticated": true,
"should_retry": false,
"retry_condition": "sufficient_credits",
"info_url": "https://context.dev/dashboard/subscription",
"credits_remaining": 1,
"credits_required": 1,
"is_credits_required_estimated": true,
"credits_charged": 1,
"next_refill_at": "2023-11-07T05:31:56Z",
"next_refill_credits": 123
},
"required_permission": "logs:read",
"key_metadata": {
"credits_consumed": 123,
"credits_remaining": 123
}
}From 1 credit per page
Requires
batches:write permission. See the guide for examples and usage.Authorizations
Send Authorization: Bearer <API_KEY>. Keys have full access unless restricted to scopes.
Headers
Unique key per submission. Retrying with the same key and body returns the original batch; a different body returns 409.
Maximum string length:
200Body
application/json
Create an asynchronous scraping job.
Scrape a list of up to 100,000 URLs.
- Scrape
- Crawl
Show child attributes
Show child attributes
Legacy URL notified when the batch finishes. Preserves one best-effort attempt. Cannot be combined with webhook.
Where to send the batch's final-status event. Omit retry for one attempt; {} uses the default retry schedule.
Show child attributes
Show child attributes
Tags stored on the batch. Filter the batch list by them later.
Maximum array length:
20Required string length:
1 - 50Example:
["docs", "competitor"]
Access policy IDs to apply to every page. A URL must pass every policy.
Example:
["pol_3f9c2a7e5b8d4c1f9a6e0b2d7c4f8a1e"]
Response
Batch accepted. Read progress and results from GET /batch/{batch_id}.
Batch ID. Poll GET /batch/{batch_id} with it.
Example:
"batch_9f2c8a"
Always queued. An accepted batch has not started yet.
Available options:
queued How pages will be selected.
Available options:
scrape, crawl Tags stored on the batch.
Example:
["docs"]
How the crawl was configured. Null for scrape batches.
Show child attributes
Show child attributes
What the submission accepted.
Show child attributes
Show child attributes
What accepting this batch cost.
Show child attributes
Show child attributes
When the batch was created.
Rejected URLs (first 100).
Show child attributes
Show child attributes
Whether this response came from cache.
Show child attributes
Show child attributes
Outputs requested for each page. Matches input.data.formats on submission.
Show child attributes
Show child attributes
Example:
{ "html": true, "markdown": true }
Unique ID of this request, also in X-Request-Id. Include it when contacting support.
Example:
"3f1c2a6e-8b4d-4c1e-9f0a-2d7b5e6c8a91"
Secret for verifying X-Context-Signature. Only submit returns it, so store it.
Credits this request used and your remaining balance.
Show child attributes
Show child attributes
Was this page helpful?