> ## Documentation Index
> Fetch the complete documentation index at: https://docs.context.dev/llms.txt
> Use this file to discover all available pages before exploring further.
# Submit a Batch
> Scrape 25K URLs or crawl large websites asynchronously.
1 Credit per successful URL
## OpenAPI
````yaml POST /batch/submit
openapi: 3.1.0
info:
title: Context API
description: API for retrieving context data from any website
version: 1.0.0
servers:
- url: https://api.context.dev/v1
security: []
tags:
- name: Batch
description: Scrape many pages or crawl a site asynchronously.
- name: Monitors
description: >-
Monitor pages, sitemaps, and extracted website data for exact or semantic
changes. Webhook payloads are documented by the
MonitorsChangeDetectedWebhookPayload and
MonitorsRunCompletedWebhookPayload schemas.
- name: WebDBs
description: Create structured tables from web pages and keep them up to date.
paths:
/batch/submit:
post:
tags:
- Batch
summary: Submit a batch
description: 'Scrape 25K URLs or crawl large websites asynchronously. '
operationId: submitBatch
parameters:
- schema:
type: string
maxLength: 200
description: >-
Any string unique to this submission. Retries with the same key
return the original batch.
required: false
description: >-
Any string unique to this submission. Retries with the same key
return the original batch.
name: Idempotency-Key
in: header
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/BatchSubmitRequest'
examples:
scrapeMarkdown:
summary: Scrape URLs as Markdown
value:
input:
type: scrape
data:
type: markdown
urls:
- url: https://example.com/products/anvil
itemId: sku-1
meta:
category: tools
- url: https://example.com/products/hammer
itemId: sku-2
options:
useMainContentOnly: true
scrapeHtml:
summary: Scrape URLs as HTML
value:
input:
type: scrape
data:
type: html
urls:
- url: https://example.com/about
- url: https://example.com/pricing
crawlStartUrlMarkdown:
summary: Crawl from a URL as Markdown
value:
input:
type: crawl
data:
type: markdown
source:
type: start_url
url: https://example.com/docs
controls:
maxUrls: 500
maxDepth: 3
followSubdomains: false
regex: ^https://example\.com/docs/
options:
includeLinks: true
crawlStartUrlHtml:
summary: Crawl from a URL as HTML
value:
input:
type: crawl
data:
type: html
source:
type: start_url
url: https://example.com/blog
controls:
maxUrls: 100
crawlSitemapMarkdown:
summary: Crawl a sitemap as Markdown
value:
input:
type: crawl
data:
type: markdown
source:
type: sitemap
domain: example.com
controls:
maxUrls: 500
regex: ^https://example\.com/docs/
crawlSitemapHtml:
summary: Crawl a sitemap as HTML
value:
input:
type: crawl
data:
type: html
source:
type: sitemap
domain: example.com
controls:
maxUrls: 500
maxDepth: 0
responses:
'202':
description: >-
Batch accepted. Read progress and results from `GET
/batch/{batch_id}`.
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
allOf:
- $ref: '#/components/schemas/BatchAccepted'
- type: object
properties:
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
description: API key usage for this request.
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'403':
description: Concurrent batch limit reached (error_code BATCH_LIMIT_EXCEEDED).
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'409':
description: >-
Idempotency-Key reused with a different body (error_code
IDEMPOTENCY_KEY_CONFLICT).
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
description: Batch input could not be staged or queued.
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
security:
- bearerAuth: []
components:
schemas:
BatchSubmitRequest:
type: object
properties:
input:
oneOf:
- type: object
properties:
mode:
type: string
enum:
- scrape
description: Scrape the pages in `data.urls`.
data:
oneOf:
- type: object
properties:
format:
type: string
enum:
- markdown
description: Return page content as Markdown.
urls:
type: array
items:
type: object
properties:
url:
type: string
minLength: 1
description: Page URL to scrape.
example: https://example.com/products/anvil
itemId:
type: string
minLength: 1
maxLength: 200
description: >-
Your ID for this page, returned with its
result. The same URL can use different IDs.
example: sku-1
meta:
type: object
additionalProperties: {}
description: >-
Custom JSON returned unchanged with this page
result.
required:
- url
additionalProperties: false
description: >-
A page to scrape, with optional data for matching
results.
minItems: 1
maxItems: 25000
description: Pages to scrape. Maximum 25000.
options:
type: object
properties:
includeLinks:
type: boolean
default: true
description: Include links in the Markdown.
includeImages:
type: boolean
default: false
description: Include image references in the Markdown.
shortenBase64Images:
type: boolean
default: true
description: Shorten inline base64 image data.
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
WEBSITE_ACCESS_ERROR is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, detect and OCR images embedded in
the selected PDF pages, inserting recognized
text at each image's position in page
reading order while preserving the PDF text
layer. This is separate from automatic
scanned-PDF OCR fallback.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
includeLinks: true
includeImages: false
shortenBase64Images: true
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for Markdown output.
required:
- format
- urls
additionalProperties: false
description: Scrape the listed pages as Markdown.
title: Markdown
- type: object
properties:
format:
type: string
enum:
- html
description: Return page content as HTML.
urls:
type: array
items:
type: object
properties:
url:
type: string
minLength: 1
description: Page URL to scrape.
example: https://example.com/products/anvil
itemId:
type: string
minLength: 1
maxLength: 200
description: >-
Your ID for this page, returned with its
result. The same URL can use different IDs.
example: sku-1
meta:
type: object
additionalProperties: {}
description: >-
Custom JSON returned unchanged with this page
result.
required:
- url
additionalProperties: false
description: >-
A page to scrape, with optional data for matching
results.
minItems: 1
maxItems: 25000
description: Pages to scrape. Maximum 25000.
options:
type: object
properties:
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
WEBSITE_ACCESS_ERROR is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, detect and OCR images embedded in
the selected PDF pages, inserting recognized
text at each image's position in page
reading order while preserving the PDF text
layer. This is separate from automatic
scanned-PDF OCR fallback.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for HTML output.
required:
- format
- urls
additionalProperties: false
description: Scrape the listed pages as HTML.
title: HTML
discriminator:
propertyName: format
title: Scrape data
description: Pages to scrape and their output format.
required:
- mode
- data
additionalProperties: false
title: Scrape
description: Scrape up to 25K URLs in one batch.
- type: object
properties:
mode:
type: string
enum:
- crawl
description: Discover and scrape pages from `data.source`.
data:
oneOf:
- type: object
properties:
format:
type: string
enum:
- markdown
description: Return page content as Markdown.
source:
oneOf:
- type: object
properties:
type:
type: string
enum:
- start_url
description: Start from one page.
url:
type: string
minLength: 1
description: >-
Page where crawling begins. A URL without a
scheme is read as https://.
example: https://example.com/docs
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
maxDepth:
type: integer
minimum: 0
maximum: 50
description: >-
Maximum link depth. Source pages are
depth 0. No limit when omitted.
followSubdomains:
type: boolean
default: false
description: Follow links to subdomains.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern for URLs to include. The
`start_url` itself is always included.
example: ^https://example\.com/docs/
default:
maxUrls: 100
followSubdomains: false
additionalProperties: false
description: Limits and filters for page discovery.
required:
- type
- url
additionalProperties: false
description: Discover pages by following links from one URL.
title: Start URL
- type: object
properties:
type:
type: string
enum:
- sitemap
description: Scrape the URLs in the domain's sitemap.
domain:
type: string
minLength: 3
description: >-
Domain whose sitemap lists the pages to
scrape. A full URL is reduced to its domain.
example: example.com
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern; only sitemap URLs matching
it are scraped.
example: ^https://example\.com/docs/
default:
maxUrls: 100
additionalProperties: false
description: >-
Limits and filters for the sitemap URLs. A
sitemap batch scrapes exactly those URLs and
never follows links off them, so there is no
crawl depth here.
required:
- type
- domain
additionalProperties: false
description: >-
Scrape the pages listed in a domain's sitemap.
Links on those pages are not followed.
title: Sitemap
discriminator:
propertyName: type
title: Crawl source
description: How to find pages to crawl.
options:
type: object
properties:
includeLinks:
type: boolean
default: true
description: Include links in the Markdown.
includeImages:
type: boolean
default: false
description: Include image references in the Markdown.
shortenBase64Images:
type: boolean
default: true
description: Shorten inline base64 image data.
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
WEBSITE_ACCESS_ERROR is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, detect and OCR images embedded in
the selected PDF pages, inserting recognized
text at each image's position in page
reading order while preserving the PDF text
layer. This is separate from automatic
scanned-PDF OCR fallback.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
includeLinks: true
includeImages: false
shortenBase64Images: true
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for Markdown output.
required:
- format
- source
additionalProperties: false
title: Markdown
description: Crawl pages and return Markdown.
- type: object
properties:
format:
type: string
enum:
- html
description: Return page content as HTML.
source:
oneOf:
- type: object
properties:
type:
type: string
enum:
- start_url
description: Start from one page.
url:
type: string
minLength: 1
description: >-
Page where crawling begins. A URL without a
scheme is read as https://.
example: https://example.com/docs
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
maxDepth:
type: integer
minimum: 0
maximum: 50
description: >-
Maximum link depth. Source pages are
depth 0. No limit when omitted.
followSubdomains:
type: boolean
default: false
description: Follow links to subdomains.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern for URLs to include. The
`start_url` itself is always included.
example: ^https://example\.com/docs/
default:
maxUrls: 100
followSubdomains: false
additionalProperties: false
description: Limits and filters for page discovery.
required:
- type
- url
additionalProperties: false
description: Discover pages by following links from one URL.
title: Start URL
- type: object
properties:
type:
type: string
enum:
- sitemap
description: Scrape the URLs in the domain's sitemap.
domain:
type: string
minLength: 3
description: >-
Domain whose sitemap lists the pages to
scrape. A full URL is reduced to its domain.
example: example.com
controls:
type: object
properties:
maxUrls:
type: integer
minimum: 1
maximum: 25000
default: 100
description: >-
Maximum pages to fetch. Unused reserved
credits are refunded. Maximum 25000.
regex:
type: string
maxLength: 256
description: >-
RE2 pattern; only sitemap URLs matching
it are scraped.
example: ^https://example\.com/docs/
default:
maxUrls: 100
additionalProperties: false
description: >-
Limits and filters for the sitemap URLs. A
sitemap batch scrapes exactly those URLs and
never follows links off them, so there is no
crawl depth here.
required:
- type
- domain
additionalProperties: false
description: >-
Scrape the pages listed in a domain's sitemap.
Links on those pages are not followed.
title: Sitemap
discriminator:
propertyName: type
title: Crawl source
description: How to find pages to crawl.
options:
type: object
properties:
useMainContentOnly:
type: boolean
default: false
description: >-
Return the main content without navigation or
footers.
pdf:
type: object
properties:
shouldParse:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: true
description: >-
When true, PDF URLs are fetched and parsed.
When false, PDF URLs are skipped and a 400
WEBSITE_ACCESS_ERROR is returned.
start:
type: integer
minimum: 1
description: >-
First 1-based PDF page to parse. When
omitted, parsing starts at the first page.
end:
type: integer
minimum: 1
description: >-
Last 1-based PDF page to parse. When
omitted, parsing ends at the last page. Must
be greater than or equal to start when both
are provided.
ocr:
anyOf:
- type: boolean
- type: string
enum:
- 'true'
- 'false'
default: false
description: >-
When true, detect and OCR images embedded in
the selected PDF pages, inserting recognized
text at each image's position in page
reading order while preserving the PDF text
layer. This is separate from automatic
scanned-PDF OCR fallback.
default:
shouldParse: true
ocr: false
description: >-
PDF parsing controls. Use start/end to limit
text extraction and embedded-image detection/OCR
to an inclusive 1-based page range.
includeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Keep only the subtrees matching these CSS
selectors. Filtered pages are always fetched
fresh, ignoring `maxAgeMs`.
excludeSelectors:
type:
- array
- 'null'
items:
type: string
minLength: 1
maxLength: 2048
maxItems: 50
description: >-
Remove elements matching these CSS selectors.
Applied after `includeSelectors`, so an element
matching both is removed.
waitForMs:
type: integer
minimum: 0
maximum: 15000
description: >-
How long to wait after initial page load, in
milliseconds. `0` waits 500 ms.
settleAnimations:
type: boolean
default: false
description: >-
Wait briefly for CSS and transition animations
to settle before extraction, on pages that
render in a browser.
country:
$ref: '#/components/schemas/BrowserCountryCode'
maxAgeMs:
type:
- integer
- 'null'
minimum: 0
maximum: 2592000000
default: 86400000
description: >-
Return a cached result if a prior scrape for the
same parameters exists and is younger than this
many milliseconds. Defaults to 1 day (86400000
ms) when omitted. Max is 30 days (2592000000
ms). Set to 0 to always scrape fresh.
default:
useMainContentOnly: false
pdf:
shouldParse: true
ocr: false
settleAnimations: false
maxAgeMs: 86400000
additionalProperties: false
description: Options for HTML output.
required:
- format
- source
additionalProperties: false
title: HTML
description: Crawl pages and return HTML.
discriminator:
propertyName: format
title: Crawl data
description: Crawl source and output format.
required:
- mode
- data
additionalProperties: false
title: Crawl
description: Crawl pages starting from a URL or from a domain's sitemap.
discriminator:
propertyName: mode
title: Batch input
description: Choose a URL list or a site crawl.
webhookUrl:
type: string
description: URL notified when the batch finishes.
tags:
type: array
items:
type: string
minLength: 1
maxLength: 50
maxItems: 20
description: Tags stored on the batch. Filter the batch list by them later.
example:
- docs
- competitor
required:
- input
title: Submit batch request
description: Create an asynchronous scraping job.
BatchAccepted:
type: object
properties:
id:
type: string
example: batch_9f2c8a
description: Batch ID. Poll GET /batch/{batch_id} with it.
status:
type: string
enum:
- queued
description: Always `queued`. An accepted batch has not started yet.
mode:
type: string
enum:
- scrape
- crawl
description: How pages will be selected.
format:
type: string
enum:
- markdown
- html
description: What each page will be returned as.
tags:
type: array
items:
type: string
description: Tags stored on the batch.
example:
- docs
crawl:
$ref: '#/components/schemas/BatchNullableCrawlControls'
input:
$ref: '#/components/schemas/BatchIntake'
credits:
type: object
properties:
reserved:
type: integer
example: 24817
description: >-
Credits just debited from your balance. Whatever the batch does
not spend is refunded when it settles.
required:
- reserved
description: What accepting this batch cost.
created_at:
type: string
description: When the batch was created.
invalid_urls:
type: array
items:
type: object
properties:
url:
type: string
description: Rejected URL.
reason:
type: string
description: Why it was rejected.
example: Must be a public http:// or https:// URL
required:
- url
- reason
description: Rejected URLs, up to 100. These are not charged.
webhook_secret:
type: string
description: >-
Signing secret for the completion webhook, returned only here and
never again. Store it now; it is not repeated by GET
/batch/{batch_id}.
required:
- id
- status
- mode
- format
- tags
- crawl
- input
- credits
- created_at
- invalid_urls
title: Accepted batch
KeyMetadata:
type: object
properties:
credits_consumed:
type: integer
description: The number of credits consumed by this request.
credits_remaining:
type: integer
description: >-
The number of credits remaining for your organization after this
request.
required:
- credits_consumed
- credits_remaining
description: >-
Metadata about the API key used for the request. Included in every
response whenever a valid API key is provided, even when the response
status is not 200.
ErrorResponse:
type: object
properties:
message:
type: string
description: Error message
error_code:
type: string
enum:
- INTERNAL_ERROR
- VALID
- NOT_FOUND
- FORBIDDEN
- USAGE_EXCEEDED
- RATE_LIMITED
- UNAUTHORIZED
- DISABLED
- PAID_PLAN_REQUIRED
- INSUFFICIENT_PERMISSIONS
- TIMEOUT_EXCEEDS_MAXIMUM
- WEBSITE_ACCESS_ERROR
- EXTERNAL_PROVIDER_ERROR
- INPUT_VALIDATION_ERROR
- ZDR_NOT_SUPPORTED
- ZDR_NOT_ENABLED
- FREE_EMAIL_DETECTED
- DISPOSABLE_EMAIL_DETECTED
- REQUEST_TIMEOUT
- UNSUPPORTED_CONTENT
- MONITOR_PAUSED
- MONITOR_NO_WEBHOOK
- COLLECTION_PAUSED
- MONITOR_LIMIT_EXCEEDED
- SEARCH_UNAVAILABLE
- BATCH_LIMIT_EXCEEDED
- BATCH_NOT_CANCELLABLE
- BATCH_NOT_COMPLETED
- IDEMPOTENCY_KEY_CONFLICT
description: Error code indicating the type of error
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
BrowserCountryCode:
type: string
enum:
- ad
- ae
- af
- ag
- ai
- al
- am
- ao
- ar
- at
- au
- aw
- az
- ba
- bb
- bd
- be
- bf
- bg
- bh
- bi
- bj
- bm
- bn
- bo
- bq
- br
- bs
- bw
- by
- bz
- ca
- cd
- cf
- cg
- ch
- ci
- cl
- cm
- cn
- co
- cr
- cv
- cw
- cy
- cz
- de
- dj
- dk
- dm
- do
- dz
- ec
- ee
- eg
- es
- et
- fi
- fj
- fr
- ga
- gb
- gd
- ge
- gf
- gg
- gh
- gm
- gn
- gp
- gq
- gr
- gt
- gu
- gw
- gy
- hk
- hn
- hr
- ht
- hu
- id
- ie
- il
- im
- in
- iq
- ir
- is
- it
- je
- jm
- jo
- jp
- ke
- kg
- kh
- kn
- kr
- kw
- ky
- kz
- la
- lb
- lc
- lk
- lr
- ls
- lt
- lu
- lv
- ly
- ma
- mc
- md
- me
- mf
- mg
- mk
- ml
- mm
- mn
- mo
- mq
- mr
- mt
- mu
- mv
- mw
- mx
- my
- mz
- na
- nc
- ne
- ng
- ni
- nl
- 'no'
- np
- nz
- om
- pa
- pe
- pf
- pg
- ph
- pk
- pl
- pr
- ps
- pt
- py
- qa
- re
- ro
- rs
- ru
- rw
- sa
- sc
- sd
- se
- sg
- si
- sk
- sl
- sm
- sn
- so
- sr
- ss
- st
- sv
- sx
- sy
- sz
- tc
- td
- tg
- th
- tj
- tl
- tm
- tn
- tr
- tt
- tw
- tz
- ua
- ug
- us
- uy
- uz
- vc
- ve
- vg
- vi
- vn
- ye
- yt
- za
- zm
- zw
example: de
description: >-
Fetch the target page through a residential proxy in this country (ISO
3166-1 alpha-2).
BatchNullableCrawlControls:
anyOf:
- allOf:
- $ref: '#/components/schemas/BatchCrawlControls'
- type: 'null'
description: How the crawl was configured. Null for scrape batches.
BatchIntake:
type: object
properties:
reserved:
type: integer
example: 24817
description: >-
Pages credits were reserved for. Everything else — progress, the
refund, the completion percentage — is measured against this.
reserved_is_ceiling:
type: boolean
example: false
description: >-
Whether `reserved` is an upper bound the batch may finish under.
True only for a crawl that follows links, whose reachable page count
is unknowable until it runs. False for a scrape and for a sitemap
crawl, where `reserved` is an exact page count.
submitted:
type:
- integer
- 'null'
example: 25000
description: >-
URLs in the list you sent, before validation and de-duplication.
Null for a crawl, which is given a source rather than a list.
duplicates:
type: integer
example: 183
description: >-
URLs dropped before reserving because another entry resolved to the
same page. Non-zero for sitemap crawls too, whose sitemaps routinely
list a page more than once.
invalid:
type:
- integer
- 'null'
example: 0
description: >-
URLs from your list rejected as unusable; the same ones are itemised
in `invalid_urls` at submission. Null for a crawl — a crawl that
resolves no usable page is rejected outright with a 400 rather than
accepted with an empty list.
required:
- reserved
- reserved_is_ceiling
- submitted
- duplicates
- invalid
x-stainless-model: batch.intake
description: What submission took in, and what it charged for.
BatchCrawlControls:
type: object
properties:
source:
anyOf:
- type: object
properties:
type:
type: string
enum:
- start_url
url:
type: string
example: https://example.com/docs
description: Page the crawl started from.
required:
- type
- url
- type: object
properties:
type:
type: string
enum:
- sitemap
domain:
type: string
example: example.com
description: Domain whose sitemap supplied the pages.
required:
- type
- domain
description: Where the crawl started.
max_pages:
type: integer
example: 500
description: >-
The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only
the URLs its sitemap actually lists, up to this many, so
`input.reserved` is often lower.
max_depth:
type:
- integer
- 'null'
example: 0
description: >-
Link depth limit. Always 0 for a sitemap crawl, which never follows
links off its URLs; null when a `start_url` crawl set no limit.
follow_subdomains:
type: boolean
description: >-
Whether links to subdomains were followed. Always false for a
sitemap crawl.
url_pattern:
type:
- string
- 'null'
example: ^https://example\.com/docs/
description: >-
RE2 pattern URLs had to match to be crawled. Null when the crawl set
none.
required:
- source
- max_pages
- max_depth
- follow_subdomains
- url_pattern
x-stainless-model: batch.crawl_controls
title: Crawl controls
description: >-
The crawl controls as submitted, so the limits requested can be compared
against what the crawl reached.
headers:
RateLimitLimit:
description: >-
Maximum requests allowed in the current fixed one-minute window.
Returned when the authenticated API key has a per-minute rate limit.
schema:
type: integer
minimum: 1
RateLimitRemaining:
description: >-
Requests remaining in the current fixed one-minute window. Returned when
the authenticated API key has a per-minute rate limit.
schema:
type: integer
minimum: 0
RateLimitReset:
description: >-
Unix timestamp in seconds when the current rate-limit window resets.
Returned when the authenticated API key has a per-minute rate limit.
schema:
type: integer
responses:
BadRequest:
description: Bad request
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
Unauthorized:
description: Unauthorized
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
securitySchemes:
bearerAuth:
type: http
scheme: bearer
description: >-
Bearer authentication header of the form `Bearer `, where
`` is your api key.
````