import ContextDev from 'context.dev';
const client = new ContextDev({
apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
});
const response = await client.web.webScrapeSitemap({ domain: 'domain' });
console.log(response.domain);{
"success": true,
"domain": "<string>",
"urls": [
"<string>"
],
"meta": {
"sitemapsDiscovered": 123,
"sitemapsFetched": 123,
"sitemapsSkipped": 123,
"errors": 123
}
}Crawl an entire website’s sitemap and return all discovered page URLs.
import ContextDev from 'context.dev';
const client = new ContextDev({
apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
});
const response = await client.web.webScrapeSitemap({ domain: 'domain' });
console.log(response.domain);{
"success": true,
"domain": "<string>",
"urls": [
"<string>"
],
"meta": {
"sitemapsDiscovered": 123,
"sitemapsFetched": 123,
"sitemapsSkipped": 123,
"errors": 123
}
}1 CreditDocumentation Index
Fetch the complete documentation index at: https://docs.context.dev/llms.txt
Use this file to discover all available pages before exploring further.
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Domain to build a sitemap for
Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Minimum is 1, maximum is 100,000.
1 <= x <= 100000Optional RE2-compatible regex pattern. Only URLs matching this pattern are returned and counted against maxLinks.
256"^https?://[^/]+/blog/"
Successful response
Was this page helpful?