import ContextDev from 'context.dev';
const client = new ContextDev({
apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
});
const response = await client.web.webScrapeSitemap({ domain: 'domain' });
console.log(response.domain);{
"success": true,
"domain": "<string>",
"urls": [
"<string>"
],
"meta": {
"sitemapsDiscovered": 123,
"sitemapsFetched": 123,
"sitemapsSkipped": 123,
"errors": 123
}
}Crawls the sitemap of the given domain and returns all discovered page URLs. Supports sitemap index files (recursive), parallel fetching with concurrency control, deduplication, and filters out non-page resources (images, PDFs, etc.).
import ContextDev from 'context.dev';
const client = new ContextDev({
apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
});
const response = await client.web.webScrapeSitemap({ domain: 'domain' });
console.log(response.domain);{
"success": true,
"domain": "<string>",
"urls": [
"<string>"
],
"meta": {
"sitemapsDiscovered": 123,
"sitemapsFetched": 123,
"sitemapsSkipped": 123,
"errors": 123
}
}Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Domain name to crawl sitemaps for (e.g., 'example.com'). The domain will be automatically normalized and validated.
Successful response
Was this page helpful?