Discover website URLs

Use GET /web/scrape/sitemap when you need a website’s URL inventory before deciding what to scrape. The endpoint reads public sitemap files and does not render the returned pages.

Prerequisites

Export an API key from the dashboard:

export CONTEXT_DEV_API_KEY="ctxt_secret_..."

List URLs

Pass a domain without a protocol. The following request keeps at most 50 URLs whose paths contain /customers/.

curl --get https://api.context.dev/v1/web/scrape/sitemap \
  --header "Authorization: Bearer $CONTEXT_DEV_API_KEY" \
  --data-urlencode "domain=stripe.com" \
  --data-urlencode "maxLinks=50" \
  --data-urlencode "urlRegex=/customers/"

import ContextDev from "context.dev";

const client = new ContextDev({
  apiKey: process.env.CONTEXT_DEV_API_KEY,
});

const sitemap = await client.web.webScrapeSitemap({
  domain: "stripe.com",
  maxLinks: 50,
  urlRegex: "/customers/",
});

console.log(sitemap.urls);

import os
from context.dev import ContextDev

client = ContextDev(api_key=os.environ["CONTEXT_DEV_API_KEY"])

sitemap = client.web.web_scrape_sitemap(
    domain="stripe.com",
    max_links=50,
    url_regex="/customers/",
)

print(sitemap.urls)

require "cgi/core"
require "context_dev"

client = ContextDev::Client.new(api_key: ENV.fetch("CONTEXT_DEV_API_KEY"))

sitemap = client.web.web_scrape_sitemap(
  domain: "stripe.com",
  max_links: 50,
  url_regex: "/customers/",
)

puts sitemap.urls

package main

import (
    "context"
    "fmt"
    "os"

    contextdev "github.com/context-dot-dev/context-go-sdk/v2"
    "github.com/context-dot-dev/context-go-sdk/v2/option"
    "github.com/context-dot-dev/context-go-sdk/v2/packages/param"
)

func main() {
    client := contextdev.NewClient(
        option.WithAPIKey(os.Getenv("CONTEXT_DEV_API_KEY")),
    )

    sitemap, err := client.Web.WebScrapeSitemap(context.Background(), contextdev.WebWebScrapeSitemapParams{
        Domain: "stripe.com",
        MaxLinks: param.NewOpt[int64](50),
        URLRegex: param.NewOpt("/customers/"),
    })
    if err != nil {
        panic(err)
    }

    fmt.Println(sitemap.URLs)
}

<?php

require __DIR__.'/vendor/autoload.php';

use ContextDev\Client;

$client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY'));

$sitemap = $client->web->webScrapeSitemap(
    domain: 'stripe.com',
    maxLinks: 50,
    urlRegex: '/customers/',
);

print_r($sitemap->urls);

The base request costs 1 credit.

Read the result

{
  "success": true,
  "domain": "stripe.com",
  "urls": [
    "https://stripe.com/customers/all",
    "https://stripe.com/customers/gamma"
  ],
  "meta": {
    "sitemapsDiscovered": 4,
    "sitemapsFetched": 4,
    "sitemapsSkipped": 0,
    "errors": 0
  }
}

urls is de-duplicated and bounded by maxLinks. Use meta to detect incomplete sitemap coverage.

Narrow the inventory

Goal	Parameter	Behavior
Cap the returned list	`maxLinks`	Defaults to 10,000; accepted range is 1 to 100,000.
Filter by URL pattern	`urlRegex`	Returns only URLs matching the expression.
Find pages about a topic	`search`	Filters and ranks discovered URLs by a 2 to 200 character phrase; costs 2 credits.
Include child hosts	`includeSubdomains=true`	Includes public pages and sitemaps on hosts such as `docs.example.com`.
Use one known sitemap	`sitemapUrl`	Crawls that sitemap instead of discovering sitemap locations from the domain.

For example, search for likely authentication documentation:

curl --get https://api.context.dev/v1/web/scrape/sitemap \
  --header "Authorization: Bearer $CONTEXT_DEV_API_KEY" \
  --data-urlencode "domain=example.com" \
  --data-urlencode "search=API authentication docs"

const sitemap = await client.web.webScrapeSitemap({
  domain: "example.com",
  search: "API authentication docs",
});

console.log(sitemap.urls);

sitemap = client.web.web_scrape_sitemap(
    domain="example.com",
    search="API authentication docs",
)

print(sitemap.urls)

require "cgi/core"
require "context_dev"

client = ContextDev::Client.new(api_key: ENV.fetch("CONTEXT_DEV_API_KEY"))

sitemap = client.web.web_scrape_sitemap(
  domain: "example.com",
  search: "API authentication docs",
)

puts sitemap.urls

package main

import (
    "context"
    "fmt"
    "os"

    contextdev "github.com/context-dot-dev/context-go-sdk/v2"
    "github.com/context-dot-dev/context-go-sdk/v2/option"
    "github.com/context-dot-dev/context-go-sdk/v2/packages/param"
)

func main() {
    client := contextdev.NewClient(
        option.WithAPIKey(os.Getenv("CONTEXT_DEV_API_KEY")),
    )

    sitemap, err := client.Web.WebScrapeSitemap(context.Background(), contextdev.WebWebScrapeSitemapParams{
        Domain: "example.com",
        Search: param.NewOpt("API authentication docs"),
    })
    if err != nil {
        panic(err)
    }

    fmt.Println(sitemap.URLs)
}

<?php

require __DIR__.'/vendor/autoload.php';

use ContextDev\Client;

$client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY'));

$sitemap = $client->web->webScrapeSitemap(
    domain: 'example.com',
    search: 'API authentication docs',
);

print_r($sitemap->urls);

Sitemap coverage depends on what the target site publishes. A URL missing from sitemap.xml can still be reachable through links, and a listed URL can fail when rendered.

Choose what happens next

Crawl a website

Discover linked pages and render their content in one request.

Scrape websites in batches

Feed the discovered URL list into an asynchronous job.

Scrape a webpage

Read one URL at a time from a small selected set.

Production behavior

A request that forwards custom headers bypasses sitemap cache reads and writes.
Set timeoutMS when URL discovery is on a latency-sensitive path; the maximum is five minutes.
Set zdr=enabled to bypass shared caches and retained content logs when Zero Data Retention is enabled for your organization.
Treat meta.errors > 0 or sitemapsSkipped > 0 as incomplete coverage, not as a total request failure.

Sitemap API reference

Review all discovery parameters and response status codes.

Get started

Web data

Brand data

Automation

Production

Discover website URLs

Prerequisites

List URLs

Read the result

Narrow the inventory

Choose what happens next

Crawl a website

Scrape websites in batches

Scrape a webpage

Production behavior

Sitemap API reference

​Prerequisites

​List URLs

​Read the result

​Narrow the inventory

​Choose what happens next

Crawl a website

Scrape websites in batches

Scrape a webpage

​Production behavior

Sitemap API reference

Prerequisites

List URLs

Read the result

Narrow the inventory

Choose what happens next

Production behavior