> ## Documentation Index
> Fetch the complete documentation index at: https://docs.context.dev/llms.txt
> Use this file to discover all available pages before exploring further.

# Crawl Sitemap

> Discover every URL in a sitemap so your crawler, index, or enrichment job starts with complete coverage.

<Badge color="blue">1 Credit</Badge>


## OpenAPI

````yaml GET /web/scrape/sitemap
openapi: 3.1.0
info:
  title: Context API
  description: API for retrieving context data from any website
  version: 1.0.0
servers:
  - url: https://api.context.dev/v1
security: []
tags:
  - name: Monitors
    description: >-
      Monitor pages, sitemaps, and extracted website data for exact or semantic
      changes. Webhook payloads are documented by the
      MonitorsChangeDetectedWebhookPayload and
      MonitorsRunCompletedWebhookPayload schemas.
paths:
  /web/scrape/sitemap:
    get:
      tags:
        - Web Scraping
      summary: Crawl Sitemap
      description: Crawl an entire website's sitemap and return all discovered page URLs.
      parameters:
        - name: domain
          in: query
          required: true
          schema:
            type: string
          description: Domain to build a sitemap for
        - name: maxLinks
          in: query
          required: false
          schema:
            type: integer
            minimum: 1
            maximum: 100000
            default: 10000
          description: >-
            Maximum number of links to return from the sitemap crawl. Defaults
            to 10,000. Minimum is 1, maximum is 100,000.
        - name: urlRegex
          in: query
          required: false
          schema:
            type: string
            maxLength: 256
            example: ^https?://[^/]+/blog/
          description: >-
            Optional RE2-compatible regex pattern. Only URLs matching this
            pattern are returned and counted against maxLinks.
        - $ref: '#/components/parameters/ScrapeHeaders'
        - $ref: '#/components/parameters/TimeoutMS'
        - $ref: '#/components/parameters/RequestTags'
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    enum:
                      - true
                    description: Indicates success
                  domain:
                    type: string
                    description: The normalized domain that was crawled
                  urls:
                    type: array
                    items:
                      type: string
                    description: Array of discovered page URLs from the sitemap (max 500)
                  meta:
                    type: object
                    description: Metadata about the sitemap crawl operation
                    properties:
                      sitemapsDiscovered:
                        type: integer
                        description: Total number of sitemap files discovered
                      sitemapsFetched:
                        type: integer
                        description: >-
                          Number of sitemap files successfully fetched and
                          parsed
                      sitemapsSkipped:
                        type: integer
                        description: >-
                          Number of sitemap files skipped (due to errors,
                          timeouts, or limits)
                      errors:
                        type: integer
                        description: Number of errors encountered during crawling
                    required:
                      - sitemapsDiscovered
                      - sitemapsFetched
                      - sitemapsSkipped
                      - errors
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
                required:
                  - success
                  - domain
                  - urls
                  - meta
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
        '400':
          description: Bad request - Invalid domain
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
                    description: Error message describing the issue
                  error_code:
                    type: string
                    enum:
                      - INPUT_VALIDATION_ERROR
                      - WEBSITE_ACCESS_ERROR
                    description: Error code indicating the type of error
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
                required:
                  - message
                  - error_code
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
        '401':
          description: Unauthorized - Invalid or missing API key
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
                    description: Error message
                  error_code:
                    type: string
                    enum:
                      - UNAUTHORIZED
                    description: Error code indicating unauthorized access
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
        '403':
          description: Forbidden - Insufficient permissions or usage limit exceeded
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
                    description: Error message
                  error_code:
                    type: string
                    enum:
                      - FORBIDDEN
                      - USAGE_EXCEEDED
                      - DISABLED
                      - INSUFFICIENT_PERMISSIONS
                    description: Error code indicating forbidden access
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
        '408':
          description: Request timeout
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
                    description: Timeout error message
                  error_code:
                    type: string
                    enum:
                      - REQUEST_TIMEOUT
                    description: Error code indicating request timeout
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
        '429':
          $ref: '#/components/responses/RateLimited'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
                    description: Error message
                  error_code:
                    type: string
                    enum:
                      - INTERNAL_ERROR
                    description: Error code indicating internal server error
                  key_metadata:
                    $ref: '#/components/schemas/KeyMetadata'
          headers:
            X-RateLimit-Limit:
              $ref: '#/components/headers/RateLimitLimit'
            X-RateLimit-Remaining:
              $ref: '#/components/headers/RateLimitRemaining'
            X-RateLimit-Reset:
              $ref: '#/components/headers/RateLimitReset'
      security:
        - bearerAuth: []
      x-codeSamples:
        - lang: JavaScript
          source: >-
            import ContextDev from 'context.dev';


            const client = new ContextDev({
              apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted
            });


            const response = await client.web.webScrapeSitemap({ domain:
            'domain' });


            console.log(response.domain);
        - lang: Python
          source: |-
            import os
            from context.dev import ContextDev

            client = ContextDev(
                api_key=os.environ.get("CONTEXT_DEV_API_KEY"),  # This is the default and can be omitted
            )
            response = client.web.web_scrape_sitemap(
                domain="domain",
            )
            print(response.domain)
        - lang: Go
          source: "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/context-dot-dev/context-go-sdk\"\n\t\"github.com/context-dot-dev/context-go-sdk/option\"\n)\n\nfunc main() {\n\tclient := contextdev.NewClient(\n\t\toption.WithAPIKey(\"My API Key\"),\n\t)\n\tresponse, err := client.Web.WebScrapeSitemap(context.TODO(), contextdev.WebWebScrapeSitemapParams{\n\t\tDomain: \"domain\",\n\t})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\tfmt.Printf(\"%+v\\n\", response.Domain)\n}\n"
        - lang: Ruby
          source: |-
            require "context_dev"

            context_dev = ContextDev::Client.new(api_key: "My API Key")

            response = context_dev.web.web_scrape_sitemap(domain: "domain")

            puts(response)
        - lang: PHP
          source: >-
            <?php


            require_once dirname(__DIR__) . '/vendor/autoload.php';


            use ContextDev\Client;

            use ContextDev\Core\Exceptions\APIException;


            $client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY') ?: 'My
            API Key');


            try {
              $response = $client->web->webScrapeSitemap(
                domain: 'domain',
                headers: ['foo' => 'J!'],
                maxLinks: 1,
                tags: ['production', 'team-alpha'],
                timeoutMs: 1000,
                urlRegex: '^https?://[^/]+/blog/',
              );

              var_dump($response);
            } catch (APIException $e) {
              echo $e->getMessage();
            }
        - lang: CLI
          source: |-
            context-dev web web-scrape-sitemap \
              --api-key 'My API Key' \
              --domain domain
components:
  parameters:
    ScrapeHeaders:
      name: headers
      in: query
      required: false
      style: form
      explode: false
      schema:
        $ref: '#/components/schemas/HeaderMap'
      description: >-
        Optional outbound HTTP headers forwarded only to the target URL, sent as
        deep-object query params such as headers[X-Custom]=value. When provided,
        caching is bypassed: the result is neither read from nor written to
        cache.
    TimeoutMS:
      name: timeoutMS
      in: query
      required: false
      schema:
        $ref: '#/components/schemas/TimeoutMS'
      description: >-
        Optional timeout in milliseconds for the request. If the request takes
        longer than this value, it will be aborted with a 408 status code.
        Maximum allowed value is 300000ms (5 minutes).
    RequestTags:
      name: tags
      in: query
      required: false
      style: form
      explode: false
      schema:
        $ref: '#/components/schemas/RequestTags'
      description: >-
        Optional comma-separated caller-defined tags for tracking this request.
        Tags are recorded on the request's usage log and can be used to filter
        usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.
      example: production,team-alpha
  schemas:
    KeyMetadata:
      type: object
      description: >-
        Metadata about the API key used for the request. Included in every
        response whenever a valid API key is provided, even when the response
        status is not 200.
      properties:
        credits_consumed:
          type: integer
          description: The number of credits consumed by this request.
        credits_remaining:
          type: integer
          description: >-
            The number of credits remaining for your organization after this
            request.
      required:
        - credits_consumed
        - credits_remaining
    HeaderMap:
      type: object
      additionalProperties:
        type: string
        maxLength: 8192
        pattern: ^[^\r\n]*$
    TimeoutMS:
      type: integer
      minimum: 1000
      maximum: 300000
      description: >-
        Optional timeout in milliseconds for the request. If the request takes
        longer than this value, it will be aborted with a 408 status code.
        Maximum allowed value is 300000ms (5 minutes).
    RequestTags:
      type: array
      items:
        type: string
        minLength: 1
        maxLength: 50
      maxItems: 20
      description: >-
        Optional caller-defined tags for tracking this request. Tags are
        recorded on the request's usage log and can be used to filter usage on
        the dashboard usage page. Up to 20 tags, each 1-50 characters.
      example:
        - production
        - team-alpha
  headers:
    RateLimitLimit:
      description: >-
        Maximum requests allowed in the current fixed one-minute window.
        Returned when the authenticated API key has a per-minute rate limit.
      schema:
        type: integer
        minimum: 1
    RateLimitRemaining:
      description: >-
        Requests remaining in the current fixed one-minute window. Returned when
        the authenticated API key has a per-minute rate limit.
      schema:
        type: integer
        minimum: 0
    RateLimitReset:
      description: >-
        Unix timestamp in seconds when the current rate-limit window resets.
        Returned when the authenticated API key has a per-minute rate limit.
      schema:
        type: integer
  responses:
    RateLimited:
      description: Rate limit exceeded
      headers:
        X-RateLimit-Limit:
          $ref: '#/components/headers/RateLimitLimit'
        X-RateLimit-Remaining:
          $ref: '#/components/headers/RateLimitRemaining'
        X-RateLimit-Reset:
          $ref: '#/components/headers/RateLimitReset'
        Retry-After:
          description: Seconds until the per-minute rate limit window resets
          schema:
            type: integer
            minimum: 1
            maximum: 60
      content:
        application/json:
          schema:
            type: object
            properties:
              message:
                type: string
                description: Error message
              error_code:
                type: string
                enum:
                  - RATE_LIMITED
                description: Error code indicating the rate limit was exceeded
              key_metadata:
                $ref: '#/components/schemas/KeyMetadata'
            required:
              - message
              - error_code
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer

````