> ## Documentation Index
> Fetch the complete documentation index at: https://docs.context.dev/llms.txt
> Use this file to discover all available pages before exploring further.

> ## Agent Instructions
> Setting up Context.dev for a user? Follow https://docs.context.dev/agent-quickstart.md. MCP clients sign in with OAuth and need no API key; application code reads CONTEXT_DEV_API_KEY.

# Large crawls

> Crawl a site in the background and retrieve Markdown or HTML when it finishes.

Submit a crawl through `POST /batch/submit` when it is too large for one synchronous response. A batch returns an ID immediately and produces per-page Markdown or HTML.

## Start from a URL

<CodeGroup>
  ```typescript TypeScript theme={null}
  import ContextDev from "context.dev";

  const client = new ContextDev({ apiKey: process.env.CONTEXT_DEV_API_KEY });

  const response = await client.batch.submit({
    input: {
      mode: "crawl",
      data: {
        format: "markdown",
        source: {
          type: "start_url",
          url: "https://docs.context.dev/introduction",
          controls: {
            maxUrls: 100,
            maxDepth: 3,
            followSubdomains: false,
            regex: "^https://docs\\.context\\.dev/",
          },
        },
        options: {
          useMainContentOnly: true,
        },
      },
    },
    tags: ["docs-crawl"],
    "Idempotency-Key": "docs-crawl-v1",
  });
  console.log(response);
  ```

  ```python Python theme={null}
  import os
  from context.dev import ContextDev

  client = ContextDev(api_key=os.environ["CONTEXT_DEV_API_KEY"])

  response = client.batch.submit(
      input={
          "mode": "crawl",
          "data": {
              "format": "markdown",
              "source": {
                  "type": "start_url",
                  "url": "https://docs.context.dev/introduction",
                  "controls": {
                      "max_urls": 100,
                      "max_depth": 3,
                      "follow_subdomains": False,
                      "regex": "^https://docs\\.context\\.dev/",
                  },
              },
              "options": {
                  "use_main_content_only": True,
              },
          },
      },
      tags=["docs-crawl"],
      idempotency_key="docs-crawl-v1",
  )
  print(response)
  ```

  ```ruby Ruby theme={null}
  require "cgi/core"
  require "context_dev"

  client = ContextDev::Client.new(api_key: ENV.fetch("CONTEXT_DEV_API_KEY"))

  response = client.batch.submit(
    input: {
      mode: "crawl",
      data: {
        format: "markdown",
        source: {
          type: "start_url",
          url: "https://docs.context.dev/introduction",
          controls: {
            max_urls: 100,
            max_depth: 3,
            follow_subdomains: false,
            regex: "^https://docs\\.context\\.dev/",
          },
        },
        options: {
          use_main_content_only: true,
        },
      },
    },
    tags: ["docs-crawl"],
    idempotency_key: "docs-crawl-v1",
  )
  puts response.inspect
  ```

  ```go Go theme={null}
  package main

  import (
  	"context"
  	"fmt"
  	"os"

  	contextdev "github.com/context-dot-dev/context-go-sdk/v2"
  	"github.com/context-dot-dev/context-go-sdk/v2/option"
  )

  func main() {
  	client := contextdev.NewClient(option.WithAPIKey(os.Getenv("CONTEXT_DEV_API_KEY")))

  	response, err := client.Batch.Submit(context.Background(), contextdev.BatchSubmitParams{
  		Input: contextdev.BatchSubmitParamsInputUnion{
  			OfCrawl: &contextdev.BatchSubmitParamsInputCrawl{
  				Mode: "crawl",
  				Data: contextdev.BatchSubmitParamsInputCrawlDataUnion{
  					OfMarkdown: &contextdev.BatchSubmitParamsInputCrawlDataMarkdown{
  						Format: "markdown",
  						Source: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceUnion{
  							OfStartURL: &contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL{
  								Type: "start_url",
  								URL:  "https://docs.context.dev/introduction",
  								Controls: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls{
  									MaxURLs:          contextdev.Int(100),
  									MaxDepth:         contextdev.Int(3),
  									FollowSubdomains: contextdev.Bool(false),
  									Regex:            contextdev.String("^https://docs\\.context\\.dev/"),
  								},
  							},
  						},
  						Options: contextdev.BatchSubmitParamsInputCrawlDataMarkdownOptions{
  							UseMainContentOnly: contextdev.Bool(true),
  						},
  					},
  				},
  			},
  		},
  		Tags:           []string{"docs-crawl"},
  		IdempotencyKey: contextdev.String("docs-crawl-v1"),
  	})
  	if err != nil {
  		panic(err)
  	}
  	fmt.Println(response)
  }
  ```

  ```php PHP theme={null}
  <?php
  require __DIR__.'/vendor/autoload.php';

  use ContextDev\Client;

  $client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY'));

  $response = $client->batch->submit(
      input: [
          "mode" => "crawl",
          "data" => [
              "format" => "markdown",
              "source" => [
                  "type" => "start_url",
                  "url" => "https://docs.context.dev/introduction",
                  "controls" => [
                      "maxURLs" => 100,
                      "maxDepth" => 3,
                      "followSubdomains" => false,
                      "regex" => "^https://docs\\.context\\.dev/",
                  ],
              ],
              "options" => [
                  "useMainContentOnly" => true,
              ],
          ],
      ],
      tags: ["docs-crawl"],
      idempotencyKey: "docs-crawl-v1",
  );
  var_dump($response);
  ```

  ```bash cURL theme={null}
  curl -X POST https://api.context.dev/v1/batch/submit \
    -H "Authorization: Bearer $CONTEXT_DEV_API_KEY" \
    -H "Content-Type: application/json" \
    -H "Idempotency-Key: docs-crawl-v1" \
    -d '{
      "input": {
        "mode": "crawl",
        "data": {
          "format": "markdown",
          "source": {
            "type": "start_url",
            "url": "https://docs.context.dev/introduction",
            "controls": {
              "maxUrls": 100,
              "maxDepth": 3,
              "followSubdomains": false,
              "regex": "^https://docs\\.context\\.dev/"
            }
          },
          "options": {
            "useMainContentOnly": true
          }
        }
      },
      "tags": ["docs-crawl"]
    }'
  ```
</CodeGroup>

## Crawl controls

| `input.data.controls` field | Default | Limit                                                                                   |
| --------------------------- | ------- | --------------------------------------------------------------------------------------- |
| `maxUrls`                   | 100     | 1–25,000.                                                                               |
| `maxDepth`                  | Unset   | 0–50 for a start-URL crawl.                                                             |
| `regex`                     | None    | RE2 pattern over normalized full URLs; up to 256 characters. The start URL is included. |
| `followSubdomains`          | `false` | Start-URL crawls only.                                                                  |

Start-URL crawls stay on the same host, ignoring `www.`, unless subdomains are enabled. They do not follow non-page files or system paths. Out-of-scope redirects fail that page.

## Start from an inventory

For sitemap discovery, set `input.data.source` to `{"type":"sitemap","domain":"example.com"}` and choose `maxUrls` and `regex`. Discovery happens at submission and excludes subdomain sitemaps. `maxDepth` and `followSubdomains` apply only to the start-URL source.

Per-page extraction settings are under `input.data.options`; see [Submit a batch](/batches/submit). Continue with [status and results](/batches/results), [webhooks](/webhooks), and [cancellation and limits](/batches/limits-and-errors).