Large crawls

import ContextDev from "context.dev";

const client = new ContextDev({ apiKey: process.env.CONTEXT_DEV_API_KEY });

const response = await client.batch.submit({
  input: {
    mode: "crawl",
    data: {
      format: "markdown",
      source: {
        type: "start_url",
        url: "https://docs.context.dev/introduction",
        controls: {
          maxUrls: 100,
          maxDepth: 3,
          followSubdomains: false,
          regex: "^https://docs\\.context\\.dev/",
        },
      },
      options: {
        useMainContentOnly: true,
      },
    },
  },
  tags: ["docs-crawl"],
  "Idempotency-Key": "docs-crawl-v1",
});
console.log(response);

import os
from context.dev import ContextDev

client = ContextDev(api_key=os.environ["CONTEXT_DEV_API_KEY"])

response = client.batch.submit(
    input={
        "mode": "crawl",
        "data": {
            "format": "markdown",
            "source": {
                "type": "start_url",
                "url": "https://docs.context.dev/introduction",
                "controls": {
                    "max_urls": 100,
                    "max_depth": 3,
                    "follow_subdomains": False,
                    "regex": "^https://docs\\.context\\.dev/",
                },
            },
            "options": {
                "use_main_content_only": True,
            },
        },
    },
    tags=["docs-crawl"],
    idempotency_key="docs-crawl-v1",
)
print(response)

require "cgi/core"
require "context_dev"

client = ContextDev::Client.new(api_key: ENV.fetch("CONTEXT_DEV_API_KEY"))

response = client.batch.submit(
  input: {
    mode: "crawl",
    data: {
      format: "markdown",
      source: {
        type: "start_url",
        url: "https://docs.context.dev/introduction",
        controls: {
          max_urls: 100,
          max_depth: 3,
          follow_subdomains: false,
          regex: "^https://docs\\.context\\.dev/",
        },
      },
      options: {
        use_main_content_only: true,
      },
    },
  },
  tags: ["docs-crawl"],
  idempotency_key: "docs-crawl-v1",
)
puts response.inspect

package main

import (
	"context"
	"fmt"
	"os"

	contextdev "github.com/context-dot-dev/context-go-sdk/v2"
	"github.com/context-dot-dev/context-go-sdk/v2/option"
)

func main() {
	client := contextdev.NewClient(option.WithAPIKey(os.Getenv("CONTEXT_DEV_API_KEY")))

	response, err := client.Batch.Submit(context.Background(), contextdev.BatchSubmitParams{
		Input: contextdev.BatchSubmitParamsInputUnion{
			OfCrawl: &contextdev.BatchSubmitParamsInputCrawl{
				Mode: "crawl",
				Data: contextdev.BatchSubmitParamsInputCrawlDataUnion{
					OfMarkdown: &contextdev.BatchSubmitParamsInputCrawlDataMarkdown{
						Format: "markdown",
						Source: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceUnion{
							OfStartURL: &contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL{
								Type: "start_url",
								URL:  "https://docs.context.dev/introduction",
								Controls: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls{
									MaxURLs:          contextdev.Int(100),
									MaxDepth:         contextdev.Int(3),
									FollowSubdomains: contextdev.Bool(false),
									Regex:            contextdev.String("^https://docs\\.context\\.dev/"),
								},
							},
						},
						Options: contextdev.BatchSubmitParamsInputCrawlDataMarkdownOptions{
							UseMainContentOnly: contextdev.Bool(true),
						},
					},
				},
			},
		},
		Tags:           []string{"docs-crawl"},
		IdempotencyKey: contextdev.String("docs-crawl-v1"),
	})
	if err != nil {
		panic(err)
	}
	fmt.Println(response)
}

<?php
require __DIR__.'/vendor/autoload.php';

use ContextDev\Client;

$client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY'));

$response = $client->batch->submit(
    input: [
        "mode" => "crawl",
        "data" => [
            "format" => "markdown",
            "source" => [
                "type" => "start_url",
                "url" => "https://docs.context.dev/introduction",
                "controls" => [
                    "maxURLs" => 100,
                    "maxDepth" => 3,
                    "followSubdomains" => false,
                    "regex" => "^https://docs\\.context\\.dev/",
                ],
            ],
            "options" => [
                "useMainContentOnly" => true,
            ],
        ],
    ],
    tags: ["docs-crawl"],
    idempotencyKey: "docs-crawl-v1",
);
var_dump($response);

curl -X POST https://api.context.dev/v1/batch/submit \
  -H "Authorization: Bearer $CONTEXT_DEV_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: docs-crawl-v1" \
  -d '{
    "input": {
      "mode": "crawl",
      "data": {
        "format": "markdown",
        "source": {
          "type": "start_url",
          "url": "https://docs.context.dev/introduction",
          "controls": {
            "maxUrls": 100,
            "maxDepth": 3,
            "followSubdomains": false,
            "regex": "^https://docs\\.context\\.dev/"
          }
        },
        "options": {
          "useMainContentOnly": true
        }
      }
    },
    "tags": ["docs-crawl"]
  }'