Start a website crawl
curl --request POST \
--url https://www.ntanduy.com/api/v1/web/crawls \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://example.com/",
"requestId": "01952140-e5b8-4e60-94fd-387c15f13257",
"limit": 10,
"maxDepth": 1
}
'import requests
url = "https://www.ntanduy.com/api/v1/web/crawls"
payload = {
"url": "https://example.com/",
"requestId": "01952140-e5b8-4e60-94fd-387c15f13257",
"limit": 10,
"maxDepth": 1
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: 'https://example.com/',
requestId: '01952140-e5b8-4e60-94fd-387c15f13257',
limit: 10,
maxDepth: 1
})
};
fetch('https://www.ntanduy.com/api/v1/web/crawls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://www.ntanduy.com/api/v1/web/crawls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://example.com/',
'requestId' => '01952140-e5b8-4e60-94fd-387c15f13257',
'limit' => 10,
'maxDepth' => 1
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://www.ntanduy.com/api/v1/web/crawls"
payload := strings.NewReader("{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://www.ntanduy.com/api/v1/web/crawls")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://www.ntanduy.com/api/v1/web/crawls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"id": "a24abf21-8c7f-49c4-91d2-43c31e71fca2",
"status": "completed",
"total": 1,
"completed": 1,
"documents": [
{
"url": "https://example.com/",
"title": "Example Domain",
"description": null,
"content": "# Example Domain",
"contentFormat": "markdown",
"contentTruncated": false,
"httpStatus": 200
}
],
"nextOffset": null,
"expiresAt": "2026-10-14T12:00:00Z"
}
}Web data
Start a website crawl
Start a website crawl with a normalized response contract.
POST
/
api
/
v1
/
web
/
crawls
Start a website crawl
curl --request POST \
--url https://www.ntanduy.com/api/v1/web/crawls \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://example.com/",
"requestId": "01952140-e5b8-4e60-94fd-387c15f13257",
"limit": 10,
"maxDepth": 1
}
'import requests
url = "https://www.ntanduy.com/api/v1/web/crawls"
payload = {
"url": "https://example.com/",
"requestId": "01952140-e5b8-4e60-94fd-387c15f13257",
"limit": 10,
"maxDepth": 1
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: 'https://example.com/',
requestId: '01952140-e5b8-4e60-94fd-387c15f13257',
limit: 10,
maxDepth: 1
})
};
fetch('https://www.ntanduy.com/api/v1/web/crawls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://www.ntanduy.com/api/v1/web/crawls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://example.com/',
'requestId' => '01952140-e5b8-4e60-94fd-387c15f13257',
'limit' => 10,
'maxDepth' => 1
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://www.ntanduy.com/api/v1/web/crawls"
payload := strings.NewReader("{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://www.ntanduy.com/api/v1/web/crawls")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://www.ntanduy.com/api/v1/web/crawls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://example.com/\",\n \"requestId\": \"01952140-e5b8-4e60-94fd-387c15f13257\",\n \"limit\": 10,\n \"maxDepth\": 1\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"id": "a24abf21-8c7f-49c4-91d2-43c31e71fca2",
"status": "completed",
"total": 1,
"completed": 1,
"documents": [
{
"url": "https://example.com/",
"title": "Example Domain",
"description": null,
"content": "# Example Domain",
"contentFormat": "markdown",
"contentTruncated": false,
"httpStatus": 200
}
],
"nextOffset": null,
"expiresAt": "2026-10-14T12:00:00Z"
}
}Requires
web:crawl through a scoped Pirehub API key or OAuth token.
See Web data for request limits, response fields, crawl lifecycle
and error handling. See Authentication
for credential headers and quota rules.Authorizations
pirehubOAuthpirehubApiKeybearerApiKey
OAuth 2.0 Authorization Code with PKCE, issued by Pirehub's Clerk authorization server. Request only the scopes required by the operation.
Body
application/json
Unique UUID for an intentional crawl. Reuse by the same credential returns 409 for 24 hours, including after ambiguous upstream failures.
Public HTTP(S) URL without credentials, using port 80 or 443; checked against private addresses and DNS.
Maximum string length:
2048Required range:
1 <= x <= 20Required range:
1 <= x <= 3