Extract a web page
curl --request POST \
--url https://www.ntanduy.com/api/v1/web/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://example.com/"
}
'import requests
url = "https://www.ntanduy.com/api/v1/web/scrape"
payload = { "url": "https://example.com/" }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({url: 'https://example.com/'})
};
fetch('https://www.ntanduy.com/api/v1/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://www.ntanduy.com/api/v1/web/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://example.com/'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://www.ntanduy.com/api/v1/web/scrape"
payload := strings.NewReader("{\n \"url\": \"https://example.com/\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://www.ntanduy.com/api/v1/web/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://example.com/\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://www.ntanduy.com/api/v1/web/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://example.com/\"\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"document": {
"url": "https://example.com/",
"title": "Example Domain",
"description": null,
"content": "# Example Domain",
"contentFormat": "markdown",
"contentTruncated": false,
"httpStatus": 200
}
}
}Web data
Extract a web page
Extract a web page with a normalized response contract.
POST
/
api
/
v1
/
web
/
scrape
Extract a web page
curl --request POST \
--url https://www.ntanduy.com/api/v1/web/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://example.com/"
}
'import requests
url = "https://www.ntanduy.com/api/v1/web/scrape"
payload = { "url": "https://example.com/" }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({url: 'https://example.com/'})
};
fetch('https://www.ntanduy.com/api/v1/web/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://www.ntanduy.com/api/v1/web/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://example.com/'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://www.ntanduy.com/api/v1/web/scrape"
payload := strings.NewReader("{\n \"url\": \"https://example.com/\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://www.ntanduy.com/api/v1/web/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://example.com/\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://www.ntanduy.com/api/v1/web/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://example.com/\"\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"document": {
"url": "https://example.com/",
"title": "Example Domain",
"description": null,
"content": "# Example Domain",
"contentFormat": "markdown",
"contentTruncated": false,
"httpStatus": 200
}
}
}Requires
web:scrape through a scoped Pirehub API key or OAuth token.
See Web data for request limits, response fields, crawl lifecycle
and error handling. See Authentication
for credential headers and quota rules.Authorizations
pirehubOAuthpirehubApiKeybearerApiKey
OAuth 2.0 Authorization Code with PKCE, issued by Pirehub's Clerk authorization server. Request only the scopes required by the operation.
Body
application/json
Public HTTP(S) URL without credentials, using port 80 or 443; checked against private addresses and DNS.
Maximum string length:
2048