Structured extraction with JSON-schema validation
curl --request POST \
--url https://api.webscrape.ai/v1/smartscraper \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"website_url": "https://news.ycombinator.com",
"user_prompt": "Extract the front-page stories with title, url, and score.",
"output_schema": {
"type": "object",
"properties": {
"stories": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"url": {
"type": "string"
},
"score": {
"type": "integer"
}
}
}
}
}
},
"page_complexity": "low",
"detail_level": "medium",
"parse_mode": "accurate",
"plain_text": false,
"include_tags": [
"<string>"
],
"exclude_tags": [
"<string>"
],
"reduce_content": true,
"experimental": false,
"headers": {},
"max_age": 1,
"stealth": false
}
'import requests
url = "https://api.webscrape.ai/v1/smartscraper"
payload = {
"website_url": "https://news.ycombinator.com",
"user_prompt": "Extract the front-page stories with title, url, and score.",
"output_schema": {
"type": "object",
"properties": { "stories": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": { "type": "string" },
"url": { "type": "string" },
"score": { "type": "integer" }
}
}
} }
},
"page_complexity": "low",
"detail_level": "medium",
"parse_mode": "accurate",
"plain_text": False,
"include_tags": ["<string>"],
"exclude_tags": ["<string>"],
"reduce_content": True,
"experimental": False,
"headers": {},
"max_age": 1,
"stealth": False
}
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
website_url: 'https://news.ycombinator.com',
user_prompt: 'Extract the front-page stories with title, url, and score.',
output_schema: {
type: 'object',
properties: {
stories: {
type: 'array',
items: {
type: 'object',
properties: {title: {type: 'string'}, url: {type: 'string'}, score: {type: 'integer'}}
}
}
}
},
page_complexity: 'low',
detail_level: 'medium',
parse_mode: 'accurate',
plain_text: false,
include_tags: ['<string>'],
exclude_tags: ['<string>'],
reduce_content: true,
experimental: false,
headers: {},
max_age: 1,
stealth: false
})
};
fetch('https://api.webscrape.ai/v1/smartscraper', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrape.ai/v1/smartscraper",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'website_url' => 'https://news.ycombinator.com',
'user_prompt' => 'Extract the front-page stories with title, url, and score.',
'output_schema' => [
'type' => 'object',
'properties' => [
'stories' => [
'type' => 'array',
'items' => [
'type' => 'object',
'properties' => [
'title' => [
'type' => 'string'
],
'url' => [
'type' => 'string'
],
'score' => [
'type' => 'integer'
]
]
]
]
]
],
'page_complexity' => 'low',
'detail_level' => 'medium',
'parse_mode' => 'accurate',
'plain_text' => false,
'include_tags' => [
'<string>'
],
'exclude_tags' => [
'<string>'
],
'reduce_content' => true,
'experimental' => false,
'headers' => [
],
'max_age' => 1,
'stealth' => false
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.webscrape.ai/v1/smartscraper"
payload := strings.NewReader("{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.webscrape.ai/v1/smartscraper")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.webscrape.ai/v1/smartscraper")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}"
response = http.request(request)
puts response.read_body{
"status": "completed",
"request_id": "req_aB3xY9Kp",
"data": {
"request_id": "<string>",
"result": {},
"latency_ms": 123
},
"credits_used": 5,
"credits_remaining": 495
}Scrape
Structured extraction with JSON-schema validation
Give a URL and a user_prompt (and optionally an output_schema),
get JSON back. Handles long content, noisy pages, and complex
schemas.
Cost: 5 credits (+5 with stealth: true).
POST
/
smartscraper
Structured extraction with JSON-schema validation
curl --request POST \
--url https://api.webscrape.ai/v1/smartscraper \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <api-key>' \
--data '
{
"website_url": "https://news.ycombinator.com",
"user_prompt": "Extract the front-page stories with title, url, and score.",
"output_schema": {
"type": "object",
"properties": {
"stories": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"url": {
"type": "string"
},
"score": {
"type": "integer"
}
}
}
}
}
},
"page_complexity": "low",
"detail_level": "medium",
"parse_mode": "accurate",
"plain_text": false,
"include_tags": [
"<string>"
],
"exclude_tags": [
"<string>"
],
"reduce_content": true,
"experimental": false,
"headers": {},
"max_age": 1,
"stealth": false
}
'import requests
url = "https://api.webscrape.ai/v1/smartscraper"
payload = {
"website_url": "https://news.ycombinator.com",
"user_prompt": "Extract the front-page stories with title, url, and score.",
"output_schema": {
"type": "object",
"properties": { "stories": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": { "type": "string" },
"url": { "type": "string" },
"score": { "type": "integer" }
}
}
} }
},
"page_complexity": "low",
"detail_level": "medium",
"parse_mode": "accurate",
"plain_text": False,
"include_tags": ["<string>"],
"exclude_tags": ["<string>"],
"reduce_content": True,
"experimental": False,
"headers": {},
"max_age": 1,
"stealth": False
}
headers = {
"X-API-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
website_url: 'https://news.ycombinator.com',
user_prompt: 'Extract the front-page stories with title, url, and score.',
output_schema: {
type: 'object',
properties: {
stories: {
type: 'array',
items: {
type: 'object',
properties: {title: {type: 'string'}, url: {type: 'string'}, score: {type: 'integer'}}
}
}
}
},
page_complexity: 'low',
detail_level: 'medium',
parse_mode: 'accurate',
plain_text: false,
include_tags: ['<string>'],
exclude_tags: ['<string>'],
reduce_content: true,
experimental: false,
headers: {},
max_age: 1,
stealth: false
})
};
fetch('https://api.webscrape.ai/v1/smartscraper', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.webscrape.ai/v1/smartscraper",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'website_url' => 'https://news.ycombinator.com',
'user_prompt' => 'Extract the front-page stories with title, url, and score.',
'output_schema' => [
'type' => 'object',
'properties' => [
'stories' => [
'type' => 'array',
'items' => [
'type' => 'object',
'properties' => [
'title' => [
'type' => 'string'
],
'url' => [
'type' => 'string'
],
'score' => [
'type' => 'integer'
]
]
]
]
]
],
'page_complexity' => 'low',
'detail_level' => 'medium',
'parse_mode' => 'accurate',
'plain_text' => false,
'include_tags' => [
'<string>'
],
'exclude_tags' => [
'<string>'
],
'reduce_content' => true,
'experimental' => false,
'headers' => [
],
'max_age' => 1,
'stealth' => false
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.webscrape.ai/v1/smartscraper"
payload := strings.NewReader("{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.webscrape.ai/v1/smartscraper")
.header("X-API-Key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.webscrape.ai/v1/smartscraper")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"website_url\": \"https://news.ycombinator.com\",\n \"user_prompt\": \"Extract the front-page stories with title, url, and score.\",\n \"output_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"stories\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"url\": {\n \"type\": \"string\"\n },\n \"score\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"page_complexity\": \"low\",\n \"detail_level\": \"medium\",\n \"parse_mode\": \"accurate\",\n \"plain_text\": false,\n \"include_tags\": [\n \"<string>\"\n ],\n \"exclude_tags\": [\n \"<string>\"\n ],\n \"reduce_content\": true,\n \"experimental\": false,\n \"headers\": {},\n \"max_age\": 1,\n \"stealth\": false\n}"
response = http.request(request)
puts response.read_body{
"status": "completed",
"request_id": "req_aB3xY9Kp",
"data": {
"request_id": "<string>",
"result": {},
"latency_ms": 123
},
"credits_used": 5,
"credits_remaining": 495
}Authorizations
Body
application/json
Example:
"https://news.ycombinator.com"
Plain-English description of what to extract.
Example:
"Extract the front-page stories with title, url, and score."
JSON Schema enforced on the extracted output. When supplied,
the result is validated against the schema and one repair
attempt is made before returning validation_failed.
Providing both user_prompt and output_schema gives the
best results.
Example:
{
"type": "object",
"properties": {
"stories": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": { "type": "string" },
"url": { "type": "string" },
"score": { "type": "integer" }
}
}
}
}
}
low (default) is faster and cheaper. Bump to high for
visually busy pages or schemas with many nested fields.
Available options:
low, high How exhaustively to populate the result.
Available options:
low, medium, high Cleaner mode. accurate is more forgiving on malformed pages;
speed is faster.
Available options:
accurate, speed Return the raw extracted text under result instead of a
parsed JSON object. Bypasses output_schema validation.
Whitelist of HTML tags to keep before extraction.
Blacklist of HTML tags to drop before extraction.
Trim long content before extraction. Helps on pages with lots of repetitive boilerplate. Uses a sensible default when omitted.
Opt in to an alternate extraction path that can do better on hard-to-parse pages. Behavior may change without notice.
Custom request headers forwarded to the fetcher. Providing any headers disables URL caching for this request.
Show child attributes
Show child attributes
URL-cache opt-in — same three-state semantics as /scrape
(omitted = no cache, 0 = bypass read but write on miss,
>0 = return entries fresher than N seconds).
Stealth requests, custom-headered requests, and URLs with query strings or fragments are never cached.
Required range:
x >= 0Use stealth mode. +5 credits.
Response
Successfully extracted.
Discriminator for the envelope variant.
Available options:
completed Per-request id. Also returned as the X-Request-ID response
header. Include it when reporting issues.
Example:
"req_aB3xY9Kp"
Show child attributes
Show child attributes
Example:
5
Example:
495
⌘I