> ## Documentation Index
> Fetch the complete documentation index at: https://webscrape.ai/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Structured extraction with JSON-schema validation

> Give a URL and a `user_prompt` (and optionally an `output_schema`),
get JSON back. Handles long content, noisy pages, and complex
schemas.

**Cost:** 5 credits (+5 with `stealth: true`).




## OpenAPI

````yaml /openapi.yaml post /smartscraper
openapi: 3.1.0
info:
  title: webscrape.ai API
  version: '1.0'
  description: |
    The public webscrape.ai API. Fetch pages and get back structured JSON,
    HTML, markdown, or replayed browser output.

    Every response is wrapped in one of three envelope shapes —
    `completed`, `queued`, or `error`. Always check the top-level
    `status` first, then read `data` (success or queued) or `error`
    (failure).
  contact:
    name: webscrape.ai support
    email: hello@webscrape.ai
    url: https://webscrape.ai
servers:
  - url: https://api.webscrape.ai/v1
    description: Production
security:
  - ApiKeyAuth: []
tags:
  - name: Scrape
    description: Raw fetching and structured extraction.
  - name: SmartBrowse
    description: Real-Chrome recipe replay with pagination and interactions.
paths:
  /smartscraper:
    post:
      tags:
        - Scrape
      summary: Structured extraction with JSON-schema validation
      description: |
        Give a URL and a `user_prompt` (and optionally an `output_schema`),
        get JSON back. Handles long content, noisy pages, and complex
        schemas.

        **Cost:** 5 credits (+5 with `stealth: true`).
      operationId: smartscraper
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SmartScraperRequest'
      responses:
        '200':
          description: Successfully extracted.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SmartScraperSuccess'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '402':
          $ref: '#/components/responses/PaymentRequired'
        '403':
          $ref: '#/components/responses/Forbidden'
        '409':
          $ref: '#/components/responses/Conflict'
        '422':
          $ref: '#/components/responses/ValidationFailed'
        '429':
          $ref: '#/components/responses/RateLimited'
        '500':
          $ref: '#/components/responses/InternalError'
        '502':
          $ref: '#/components/responses/ExtractionUnavailable'
components:
  schemas:
    SmartScraperRequest:
      type: object
      required:
        - website_url
        - user_prompt
      properties:
        website_url:
          type: string
          format: uri
          example: https://news.ycombinator.com
        user_prompt:
          type: string
          description: Plain-English description of what to extract.
          example: Extract the front-page stories with title, url, and score.
        output_schema:
          type: object
          additionalProperties: true
          description: |
            JSON Schema enforced on the extracted output. When supplied,
            the result is validated against the schema and one repair
            attempt is made before returning `validation_failed`.
            Providing both `user_prompt` and `output_schema` gives the
            best results.
          example:
            type: object
            properties:
              stories:
                type: array
                items:
                  type: object
                  properties:
                    title:
                      type: string
                    url:
                      type: string
                    score:
                      type: integer
        page_complexity:
          type: string
          enum:
            - low
            - high
          default: low
          description: |
            `low` (default) is faster and cheaper. Bump to `high` for
            visually busy pages or schemas with many nested fields.
        detail_level:
          type: string
          enum:
            - low
            - medium
            - high
          default: medium
          description: How exhaustively to populate the result.
        parse_mode:
          type: string
          enum:
            - accurate
            - speed
          default: accurate
          description: |
            Cleaner mode. `accurate` is more forgiving on malformed pages;
            `speed` is faster.
        plain_text:
          type: boolean
          default: false
          description: |
            Return the raw extracted text under `result` instead of a
            parsed JSON object. Bypasses `output_schema` validation.
        include_tags:
          type: array
          items:
            type: string
          description: Whitelist of HTML tags to keep before extraction.
        exclude_tags:
          type: array
          items:
            type: string
          description: Blacklist of HTML tags to drop before extraction.
        reduce_content:
          type: boolean
          description: |
            Trim long content before extraction. Helps on pages with lots
            of repetitive boilerplate. Uses a sensible default when omitted.
        experimental:
          type: boolean
          default: false
          description: |
            Opt in to an alternate extraction path that can do better on
            hard-to-parse pages. Behavior may change without notice.
        headers:
          type: object
          additionalProperties:
            type: string
          description: |
            Custom request headers forwarded to the fetcher. Providing
            any headers disables URL caching for this request.
        max_age:
          type: integer
          minimum: 0
          description: |
            URL-cache opt-in — same three-state semantics as `/scrape`
            (omitted = no cache, 0 = bypass read but write on miss,
            `>0` = return entries fresher than N seconds).

            Stealth requests, custom-headered requests, and URLs with
            query strings or fragments are never cached.
        stealth:
          type: boolean
          default: false
          description: Use stealth mode. +5 credits.
    SmartScraperSuccess:
      allOf:
        - $ref: '#/components/schemas/EnvelopeBase'
        - type: object
          required:
            - data
            - credits_used
            - credits_remaining
          properties:
            status:
              type: string
              enum:
                - completed
            data:
              $ref: '#/components/schemas/SmartScraperData'
            credits_used:
              type: integer
              example: 5
            credits_remaining:
              type: integer
              example: 495
    EnvelopeBase:
      type: object
      required:
        - status
        - request_id
      properties:
        status:
          type: string
          enum:
            - completed
            - queued
            - error
          description: Discriminator for the envelope variant.
        request_id:
          type: string
          example: req_aB3xY9Kp
          description: |
            Per-request id. Also returned as the `X-Request-ID` response
            header. Include it when reporting issues.
    SmartScraperData:
      type: object
      required:
        - request_id
      properties:
        request_id:
          type: string
          description: Extraction id. Distinct from the envelope's `request_id`.
        result:
          description: |
            The extracted output.

              * When `output_schema` is supplied → a JSON object that
                matches the schema.
              * When `plain_text: true` → a string.
              * Otherwise → a free-shape JSON object inferred from
                `user_prompt`.
          oneOf:
            - type: object
              additionalProperties: true
            - type: array
              items: {}
            - type: string
        latency_ms:
          type: integer
          description: Total extraction time in milliseconds.
    EnvelopeError:
      allOf:
        - $ref: '#/components/schemas/EnvelopeBase'
        - type: object
          required:
            - error
          properties:
            status:
              type: string
              enum:
                - error
            error:
              type: object
              required:
                - code
                - message
              properties:
                code:
                  $ref: '#/components/schemas/ErrorCode'
                message:
                  type: string
                  description: >-
                    Human-readable. May change between releases. Log it; don't
                    pattern-match.
                details:
                  description: >-
                    Optional structured context (e.g. `{balance, required}` for
                    `insufficient_credits`).
    ErrorCode:
      type: string
      description: Stable error code. SDKs branch on this.
      enum:
        - invalid_request
        - unauthorized
        - insufficient_credits
        - email_verification_required
        - forbidden
        - not_found
        - conflict
        - account_deletion_pending
        - validation_failed
        - rate_limited
        - internal_error
        - service_unavailable
  responses:
    BadRequest:
      description: 'Malformed request: missing field, invalid JSON, invalid URL.'
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: invalid_request
              message: url is required
            request_id: req_aB3xY9Kp
    Unauthorized:
      description: Missing, invalid, or revoked API key.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: unauthorized
              message: missing or invalid credentials
            request_id: req_aB3xY9Kp
    PaymentRequired:
      description: |
        Out of credits OR email not verified. Branch on `error.code`:
        `insufficient_credits` (with `details: {balance, required}`) vs
        `email_verification_required`.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          examples:
            insufficient:
              summary: Out of credits
              value:
                status: error
                error:
                  code: insufficient_credits
                  message: not enough credits to complete this request
                  details:
                    balance: 2
                    required: 5
                request_id: req_aB3xY9Kp
            unverified:
              summary: Email not verified
              value:
                status: error
                error:
                  code: email_verification_required
                  message: verify your email address before spending credits
                request_id: req_aB3xY9Kp
    Forbidden:
      description: |
        The caller is authenticated but not allowed to perform this
        action.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: forbidden
              message: forbidden
            request_id: req_aB3xY9Kp
    Conflict:
      description: |
        Resource is in a state that disallows the action. The most
        common case is `account_deletion_pending`: the account is in
        the 30-day deletion grace window and cannot spend credits until
        deletion is cancelled.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: account_deletion_pending
              message: account is scheduled for deletion — cancel to restore access
              details:
                deletion_scheduled_for: '2026-06-15T12:00:00Z'
            request_id: req_aB3xY9Kp
    ValidationFailed:
      description: |
        The request was accepted but extraction failed logically — the
        output didn't match the requested schema after one repair
        attempt, or the extraction itself errored.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: validation_failed
              message: >-
                Output did not match the requested schema after one repair
                attempt.
            request_id: req_aB3xY9Kp
    RateLimited:
      description: |
        Per-plan throttle exceeded. `error.details.reason` distinguishes
        the cause:

          * `rate_limit_per_min`     — too many requests per minute.
          * `max_concurrent_requests` — too many in-flight at once.
          * `sb_runs_per_month`       — SmartBrowse run quota for the
                                        rolling 30-day window.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          examples:
            rate_per_min:
              summary: Request rate exceeds plan
              value:
                status: error
                error:
                  code: rate_limited
                  message: request rate exceeds plan limit — upgrade plan or slow down
                  details:
                    limit_per_min: 60
                    reason: rate_limit_per_min
                request_id: req_aB3xY9Kp
            max_concurrent:
              summary: Too many in-flight requests
              value:
                status: error
                error:
                  code: rate_limited
                  message: too many in-flight requests for plan — upgrade plan or wait
                  details:
                    max_concurrent: 5
                    reason: max_concurrent_requests
                request_id: req_aB3xY9Kp
            sb_runs:
              summary: SmartBrowse monthly run quota exceeded
              value:
                status: error
                error:
                  code: rate_limited
                  message: >-
                    smartbrowse run quota exceeded for plan — upgrade for more
                    runs
                  details:
                    used: 50
                    limit: 50
                    window: 30d
                    reason: sb_runs_per_month
                request_id: req_aB3xY9Kp
    InternalError:
      description: Internal server error. Safe to retry with backoff.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: internal_error
              message: internal error
            request_id: req_aB3xY9Kp
    ExtractionUnavailable:
      description: |
        Extraction service is temporarily unreachable. Safe to retry
        with backoff.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/EnvelopeError'
          example:
            status: error
            error:
              code: service_unavailable
              message: extraction service is unreachable
            request_id: req_aB3xY9Kp
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-Key
      description: |
        Generate from the [dashboard](https://webscrape.ai/app/api-keys).
        Format: `wsg_live_<32 base62 chars>`.

````