Initiate a web page scrape

Initiate a web page scrape

Paid test badge: not yet. The checks above are free: we call the tool without paying and read the payment request it sends back. The Verified badge needs paid calls whose answers match the promised output, and nobody can buy a badge.

Endpoint

POST https://x402.orthogonal.com/olostep/v1/scrapes

Our checks, last 30 days

Example input (from the provider)

{
  "body": {
    "url_to_scrape": "url_to_scrape"
  },
  "bodyType": "json",
  "method": "POST",
  "type": "http"
}

Promised output schema (from the provider)

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "properties": {
    "input": {
      "additionalProperties": false,
      "properties": {
        "body": {
          "properties": {
            "actions": {
              "description": "Actions to perform on the page before getting the content.",
              "properties": {
                "milliseconds": {
                  "type": "number"
                },
                "type": {
                  "type": "string"
                }
              },
              "type": "object"
            },
            "country": {
              "description": "Residential country to load the request from. Supported values are: * US (United States) * CA (Canada) * IT (Italy) * IN (India) * GB (England) * JP (Japan) * MX (Mexico) * AU (Australia) * ID (Indonesia) * UA (UAE) * RU (Russia) * RANDOM Some operations, like scraping Google Search and Google News, support all countries.",
              "type": "string"
            },
            "formats": {
              "description": "Formats in which you want the content.",
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            "links_on_page": {
              "description": "With this option, you can get all the links present on the page you scrape.",
              "properties": {
                "absolute_links": {
                  "type": "boolean"
                },
                "exclude_links": {
                  "items": {
                    "type": "string"
                  },
                  "type": "array"
                },
                "include_links": {
                  "items": {
                    "type": "string"
                  },
                  "type": "array"
                },
                "query_to_order_links_by": {
                  "type": "string"
                }
              },
              "type": "object"
            },
            "llm_extract": {
              "properties": {
                "schema": {
                  "type": "object"
                }
              },
              "type": "object"
            },
            "metadata": {
              "description": "User-defined metadata. Not supported yet",
              "type": "object"
            },
            "parser": {
              "description": "When defining json as a format, you can use this parameter to specify the parser to use. Parsers are useful to extract structured content from web pages. Olostep has a few parsers built in for most common web pages, and you can also create your own parsers.",
              "properties": {
                "id": {
                  "type": "string"
                }
              },
              "type": "object"
            },
            "remove_class_names": {
              "description": "List of class names to remove from the content.",
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            "remove_css_selectors": {
              "description": "Option to remove certain CSS selectors from the content. Optionally, you can also pass a JSON stringified array of specific selectors you want to remove. The CSS selectors removed when this option is set to default are ['nav','footer','script','style','noscript','svg',[role=alert],[role=banner],[role=dialog],[role=alertdialog],[role=region][aria-label*=skip i],[aria-modal=true]] Available options: `default`, `none`, `array`",
              "type": "string"
            },
            "remove_images": {
              "description": "Option to remove images from the scraped content. Defaults to false.",
              "type": "boolean"
            },
            "screen_size": {
              "description": "Configuration for screen size. Preset dimensions are available through screen_type: desktop (1920x1080), mobile (414x896), or default (768x1024).",
              "properties": {
                "screen_height": {
                  "type": "number"
                },
                "screen_type": {
                  "type": "string"
                },
                "screen_width": {
                  "type": "number"
                }
              },
              "type": "object"
            },
            "transformer": {
              "description": "Specify the HTML transformer to use, if any. Postlight's Mercury Parser library is used to remove ads and other unwanted content from the scraped content. Available options: `postlight`, `none`",
              "type": "string"
            },
            "url_to_scrape": {
              "description": "The URL to start scraping from.",
              "type": "string"
            },
            "wait_before_scraping": {
              "description": "Time to wait in milliseconds before starting the scraping.",
              "type": "integer"
            }
          },
          "type": "object"
        },
        "bodyType": {
          "enum": [
            "json",
            "form-data",
            "text"
          ],
          "type": "string"
        },
        "method": {
          "enum": [
            "POST"
          ],
          "type": "string"
        },
        "type": {
          "const": "http",
          "type": "string"
        }
      },
      "required": [
        "type",
        "method",
        "bodyType",
        "body"
      ],
      "type": "object"
    }
  },
  "required": [
    "input"
  ],
  "type": "object"
}

This page as JSON