643 matches
Other/Dataflow Kit Web Scraper
To crawl search engine result pages, you can use /serp endpoint. SERP collection service extracts a list of organic results, news, images, and more. Specify configuration parameters, such as country or languages, to customize output SERP data.
The following search engines are supported
Generate ready-to-run code for your favorite language at
{
"type": "object",
"required": [
"format",
"name",
"proxy",
"type",
"url"
],
"properties": {
"fields": {
"type": "array",
"description": "Specify CSS selectors (patterns) used to gather data from Search Engine Result Pages.\n\nReady-to-use payloads for collecting search results from the most popular Search Engines are available. These payloads are customizable, though.\n",
"items": {
"type": "object",
"required": [
"attrs",
"name",
"selector",
"type"
],
"properties": {
"attrs": {
"type": "array",
"description": "A set of attributes to extract from a Field. Find more information about attributes",
"items": {
"type": "string",
"enum": [
"text",
"href",
"src",
"alt"
]
}
},
"details": {
"description": "Details themself represent independent Parse request that extracts data from linked pages.",
"type": "object"
},
"filters": {
"type": "array",
"description": "Filters are used to pre-processing of text data when extracting.",
"items": {
"type": "object"
}
},
"name": {
"type": "string",
"description": "Field name is used to aggregate results."
},
"selector": {
"type": "string",
"description": "Selector represents a CSS selector for data extraction within the given block."
},
"type": {
"type": "integer",
"description": "Selector type. ( 0 - image, 1 - text, 2 - link)",
"enum": [
0,
1,
2
]
}
}
}
},
"format": {
"type": "string",
"description": "Extracted data is returned either in CSV, MS Excel, JSON, JSON(Lines) or XML format.",
"enum": [
"csv",
"json",
"jsonl",
"excel",
"xml"
]
},
"name": {
"type": "string",
"description": "Collection name."
},
"pageNum": {
"type": "integer",
"description": "Specify number of pages to crawl."
},
"proxy": {
"type": "string",
"description": "Always specify proxy for sending SERP requests. Add choosen [country ISO code](https://en.wikipedia.org/wiki/ISO_3166-2) to `country-` value to send requests through a proxy in the specified country. Use `country-any` to use random geo-targets."
},
"type": {
"type": "string",
"description": "For SERP requests you should _always_ use `chrome` type to fetch content with a Headless chrome browser"
},
"url": {
"type": "string",
"description": "url holds the link to a Search Engine to use, and other optional parameters like languages or country."
}
}
}{
"type": "object"
}