643 matches
Other/Dataflow Kit Web Scraper
Dataflow kit uses CSS selectors to find HTML elements in web pages for later data extraction.
Open and click desired elements on a page to specify extracting data.
Then you can send generated payload to /parse endpoint. We crawl web pages and extract data like text, links, or images for you following the specified rules.
Extracted data is returned in CSV, MS Excel, JSON, JSON(Lines) or XML format.
{
"type": "object",
"required": [
"fields",
"format",
"name",
"proxy",
"type",
"url"
],
"properties": {
"commonParent": {
"type": "string",
"description": "Specifies common ancestor block for a set of fields used to extract data from a web page. _(CSS Selector)_"
},
"fields": {
"type": "array",
"description": "Define a set of fields used to extract data from a web page. A Field represents a given chunk of extracted data from every block on each page.\n",
"items": {
"type": "object",
"required": [
"attrs",
"name",
"selector",
"type"
],
"properties": {
"attrs": {
"type": "array",
"description": "A set of attributes to extract from a Field. Find more information about attributes",
"items": {
"type": "string",
"enum": [
"text",
"href",
"src",
"alt"
]
}
},
"details": {
"description": "Details themself represent independent Parse request that extracts data from linked pages.",
"type": "object"
},
"filters": {
"type": "array",
"description": "Filters are used to pre-processing of text data when extracting.",
"items": {
"type": "object"
}
},
"name": {
"type": "string",
"description": "Field name is used to aggregate results."
},
"selector": {
"type": "string",
"description": "Selector represents a CSS selector for data extraction within the given block."
},
"type": {
"type": "integer",
"description": "Selector type. ( 0 - image, 1 - text, 2 - link)",
"enum": [
0,
1,
2
]
}
}
}
},
"format": {
"type": "string",
"description": "Extracted data is returned either in CSV, MS Excel, JSON, JSON(Lines) or XML format.",
"enum": [
"csv",
"json",
"jsonl",
"excel",
"xml"
]
},
"name": {
"type": "string",
"description": "Collection name."
},
"paginator": {
"type": "object",
"description": "Specify _Next link_ paginator on pages containing a link pointing to the next page. The next page link is extracted from a document by querying href attribute of a given element's CSS selector.",
"properties": {
"nextPageSelector": {
"type": "string"
},
"pageNum": {
"type": "integer"
}
}
},
"path": {
"type": "boolean",
"description": "Path is a special parameter specifying navigation pages only. It collects information from detailed pages. No results from the current page return. Defaults to false."
},
"request": {
"type": "object",
"required": [
"type",
"url"
],
"properties": {
"actions": {
"type": "array",
"description": "Use actions to automate manual workflows while rendering web pages. They simulate real-world human interaction with pages. _(Chrome fetcher type only)_",
"items": {
"type": "object"
}
},
"ignoreHTTPStatusErrCodes": {
"type": "boolean",
"description": "The HTTP 200 OK success status response code indicates that the request has succeeded. Sometimes a server returns normal HTML content even with an erroneous Non-200 HTTP response status code. The IgnoreHTTPStatusCode option is useful when you need to force the return of HTML content. Defaults to \"false.\""
},
"initialCookies": {
"type": "array",
"description": "The \"Initial Cookies\" option is useful for crawling websites that require a login. The simplest solution to get an array of cookies for specific websites is to use a web browser \"EditThisCookie\" extension. Copy a cookie array with \"EditThisCookie\" and paste it into the \"Initial cookie\" field.",
"items": {
"type": "object",
"description": "InitialCookie structure keep cookies that optionally can be passed to the new fetcher crawl a website that requires a login. Generate Cookies array with EditThisCookie chrome extension.",
"properties": {
"domain": {
"type": "string"
},
"expirationDate": {
"type": "number"
},
"hostOnly": {
"type": "boolean"
},
"httpOnly": {
"type": "boolean"
},
"id": {
"type": "number"
},
"name": {
"type": "string"
},
"path": {
"type": "string"
},
"sameSite": {
"type": "string",
"enum": [
"unspecified",
"strict",
"lax",
"no_restriction"
]
},
"secure": {
"type": "boolean"
},
"session": {
"type": "boolean"
},
"storeID": {
"type": "string"
},
"value": {
"type": "string"
}
}
}
},
"output": {
"type": "string",
"description": "If set to _file_, the content of downloaded HTML is uploaded to Dataflow Kit Storage first. Then the link to this file is returned. Overwise, downloaded content is returned in the response body.",
"enum": [
"buffer",
"file"
]
},
"proxy": {
"type": "string",
"description": "Specify proxy by adding [country ISO code](https://en.wikipedia.org/wiki/ISO_3166-2) to `country-` value to send requests through a proxy in the specified country. Use `country-any` to use random geo-targets."
},
"type": {
"type": "string",
"description": "If set to `base`, the Base fetcher is used for downloading web page content. Use `chrome` for fetching content with a Headless chrome browser. If omitted `base` fetcher is used by default.",
"enum": [
"base",
"chrome"
]
},
"url": {
"type": "string",
"description": "Specify URL to download."
},
"waitDelay": {
"type": "number",
"description": "Specify a wait delay (in seconds). This may be useful if certain elements of the web site need to be rendered after the initial page load. _(Chrome fetcher type only)_"
}
}
}
}
}{
"type": "object"
}