Custom Headers

The custom headers feature lets you pass your own HTTP headers when making scraping requests. Enable it with custom_headers=true and include your headers in the request. This is particularly useful for accessing authentication-protected content or handling responses that require specific headers. There is no additional cost for this feature.

Endpoint: https://api.scrapingdog.com/scrape

API Examples

Code to Integrate

curl --header "X-customheader: bar" \
    "https://api.scrapingdog.com/scrape?api_key=APIKEY&url=https://www.scrapingdog.com&custom_headers=true"

import requests

headers = {"X-customheader": "bar"}
params = {
    "api_key": "APIKEY",
    "url": "https://www.scrapingdog.com",
    "custom_headers": "true"
}

response = requests.get("https://api.scrapingdog.com/scrape", headers=headers, params=params)
if response.status_code == 200:
    print(response.text)
else:
    print(f"Request failed with status code: {response.status_code}")

const axios = require('axios');

const headers = { 'X-customheader': 'bar' };
const params = {
  api_key: 'APIKEY',
  url: 'https://www.scrapingdog.com',
  custom_headers: 'true',
};

axios.get('https://api.scrapingdog.com/scrape', { headers, params })
  .then((response) => {
    if (response.status === 200) console.log(response.data);
  })
  .catch((error) => console.error('Error:', error.message));

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://api.scrapingdog.com/scrape?api_key=APIKEY&url=https://www.scrapingdog.com&custom_headers=true");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_HTTPHEADER, ['X-customheader: bar']);

$response = curl_exec($ch);
echo $response;
curl_close($ch);

require 'net/http'

url = URI.parse("https://api.scrapingdog.com/scrape?api_key=APIKEY&url=https://www.scrapingdog.com&custom_headers=true")
request = Net::HTTP::Get.new(url)
request['X-customheader'] = 'bar'

response = Net::HTTP.start(url.host, url.port, use_ssl: true) { |http| http.request(request) }
puts response.body

import java.io.*;
import java.net.*;

public class Main {
    public static void main(String[] args) throws Exception {
        String url = "https://api.scrapingdog.com/scrape?api_key=APIKEY&url=https://www.scrapingdog.com&custom_headers=true";
        HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
        conn.setRequestProperty("X-customheader", "bar");

        BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String line; StringBuilder sb = new StringBuilder();
        while ((line = in.readLine()) != null) sb.append(line);
        in.close();
        System.out.println(sb);
    }
}