Code examples

The same search in several languages: find sites carrying a given script, page through the results, and back off politely when told to wait. Put your key in an environment variable rather than in the file.

The same, packaged as small clients with a command line - standard library only: Python, JavaScript, PHP, Go, Ruby. In a spreadsheet: Google Sheets. In a prospecting table: Clay.

curl

export PUBLICWWW_KEY=...

# One page of results
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' --data 'per_page=20' \
     https://api.publicwww.com/v1/search

# Every match, as a plain list of urls
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=txt&per_page=100000' \
     https://api.publicwww.com/v1/search > sites.txt

# A long, multi-line query is easier as a POST
curl https://api.publicwww.com/v1/search \
     -H "Authorization: Bearer $PUBLICWWW_KEY" \
     -H "Content-Type: application/json" \
     -d '{"query": ["\"angular.min.js\"", "\"bootstrap.min.css\""], "per_page": 50}'

Python

import os, time, requests

KEY  = os.environ["PUBLICWWW_KEY"]
BASE = "https://api.publicwww.com"

def search(query, **params):
    """One page of results, waiting out the rate limit if asked to."""
    while True:
        r = requests.get(BASE + "/v1/search",
                         headers={"Authorization": "Bearer " + KEY},
                         params={"query": query, **params})
        if r.status_code == 429:
            body = r.json()["error"]
            if body["code"] != "too_many_requests":
                raise RuntimeError(body["message"])       # quota, not pace
            time.sleep(int(r.headers.get("Retry-After", 30)))
            continue
        r.raise_for_status()
        return r.json()

page = search('"angular.min.js"', per_page=20)
print(page["total"], "sites match")
for row in page["results"]:
    print(row["rank"], row["domain"], row["url"])

Streaming a large result set

import json, os, requests

r = requests.get("https://api.publicwww.com/v1/search",
                 headers={"Authorization": "Bearer " + os.environ["PUBLICWWW_KEY"]},
                 params={"query": '"angular.min.js"',
                         "format": "ndjson", "per_page": 1000000},
                 stream=True)
r.raise_for_status()

meta = None
for line in r.iter_lines():
    if not line:
        continue
    obj = json.loads(line)
    if obj.get("object") == "meta":          # always the first line
        meta = obj
        print("expecting", meta["returned"], "of", meta["total"])
        continue
    print(obj["domain"])

PHP

<?php
$key = getenv ("PUBLICWWW_KEY");

$url = "https://api.publicwww.com/v1/search?" . http_build_query (array (
	"query"    => '"angular.min.js"',
	"per_page" => 20,
));

$ch = curl_init ($url);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt ($ch, CURLOPT_HTTPHEADER, array ("Authorization: Bearer " . $key));

$body   = curl_exec ($ch);
$status = curl_getinfo ($ch, CURLINFO_HTTP_CODE);
curl_close ($ch);

$data = json_decode ($body, true);

if ($status != 200) {
	die ($data ["error"] ["code"] . ": " . $data ["error"] ["message"] . "\n");
}

echo $data ["total"] . " sites match\n";
foreach ($data ["results"] as $r) {
	echo $r ["rank"] . "\t" . $r ["domain"] . "\t" . $r ["url"] . "\n";
}

JavaScript (Node)

const KEY  = process.env.PUBLICWWW_KEY;
const BASE = "https://api.publicwww.com";

async function search (query, params = {}) {
  const url = new URL (BASE + "/v1/search");
  url.searchParams.set ("query", query);
  for (const [k, v] of Object.entries (params)) url.searchParams.set (k, v);

  for (;;) {
    const res  = await fetch (url, { headers: { Authorization: `Bearer ${KEY}` } });
    const body = await res.json ();

    if (res.status === 429 && body.error.code === "too_many_requests") {
      await new Promise (r => setTimeout (r, body.error.retry_after * 1000));
      continue;                                   // told to wait, not refused
    }
    if (!res.ok) throw new Error (`${body.error.code}: ${body.error.message}`);
    return body;
  }
}

const page = await search ('"angular.min.js"', { per_page: 20 });
console.log (page.total, "sites match");
for (const r of page.results) console.log (r.rank, r.domain, r.url);

Go

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
)

type Result struct {
	Domain string `json:"domain"`
	URL    string `json:"url"`
	Rank   *int   `json:"rank"`          // nil when the site has no rank
}

type Page struct {
	Total   int      `json:"total"`
	Results []Result `json:"results"`
}

func main () {
	q := url.Values{}
	q.Set ("query", `"angular.min.js"`)
	q.Set ("per_page", "20")

	req, _ := http.NewRequest ("GET", "https://api.publicwww.com/v1/search?"+q.Encode (), nil)
	req.Header.Set ("Authorization", "Bearer "+os.Getenv ("PUBLICWWW_KEY"))

	res, err := http.DefaultClient.Do (req)
	if err != nil { panic (err) }
	defer res.Body.Close ()

	if res.StatusCode != 200 {
		fmt.Println ("http", res.StatusCode, res.Header.Get ("Retry-After"))
		os.Exit (1)
	}

	var page Page
	json.NewDecoder (res.Body).Decode (&page)

	fmt.Println (page.Total, "sites match")
	for _, r := range page.Results {
		fmt.Println (r.Domain, r.URL)
	}
}

Ruby

require "json"
require "net/http"

KEY = ENV.fetch ("PUBLICWWW_KEY")

uri = URI ("https://api.publicwww.com/v1/search")
uri.query = URI.encode_www_form (query: '"angular.min.js"', per_page: 20)

req = Net::HTTP::Get.new (uri)
req["Authorization"] = "Bearer #{KEY}"

res = Net::HTTP.start (uri.host, uri.port, use_ssl: true) { |http| http.request (req) }
body = JSON.parse (res.body)

abort "#{body['error']['code']}: #{body['error']['message']}" unless res.code == "200"

puts "#{body['total']} sites match"
body["results"].each { |r| puts [r["rank"], r["domain"], r["url"]].join ("\t") }

Loading into a spreadsheet or a shell pipeline

# A csv of domain and rank, ready for a spreadsheet
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&header=1&delimiter=,&per_page=10000' \
     https://api.publicwww.com/v1/search > sites.csv

# Just the domains, one per line, into whatever comes next
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&columns=domain&per_page=10000' \
     https://api.publicwww.com/v1/search | sort -u

Things worth getting right

  • Tell the two 429s apart. too_many_requests means slow down and repeat. quota_exceeded means come back tomorrow; retrying in a loop will not help and will look like an attack.
  • Honour Retry-After rather than guessing an interval. It is the exact number of seconds until a slot frees.
  • Check truncated before treating a result set as complete.
  • Expect rank: null. Plenty of indexed sites have no ranking; a language with non-nullable integers needs a pointer or an option type, as the Go example shows.
  • Page with page rather than re-running variations of the query to reach further rows.
Next Google Sheets