Code examples

The same search in several languages: find sites carrying a given script, page through the results, and back off politely when told to wait. Put your key in an environment variable rather than in the file.

curl

export PUBLICWWW_KEY=...

# One page of results
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' --data 'per_page=20' \
     https://api.publicwww.com/v1/search

# Every match, as a plain list of urls
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=txt&per_page=100000' \
     https://api.publicwww.com/v1/search > sites.txt

# A long, multi-line query is easier as a POST
curl https://api.publicwww.com/v1/search \
     -H "Authorization: Bearer $PUBLICWWW_KEY" \
     -H "Content-Type: application/json" \
     -d '{"query": ["\"angular.min.js\"", "\"bootstrap.min.css\""], "per_page": 50}'

Python

import os, time, requests

KEY  = os.environ["PUBLICWWW_KEY"]
BASE = "https://api.publicwww.com"

def search(query, **params):
    """One page of results, waiting out the rate limit if asked to."""
    while True:
        r = requests.get(BASE + "/v1/search",
                         headers={"Authorization": "Bearer " + KEY},
                         params={"query": query, **params})
        if r.status_code == 429:
            body = r.json()["error"]
            if body["code"] != "too_many_requests":
                raise RuntimeError(body["message"])       # quota, not pace
            time.sleep(int(r.headers.get("Retry-After", 30)))
            continue
        r.raise_for_status()
        return r.json()

page = search('"angular.min.js"', per_page=20)
print(page["total"], "sites match")
for row in page["results"]:
    print(row["rank"], row["domain"], row["url"])

Streaming a large result set

import json, os, requests

r = requests.get("https://api.publicwww.com/v1/search",
                 headers={"Authorization": "Bearer " + os.environ["PUBLICWWW_KEY"]},
                 params={"query": '"angular.min.js"',
                         "format": "ndjson", "per_page": 1000000},
                 stream=True)
r.raise_for_status()

meta = None
for line in r.iter_lines():
    if not line:
        continue
    obj = json.loads(line)
    if obj.get("object") == "meta":          # always the first line
        meta = obj
        print("expecting", meta["returned"], "of", meta["total"])
        continue
    print(obj["domain"])

PHP

<?php
$key = getenv ("PUBLICWWW_KEY");

$url = "https://api.publicwww.com/v1/search?" . http_build_query (array (
	"query"    => '"angular.min.js"',
	"per_page" => 20,
));

$ch = curl_init ($url);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt ($ch, CURLOPT_HTTPHEADER, array ("Authorization: Bearer " . $key));

$body   = curl_exec ($ch);
$status = curl_getinfo ($ch, CURLINFO_HTTP_CODE);
curl_close ($ch);

$data = json_decode ($body, true);

if ($status != 200) {
	die ($data ["error"] ["code"] . ": " . $data ["error"] ["message"] . "\n");
}

echo $data ["total"] . " sites match\n";
foreach ($data ["results"] as $r) {
	echo $r ["rank"] . "\t" . $r ["domain"] . "\t" . $r ["url"] . "\n";
}

JavaScript (Node)

const KEY  = process.env.PUBLICWWW_KEY;
const BASE = "https://api.publicwww.com";

async function search (query, params = {}) {
  const url = new URL (BASE + "/v1/search");
  url.searchParams.set ("query", query);
  for (const [k, v] of Object.entries (params)) url.searchParams.set (k, v);

  for (;;) {
    const res  = await fetch (url, { headers: { Authorization: `Bearer ${KEY}` } });
    const body = await res.json ();

    if (res.status === 429 && body.error.code === "too_many_requests") {
      await new Promise (r => setTimeout (r, body.error.retry_after * 1000));
      continue;                                   // told to wait, not refused
    }
    if (!res.ok) throw new Error (`${body.error.code}: ${body.error.message}`);
    return body;
  }
}

const page = await search ('"angular.min.js"', { per_page: 20 });
console.log (page.total, "sites match");
for (const r of page.results) console.log (r.rank, r.domain, r.url);

Go

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
)

type Result struct {
	Domain string `json:"domain"`
	URL    string `json:"url"`
	Rank   *int   `json:"rank"`          // nil when the site has no rank
}

type Page struct {
	Total   int      `json:"total"`
	Results []Result `json:"results"`
}

func main () {
	q := url.Values{}
	q.Set ("query", `"angular.min.js"`)
	q.Set ("per_page", "20")

	req, _ := http.NewRequest ("GET", "https://api.publicwww.com/v1/search?"+q.Encode (), nil)
	req.Header.Set ("Authorization", "Bearer "+os.Getenv ("PUBLICWWW_KEY"))

	res, err := http.DefaultClient.Do (req)
	if err != nil { panic (err) }
	defer res.Body.Close ()

	if res.StatusCode != 200 {
		fmt.Println ("http", res.StatusCode, res.Header.Get ("Retry-After"))
		os.Exit (1)
	}

	var page Page
	json.NewDecoder (res.Body).Decode (&page)

	fmt.Println (page.Total, "sites match")
	for _, r := range page.Results {
		fmt.Println (r.Domain, r.URL)
	}
}

Ruby

require "json"
require "net/http"

KEY = ENV.fetch ("PUBLICWWW_KEY")

uri = URI ("https://api.publicwww.com/v1/search")
uri.query = URI.encode_www_form (query: '"angular.min.js"', per_page: 20)

req = Net::HTTP::Get.new (uri)
req["Authorization"] = "Bearer #{KEY}"

res = Net::HTTP.start (uri.host, uri.port, use_ssl: true) { |http| http.request (req) }
body = JSON.parse (res.body)

abort "#{body['error']['code']}: #{body['error']['message']}" unless res.code == "200"

puts "#{body['total']} sites match"
body["results"].each { |r| puts [r["rank"], r["domain"], r["url"]].join ("\t") }

Loading into a spreadsheet or a shell pipeline

# A csv of domain and rank, ready for a spreadsheet
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&header=1&delimiter=,&per_page=10000' \
     https://api.publicwww.com/v1/search > sites.csv

# Just the domains, one per line, into whatever comes next
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
     --get --data-urlencode 'query="angular.min.js"' \
     --data 'format=csv&columns=domain&per_page=10000' \
     https://api.publicwww.com/v1/search | sort -u

Things worth getting right

Tell the two 429s apart. too_many_requests means slow down and repeat. quota_exceeded means come back tomorrow; retrying in a loop will not help and will look like an attack.
Honour Retry-After rather than guessing an interval. It is the exact number of seconds until a slot frees.
Check truncated before treating a result set as complete.
Expect rank: null. Plenty of indexed sites have no ranking; a language with non-nullable integers needs a pointer or an option type, as the Go example shows.
Page with page, not by re-querying with filters. Repeating a query you already ran today is not charged again.

Next The old export urls