# Code examples

The same search in several languages: find sites carrying a given script, page through the results, and back off politely when told to wait. Put your key in an environment variable rather than in the file.

The same, packaged as small clients with a command line - standard library only: [Python](https://github.com/publicwww-com/publicwww-api-python), [JavaScript](https://github.com/publicwww-com/publicwww-api-javascript), [PHP](https://github.com/publicwww-com/publicwww-api-php), [Go](https://github.com/publicwww-com/publicwww-api-go), [Ruby](https://github.com/publicwww-com/publicwww-api-ruby). In a spreadsheet: [Google Sheets](https://publicwww.com/docs/api/sheets/). In a prospecting table: [Clay](https://publicwww.com/docs/api/clay/).

## curl

```
export PUBLICWWW_KEY=...

# One page of results
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
--get --data-urlencode 'query="angular.min.js"' --data 'per_page=20' \
https://api.publicwww.com/v1/search

# Every match, as a plain list of urls
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
--get --data-urlencode 'query="angular.min.js"' \
--data 'format=txt&per_page=100000' \
https://api.publicwww.com/v1/search > sites.txt

# A long, multi-line query is easier as a POST
curl https://api.publicwww.com/v1/search \
-H "Authorization: Bearer $PUBLICWWW_KEY" \
-H "Content-Type: application/json" \
-d '{"query": ["\"angular.min.js\"", "\"bootstrap.min.css\""], "per_page": 50}'
```

## Python

```
import os, time, requests

KEY  = os.environ["PUBLICWWW_KEY"]
BASE = "https://api.publicwww.com"

def search(query, **params):
"""One page of results, waiting out the rate limit if asked to."""
while True:
r = requests.get(BASE + "/v1/search",
headers={"Authorization": "Bearer " + KEY},
params={"query": query, **params})
if r.status_code == 429:
body = r.json()["error"]
if body["code"] != "too_many_requests":
raise RuntimeError(body["message"])       # quota, not pace
time.sleep(int(r.headers.get("Retry-After", 30)))
continue
r.raise_for_status()
return r.json()

page = search('"angular.min.js"', per_page=20)
print(page["total"], "sites match")
for row in page["results"]:
print(row["rank"], row["domain"], row["url"])
```

### Streaming a large result set

```
import json, os, requests

r = requests.get("https://api.publicwww.com/v1/search",
headers={"Authorization": "Bearer " + os.environ["PUBLICWWW_KEY"]},
params={"query": '"angular.min.js"',
"format": "ndjson", "per_page": 1000000},
stream=True)
r.raise_for_status()

meta = None
for line in r.iter_lines():
if not line:
continue
obj = json.loads(line)
if obj.get("object") == "meta":          # always the first line
meta = obj
print("expecting", meta["returned"], "of", meta["total"])
continue
print(obj["domain"])
```

## PHP

```
<?php
$key = getenv ("PUBLICWWW_KEY");

$url = "https://api.publicwww.com/v1/search?" . http_build_query (array (
"query"    => '"angular.min.js"',
"per_page" => 20,
));

$ch = curl_init ($url);
curl_setopt ($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt ($ch, CURLOPT_HTTPHEADER, array ("Authorization: Bearer " . $key));

$body   = curl_exec ($ch);
$status = curl_getinfo ($ch, CURLINFO_HTTP_CODE);
curl_close ($ch);

$data = json_decode ($body, true);

if ($status != 200) {
die ($data ["error"] ["code"] . ": " . $data ["error"] ["message"] . "\n");
}

echo $data ["total"] . " sites match\n";
foreach ($data ["results"] as $r) {
echo $r ["rank"] . "\t" . $r ["domain"] . "\t" . $r ["url"] . "\n";
}
```

## JavaScript (Node)

```
const KEY  = process.env.PUBLICWWW_KEY;
const BASE = "https://api.publicwww.com";

async function search (query, params = {}) {
const url = new URL (BASE + "/v1/search");
url.searchParams.set ("query", query);
for (const [k, v] of Object.entries (params)) url.searchParams.set (k, v);

for (;;) {
const res  = await fetch (url, { headers: { Authorization: `Bearer ${KEY}` } });
const body = await res.json ();

if (res.status === 429 && body.error.code === "too_many_requests") {
await new Promise (r => setTimeout (r, body.error.retry_after * 1000));
continue;                                   // told to wait, not refused
}
if (!res.ok) throw new Error (`${body.error.code}: ${body.error.message}`);
return body;
}
}

const page = await search ('"angular.min.js"', { per_page: 20 });
console.log (page.total, "sites match");
for (const r of page.results) console.log (r.rank, r.domain, r.url);
```

## Go

```
package main

import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
)

type Result struct {
Domain string `json:"domain"`
URL    string `json:"url"`
Rank   *int   `json:"rank"`          // nil when the site has no rank
}

type Page struct {
Total   int      `json:"total"`
Results []Result `json:"results"`
}

func main () {
q := url.Values{}
q.Set ("query", `"angular.min.js"`)
q.Set ("per_page", "20")

req, _ := http.NewRequest ("GET", "https://api.publicwww.com/v1/search?"+q.Encode (), nil)
req.Header.Set ("Authorization", "Bearer "+os.Getenv ("PUBLICWWW_KEY"))

res, err := http.DefaultClient.Do (req)
if err != nil { panic (err) }
defer res.Body.Close ()

if res.StatusCode != 200 {
fmt.Println ("http", res.StatusCode, res.Header.Get ("Retry-After"))
os.Exit (1)
}

var page Page
json.NewDecoder (res.Body).Decode (&page)

fmt.Println (page.Total, "sites match")
for _, r := range page.Results {
fmt.Println (r.Domain, r.URL)
}
}
```

## Ruby

```
require "json"
require "net/http"

KEY = ENV.fetch ("PUBLICWWW_KEY")

uri = URI ("https://api.publicwww.com/v1/search")
uri.query = URI.encode_www_form (query: '"angular.min.js"', per_page: 20)

req = Net::HTTP::Get.new (uri)
req["Authorization"] = "Bearer #{KEY}"

res = Net::HTTP.start (uri.host, uri.port, use_ssl: true) { |http| http.request (req) }
body = JSON.parse (res.body)

abort "#{body['error']['code']}: #{body['error']['message']}" unless res.code == "200"

puts "#{body['total']} sites match"
body["results"].each { |r| puts [r["rank"], r["domain"], r["url"]].join ("\t") }
```

## Loading into a spreadsheet or a shell pipeline

```
# A csv of domain and rank, ready for a spreadsheet
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
--get --data-urlencode 'query="angular.min.js"' \
--data 'format=csv&header=1&delimiter=,&per_page=10000' \
https://api.publicwww.com/v1/search > sites.csv

# Just the domains, one per line, into whatever comes next
curl -H "Authorization: Bearer $PUBLICWWW_KEY" \
--get --data-urlencode 'query="angular.min.js"' \
--data 'format=csv&columns=domain&per_page=10000' \
https://api.publicwww.com/v1/search | sort -u
```

## Things worth getting right

- **Tell the two 429s apart.** `too_many_requests` means slow down and repeat. `quota_exceeded` means come back tomorrow; retrying in a loop will not help and will look like an attack.
- **Honour `Retry-After`** rather than guessing an interval. It is the exact number of seconds until a slot frees.
- **Check `truncated`** before treating a result set as complete.
- **Expect `rank: null`.** Plenty of indexed sites have no ranking; a language with non-nullable integers needs a pointer or an option type, as the Go example shows.
- **Page with `page`** rather than re-running variations of the query to reach further rows.

Source: https://publicwww.com/docs/api/examples/