little-things/bing-scrape

#!/bin/bash
q="$1"
declare -i max=10000
if [[ $# -eq 2 ]]; then max=$2; fi
{
	declare -i first=1
	queryStr="q=${q}"
	while [[ ${first} -lt ${max} ]]
	do
		echo "http://www.bing.com/search?${queryStr}" >&2
		curl -s -A 'Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0' "http://www.bing.com/search?${queryStr}"
		first+=10
		queryStr="q=${q}&go=Search&qs=ds&first=${first}&FORM=PORE"
		sleep 2
	done
} | grep -Po '<li class="b_algo">.*?</li>' | grep -Po 'href="https://www\.bing\.com/ck/a\?!(.*&amp;)?u=a1\KaHR0c[^&]*' | while read -r ln; do echo "$(base64 -d <<<"${ln}" 2>/dev/null)"; done | awk '!seen[$0]++'