little-things/ia-cdx-search-subdomains

#!/bin/bash
set -euo pipefail

if [[ $# -eq 0 || "$1" == '-h' || "$1" == '--help' ]]
then
	echo "Usage: ia-cdx-search-subdomains [OPTIONS] DOMAIN" >&2
	echo "Extracts all known subdomains from IA's CDX API, printing unique subdomains to stdout" >&2
	echo "Any options are passed through to ia-cdx-search; use this to customise parallelism or resume after errors" >&2
	exit 1
fi

domain="${@: -1}"
set -- "${@:1:$(($#-1))}"
scriptpath="$(cd "$(dirname "$0")"; pwd -P)"
"${scriptpath}/ia-cdx-search" "$@" "url=${domain}&collapse=urlkey&fl=original&matchType=domain" | grep -o '^[^/]*//[^/"]*' | sed 's,^.*/,,; s,:[0-9]\+$,,' | "${scriptpath}/uniqify"