|
|
@@ -0,0 +1,165 @@ |
|
|
|
#!/bin/bash |
|
|
|
set -e |
|
|
|
|
|
|
|
if [[ $# -ne 2 || "$1" == '-h' || "$1" == '--help' ]] |
|
|
|
then |
|
|
|
echo 'Lists all relevant files in a Debian-style repository' >&2 |
|
|
|
echo >&2 |
|
|
|
echo 'Usage: deb-repo-ls ARCHIVEROOT DISTRIBUTION' >&2 |
|
|
|
exit 1 |
|
|
|
fi |
|
|
|
|
|
|
|
archiveRoot="$1" |
|
|
|
distribution="$2" |
|
|
|
|
|
|
|
while [[ "${archiveRoot}" == */ ]]; do archiveRoot="${archiveRoot%/}"; done # Strip trailing slashes since they're added again manually below |
|
|
|
if [[ "${distribution}" == */ || "${distribution}" == /* ]]; then echo "Invalid distribution" >&2; exit 1; fi |
|
|
|
|
|
|
|
declare -i fetchedSize |
|
|
|
declare -i totalSize=0 |
|
|
|
|
|
|
|
pipe=$(mktemp -u); mkfifo "${pipe}"; exec 3<>"${pipe}"; rm "${pipe}"; unset pipe # For fetch size |
|
|
|
|
|
|
|
function maybe_decompress { |
|
|
|
url="$1" |
|
|
|
if [[ "${url}" == *'.gz' ]] |
|
|
|
then |
|
|
|
gunzip |
|
|
|
elif [[ "${url}" == *'.bz2' ]] |
|
|
|
then |
|
|
|
bunzip2 |
|
|
|
elif [[ "${url}" == *'.xz' ]] |
|
|
|
then |
|
|
|
unxz |
|
|
|
else |
|
|
|
# Peek at the data to see if it's compressed |
|
|
|
beginning="$(head -c 10 | xxd -p)" |
|
|
|
{ |
|
|
|
xxd -p -r <<<"${beginning}" |
|
|
|
cat |
|
|
|
} | { |
|
|
|
if [[ "${beginning}" == '1f8b'* ]] |
|
|
|
then |
|
|
|
gunzip |
|
|
|
elif [[ "${beginning}" == '425a68'??'314159265359'* ]] |
|
|
|
then |
|
|
|
bunzip2 |
|
|
|
elif [[ "${beginning}" == 'fd377a585a00'* ]] |
|
|
|
then |
|
|
|
unxz |
|
|
|
else |
|
|
|
cat |
|
|
|
fi |
|
|
|
} |
|
|
|
fi |
|
|
|
} |
|
|
|
|
|
|
|
function fetch_and_print { |
|
|
|
url="$1" |
|
|
|
# Fetch url, print it to stdout; as side effects, the HTTP body is stored in fetchedBody and its size is added to totalSize |
|
|
|
# If the url ends with .gz, it's gunzipped; if it ends in .bz2, it gets bunzip2'd. The totalSize reflects the compressed size. |
|
|
|
echo "Fetching ${url}" >&2 |
|
|
|
fetchedBody="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s "${url}" | tee >(wc -c >&3) | maybe_decompress "${url}")" |
|
|
|
echo "${url}" |
|
|
|
#totalSize+=$(LANG=C; LC_ALL=C; echo ${#fetchedBody}) |
|
|
|
totalSize+=$(head -1 <&3) |
|
|
|
} |
|
|
|
|
|
|
|
function head_and_print { |
|
|
|
url="$1" |
|
|
|
# Issue a HEAD request on url, print it to stdout, and add its content length to totalSize |
|
|
|
echo "Heading ${url}" >&2 |
|
|
|
declare -i contentLength="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s --head "${url}" | grep -i '^Content-Length: ' | awk '{print $2}' | tr -d '\r')" |
|
|
|
echo "${url}" |
|
|
|
totalSize+=${contentLength} |
|
|
|
} |
|
|
|
|
|
|
|
# Retrieve Release and accompanying files, parse it and extract the next level |
|
|
|
fetch_and_print "${archiveRoot}/dists/${distribution}/Release" |
|
|
|
inHashSection= |
|
|
|
declare -A files # filename -> size |
|
|
|
while IFS='' read -r line |
|
|
|
do |
|
|
|
if [[ "${line}" == 'MD5Sum:' || "${line}" == 'SHA1:' || "${line}" == 'SHA256:' || "${line}" == 'SHA512:' ]] |
|
|
|
then |
|
|
|
inHashSection=1 |
|
|
|
continue |
|
|
|
fi |
|
|
|
if [[ "${line}" != ' '* ]] |
|
|
|
then |
|
|
|
inHashSection= |
|
|
|
continue |
|
|
|
fi |
|
|
|
if [[ "${inHashSection}" ]] |
|
|
|
then |
|
|
|
files["$(awk '{print $3}' <<<"${line}")"]="$(awk '{print $2}' <<<"${line}")" |
|
|
|
fi |
|
|
|
done <<<"${fetchedBody}" |
|
|
|
|
|
|
|
head_and_print "${archiveRoot}/dists/${distribution}/Release.gpg" |
|
|
|
head_and_print "${archiveRoot}/dists/${distribution}/InRelease" |
|
|
|
|
|
|
|
# Process files |
|
|
|
filename= |
|
|
|
declare -i size=-1 |
|
|
|
declare -A debfiles # filename -> size |
|
|
|
for fn in "${!files[@]}" |
|
|
|
do |
|
|
|
# Legacy releases |
|
|
|
if [[ "${fn}" =~ /Release$ ]] |
|
|
|
then |
|
|
|
echo "${archiveRoot}/dists/${distribution}/${fn}" |
|
|
|
totalSize+="${files[$fn]}" |
|
|
|
|
|
|
|
# Package indices |
|
|
|
elif [[ "${fn}" =~ /Packages(\.[^/]*)?$ ]] |
|
|
|
then |
|
|
|
echo "Processing package: ${fn}" |
|
|
|
fetch_and_print "${archiveRoot}/dists/${distribution}/${fn}" |
|
|
|
filename= |
|
|
|
size=-1 |
|
|
|
while IFS= read -r line |
|
|
|
do |
|
|
|
if [[ "${line}" == '' ]] |
|
|
|
then |
|
|
|
filename= |
|
|
|
size=-1 |
|
|
|
elif [[ "${line}" == 'Filename: '* ]] |
|
|
|
then |
|
|
|
filename="${line:10}" |
|
|
|
elif [[ "${line}" == 'Size: '* ]] |
|
|
|
then |
|
|
|
size="${line:6}" |
|
|
|
fi |
|
|
|
if [[ "${filename}" && ${size} -ge 0 ]] |
|
|
|
then |
|
|
|
debfiles["${archiveRoot}/dists/${distribution}/${filename}"]=${size} |
|
|
|
filename= |
|
|
|
size=-1 |
|
|
|
fi |
|
|
|
done <<<"${fetchedBody}" |
|
|
|
|
|
|
|
# Contents indices |
|
|
|
elif [[ "${fn}" =~ /Contents-[^/]*$ ]] |
|
|
|
then |
|
|
|
# Nothing really to do here but take note of its existence |
|
|
|
echo "${archiveRoot}/dists/${distribution}/${fn}" |
|
|
|
totalSize+="${files[$fn]}" |
|
|
|
|
|
|
|
# Anything else |
|
|
|
else |
|
|
|
echo "${archiveRoot}/dists/${distribution}/${fn}" |
|
|
|
totalSize+="${files[$fn]}" |
|
|
|
echo "Skipping unknown file: ${fn}" >&2 |
|
|
|
fi |
|
|
|
done |
|
|
|
|
|
|
|
# Print the debfiles and add them to the total size |
|
|
|
for debfile in "${!debfiles[@]}" |
|
|
|
do |
|
|
|
echo "${debfile}" |
|
|
|
totalSize+="${debfiles[$debfile]}" |
|
|
|
done |
|
|
|
|
|
|
|
# Report total size to stderr |
|
|
|
echo "Total size: ${totalSize} bytes" >&2 |