@@ -5,9 +5,11 @@ ENV TINI_VERSION v0.19.0 | |||||
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini | ||||
RUN chmod +x /tini | RUN chmod +x /tini | ||||
#RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \ | |||||
# build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \ | |||||
# && rm -rf /var/lib/apt/lists/* | |||||
# Install ia cli tool. | |||||
RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \ | |||||
build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \ | |||||
&& rm -rf /var/lib/apt/lists/* | |||||
RUN pip3 install internetarchive | |||||
# Create data mount | # Create data mount | ||||
RUN mkdir -p /data | RUN mkdir -p /data | ||||
@@ -15,13 +17,10 @@ RUN chown nobody:nogroup /data | |||||
WORKDIR / | WORKDIR / | ||||
# Include files | |||||
COPY mover.sh /mover.sh | COPY mover.sh /mover.sh | ||||
COPY entrypoint.sh /entrypoint.sh | COPY entrypoint.sh /entrypoint.sh | ||||
COPY uploader.sh /uploader.sh | COPY uploader.sh /uploader.sh | ||||
COPY upload-single.sh /upload-single.sh | COPY upload-single.sh /upload-single.sh | ||||
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt | |||||
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py | |||||
#RUN pip3 install -r /requirements.txt | |||||
ENTRYPOINT [ "/tini", "--", "/entrypoint.sh" ] | ENTRYPOINT [ "/tini", "--", "/entrypoint.sh" ] |
@@ -17,10 +17,10 @@ while [[ 1 ]] ; do | |||||
export cur_size=$( du -B1 -s "${MOVER_WORKING_DIR}/current" | grep -oE "^[0-9]+" ) | export cur_size=$( du -B1 -s "${MOVER_WORKING_DIR}/current" | grep -oE "^[0-9]+" ) | ||||
find "${INCOMING}" -type f -not -name ".*"\ | find "${INCOMING}" -type f -not -name ".*"\ | ||||
| while read filename | |||||
| while read -r filename | |||||
do | do | ||||
# skip partial uploads | # skip partial uploads | ||||
if [[ $filename =~ rsync-tmp ]] | |||||
if [[ "${filename}" =~ rsync-tmp ]] | |||||
then | then | ||||
continue | continue | ||||
fi | fi | ||||
@@ -11,17 +11,17 @@ fi | |||||
set -x | set -x | ||||
cd "$1" | cd "$1" | ||||
upload="$(basename "$(pwd)")" | |||||
export upload="$(basename "$(pwd)")" | |||||
export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" ) | export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" ) | ||||
export YEAR=$(date +%Y) | export YEAR=$(date +%Y) | ||||
echo ia upload "archiveteam_archivebot_go_${upload}" * \ | |||||
ia upload "archiveteam_archivebot_go_${upload}" * \ | |||||
--sleep=120 \ | --sleep=120 \ | ||||
--retries=100 \ | --retries=100 \ | ||||
--metadata="noarchivetorrent:true" \ | --metadata="noarchivetorrent:true" \ | ||||
--metadata="size-hint:${CUR_SIZE}" \ | --metadata="size-hint:${CUR_SIZE}" \ | ||||
--metadata="mediatype:web" \ | --metadata="mediatype:web" \ | ||||
--metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \ | --metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \ | ||||
--metadata="title: Archiveteam: Archivebot GO Pack ${upload}" \ | |||||
--metadata="title:Archiveteam: Archivebot GO Pack ${upload}" \ | |||||
--metadata="collection:archivebot" \ | --metadata="collection:archivebot" \ | ||||
--metadata="creator:Archive Team" \ | --metadata="creator:Archive Team" \ | ||||
--metadata="date:${YEAR}" \ | --metadata="date:${YEAR}" \ | ||||
@@ -29,8 +29,5 @@ echo ia upload "archiveteam_archivebot_go_${upload}" * \ | |||||
--verify \ | --verify \ | ||||
--delete | --delete | ||||
# TEST SLEEP | |||||
sleep 6000 | |||||
cd .. | cd .. | ||||
echo rmdir "$upload" | |||||
rmdir "$upload" |
@@ -23,6 +23,9 @@ do | |||||
exit "${lock_rc}" | exit "${lock_rc}" | ||||
fi | fi | ||||
fi | fi | ||||
if [ -f ../STOP ]; then | |||||
exit 0 | |||||
fi | |||||
done | done | ||||
sleep 30 | sleep 30 | ||||
done | done |