diff --git a/Dockerfile b/Dockerfile index 93eccab..63d86d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,11 @@ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini RUN chmod +x /tini -#RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \ -# build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \ -# && rm -rf /var/lib/apt/lists/* +# Install ia cli tool. +RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \ + build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \ + && rm -rf /var/lib/apt/lists/* +RUN pip3 install internetarchive # Create data mount RUN mkdir -p /data @@ -15,13 +17,10 @@ RUN chown nobody:nogroup /data WORKDIR / +# Include files COPY mover.sh /mover.sh COPY entrypoint.sh /entrypoint.sh COPY uploader.sh /uploader.sh COPY upload-single.sh /upload-single.sh -#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt -#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py - -#RUN pip3 install -r /requirements.txt ENTRYPOINT [ "/tini", "--", "/entrypoint.sh" ] diff --git a/mover.sh b/mover.sh index e2ed1b8..a10e991 100755 --- a/mover.sh +++ b/mover.sh @@ -17,10 +17,10 @@ while [[ 1 ]] ; do export cur_size=$( du -B1 -s "${MOVER_WORKING_DIR}/current" | grep -oE "^[0-9]+" ) find "${INCOMING}" -type f -not -name ".*"\ - | while read filename + | while read -r filename do # skip partial uploads - if [[ $filename =~ rsync-tmp ]] + if [[ "${filename}" =~ rsync-tmp ]] then continue fi diff --git a/upload-single.sh b/upload-single.sh index bf7285b..72fdd42 100755 --- a/upload-single.sh +++ b/upload-single.sh @@ -11,17 +11,17 @@ fi set -x cd "$1" -upload="$(basename "$(pwd)")" +export upload="$(basename "$(pwd)")" export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" ) export YEAR=$(date +%Y) -echo ia upload "archiveteam_archivebot_go_${upload}" * \ +ia upload "archiveteam_archivebot_go_${upload}" * \ --sleep=120 \ --retries=100 \ --metadata="noarchivetorrent:true" \ --metadata="size-hint:${CUR_SIZE}" \ --metadata="mediatype:web" \ --metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \ - --metadata="title: Archiveteam: Archivebot GO Pack ${upload}" \ + --metadata="title:Archiveteam: Archivebot GO Pack ${upload}" \ --metadata="collection:archivebot" \ --metadata="creator:Archive Team" \ --metadata="date:${YEAR}" \ @@ -29,8 +29,5 @@ echo ia upload "archiveteam_archivebot_go_${upload}" * \ --verify \ --delete -# TEST SLEEP -sleep 6000 - cd .. -echo rmdir "$upload" +rmdir "$upload" diff --git a/uploader.sh b/uploader.sh index 0beaa6d..87d1872 100755 --- a/uploader.sh +++ b/uploader.sh @@ -23,6 +23,9 @@ do exit "${lock_rc}" fi fi + if [ -f ../STOP ]; then + exit 0 + fi done sleep 30 done