Browse Source

Fix small issues pointed out by JAA.

master
Roelf Wichertjes 2 years ago
parent
commit
3c9fc36096
Signed by: rewby GPG Key ID: 4C2B6D2972EE5423
4 changed files with 15 additions and 16 deletions
  1. +6
    -7
      Dockerfile
  2. +2
    -2
      mover.sh
  3. +4
    -7
      upload-single.sh
  4. +3
    -0
      uploader.sh

+ 6
- 7
Dockerfile View File

@@ -5,9 +5,11 @@ ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini

#RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \
# build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \
# && rm -rf /var/lib/apt/lists/*
# Install ia cli tool.
RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \
build-essential dpkg-dev devscripts cmake git python3 python3-dev python3-pip \
&& rm -rf /var/lib/apt/lists/*
RUN pip3 install internetarchive

# Create data mount
RUN mkdir -p /data
@@ -15,13 +17,10 @@ RUN chown nobody:nogroup /data

WORKDIR /

# Include files
COPY mover.sh /mover.sh
COPY entrypoint.sh /entrypoint.sh
COPY uploader.sh /uploader.sh
COPY upload-single.sh /upload-single.sh
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/pipeline/requirements.txt /requirements.txt
#ADD https://raw.githubusercontent.com/ArchiveTeam/ArchiveBot/master/uploader/uploader.py /uploader.py

#RUN pip3 install -r /requirements.txt

ENTRYPOINT [ "/tini", "--", "/entrypoint.sh" ]

+ 2
- 2
mover.sh View File

@@ -17,10 +17,10 @@ while [[ 1 ]] ; do
export cur_size=$( du -B1 -s "${MOVER_WORKING_DIR}/current" | grep -oE "^[0-9]+" )

find "${INCOMING}" -type f -not -name ".*"\
| while read filename
| while read -r filename
do
# skip partial uploads
if [[ $filename =~ rsync-tmp ]]
if [[ "${filename}" =~ rsync-tmp ]]
then
continue
fi


+ 4
- 7
upload-single.sh View File

@@ -11,17 +11,17 @@ fi
set -x

cd "$1"
upload="$(basename "$(pwd)")"
export upload="$(basename "$(pwd)")"
export CUR_SIZE=$( du -B1 -s . | grep -oE "^[0-9]+" )
export YEAR=$(date +%Y)
echo ia upload "archiveteam_archivebot_go_${upload}" * \
ia upload "archiveteam_archivebot_go_${upload}" * \
--sleep=120 \
--retries=100 \
--metadata="noarchivetorrent:true" \
--metadata="size-hint:${CUR_SIZE}" \
--metadata="mediatype:web" \
--metadata="description:ArchiveBot is an Archive Team service to quickly grab smaller at-risk or critical sites to bring copies into the Internet Archive Wayback machine." \
--metadata="title: Archiveteam: Archivebot GO Pack ${upload}" \
--metadata="title:Archiveteam: Archivebot GO Pack ${upload}" \
--metadata="collection:archivebot" \
--metadata="creator:Archive Team" \
--metadata="date:${YEAR}" \
@@ -29,8 +29,5 @@ echo ia upload "archiveteam_archivebot_go_${upload}" * \
--verify \
--delete

# TEST SLEEP
sleep 6000

cd ..
echo rmdir "$upload"
rmdir "$upload"

+ 3
- 0
uploader.sh View File

@@ -23,6 +23,9 @@ do
exit "${lock_rc}"
fi
fi
if [ -f ../STOP ]; then
exit 0
fi
done
sleep 30
done

Loading…
Cancel
Save