archivebot-docker/Dockerfile

FROM debian:stretch as builder
RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \
      build-essential dpkg-dev devscripts cmake git libmnl-dev \
      && rm -rf /var/lib/apt/lists/*

RUN git clone https://github.com/kristrev/tcp_closer.git /tcp_closer
RUN mkdir -p tcp_closer/src/build
WORKDIR /tcp_closer/src/build
RUN cmake ..
RUN make package

FROM python:3.6-stretch
RUN apt-get update && apt-get dist-upgrade -y && apt-get install -y \
      build-essential \
      libxml2-dev libxslt-dev zlib1g-dev libssl-dev libsqlite3-dev \
      libffi-dev git tmux fontconfig-config fonts-dejavu-core curl \
      libfontconfig1 libjpeg62-turbo-dev libjpeg62-turbo libjpeg-dev libjpeg-turbo-progs lsof ffmpeg \
      autossh rsync bundler git tmux python3 libtool pkg-config \
      build-essential autoconf automake libzmq3-dev libmnl0 \
      && rm -rf /var/lib/apt/lists/*

# Install tcp-closer
COPY --from=builder /tcp_closer/src/build/tcp-closer-0.1.1-Linux.deb /tcp-closer.deb
RUN dpkg -i /tcp-closer.deb

# Add user
RUN groupadd -g 1337 ab
RUN groupadd -r psudo
RUN useradd -rm -d /home/ab -s /bin/bash -g ab -G psudo -u 1337 ab

# Install python dependencies
RUN pip install websockets requests
RUN curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl

# Setup output directory
RUN mkdir -p /data
RUN mkdir -p /local-staging
RUN chown ab:ab /data
RUN chown ab:ab /local-staging
RUN chmod a+rx /usr/local/bin/youtube-dl

# Install tini
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini

# Give ab sudo for a bit
RUN echo "%psudo	ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/psudo
RUN chmod 0440 /etc/sudoers.d/psudo

USER ab

# Clone the repo
RUN git clone --recursive https://github.com/ArchiveTeam/ArchiveBot.git /home/ab/ArchiveBot
WORKDIR /home/ab/ArchiveBot

# Setup symlinks for pipeline
RUN ln -s /usr/local/bin/wpull /home/ab/ArchiveBot/pipeline/wpull
RUN rm /home/ab/ArchiveBot/pipeline/youtube-dl
RUN ln -s /usr/local/bin/youtube-dl /home/ab/ArchiveBot/pipeline/youtube-dl

# Setup env var
ENV OPENSSL_CONF=/home/ab/ArchiveBot/ops/openssl-less-secure.cnf

# Bundle install
RUN bundle install
RUN cd plumbing && bundle install && cd ..

# Install pip dependencies
USER root
RUN pip install -r pipeline/requirements.txt

# Copy in entrypoint
COPY entrypoint.sh /entrypoint.sh
COPY stager.sh /stager.sh
COPY wait-empty.sh /wait-empty.sh

# Button up image
USER ab
ENTRYPOINT [ "/tini", "--", "/entrypoint.sh" ]