2 Commits

Author SHA1 Message Date
  JustAnotherArchivist fc5e2dcb5b Cleanup 3 years ago
  JustAnotherArchivist 581332d9c9 Split up into a proper build with context 3 years ago
5 changed files with 57 additions and 30 deletions
Split View
  1. +18
    -30
      backend/Dockerfile
  2. +7
    -0
      backend/README.md
  3. +12
    -0
      backend/launch-archivebot.sh
  4. +7
    -0
      backend/launch-root.sh
  5. +13
    -0
      backend/redis.conf

Dockerfile.backend → backend/Dockerfile View File

@@ -1,5 +1,7 @@
FROM debian:buster-slim
ENV LC_ALL=C

# Software installation
RUN echo 'deb http://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list \
&& DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \
&& DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install apt-utils \
@@ -10,50 +12,36 @@ RUN echo 'deb http://deb.debian.org/debian buster-backports main' >/etc/apt/sour
&& DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install build-essential ruby ruby-dev bundler python3 python3-websockets redis-server couchdb=2.3.1-1~buster git libzmq5 libzmq3-dev \
&& DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd \
&& rm -rf /var/lib/apt/lists/*

# Set up things under archivebot user and launch scripts
RUN groupadd -r archivebot && useradd -r -m -g archivebot archivebot
USER archivebot
RUN git clone https://github.com/ArchiveTeam/ArchiveBot.git /home/archivebot/ArchiveBot \
&& cd /home/archivebot/ArchiveBot \
&& git checkout ed1feffa53a9dec2029ce8a14cd4d20e13673a61 \
&& rm Gemfile.lock \
&& sed -i "s|, '~> 3\.0\.2'||; s/gem 'webmachine', .*$/gem 'webmachine', '~> 1.2.2'/; s/gem 'reel'/gem 'reel', '~> 0.4.0'/" Gemfile \
&& GEM_HOME=/home/archivebot/.gems bundle install \
&& cd /home/archivebot/ArchiveBot/plumbing \
&& rm Gemfile.lock \
&& GEM_HOME=/home/archivebot/.gems bundle install
RUN echo '#!/bin/bash' >/home/archivebot/launch.sh \
&& echo 'echo "Starting CouchDB"' >>/home/archivebot/launch.sh \
&& echo '/etc/init.d/couchdb start' >>/home/archivebot/launch.sh \
&& echo 'sleep 5' >>/home/archivebot/launch.sh \
&& echo 'echo "Dropping privileges"' >>/home/archivebot/launch.sh \
&& echo 'setpriv --reuid=archivebot --regid=archivebot --init-groups /home/archivebot/launch2.sh' >>/home/archivebot/launch.sh \
&& chmod +x /home/archivebot/launch.sh \
&& echo '#!/bin/bash' >/home/archivebot/launch2.sh \
&& echo 'export GEM_HOME=/home/archivebot/.gems' >>/home/archivebot/launch2.sh \
&& echo 'echo "Starting Redis"' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/redis && redis-server redis.conf) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot/bot && bundle exec ruby bot.rb -s "ircs://irc.hackint.org:6697" -r "redis://127.0.0.1:6379/0" -c "#notarchivebot" -n Puppeteer) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot && export REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 && plumbing/updates-listener | plumbing/log-firehose) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot && bundle exec ruby dashboard/app.rb -u http://0.0.0.0:4567) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot && FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 plumbing/firehose-client | python3 dashboard/websocket.py) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot/cogs && bundle exec ruby start.rb) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./analyzer) &' >>/home/archivebot/launch2.sh \
&& echo '(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./trimmer >/dev/null) &' >>/home/archivebot/launch2.sh \
&& echo 'sleep inf' >>/home/archivebot/launch2.sh \
&& chmod +x /home/archivebot/launch2.sh \
&& mkdir /home/archivebot/redis \
&& echo 'dir ./' >/home/archivebot/redis/redis.conf
&& GEM_HOME=/home/archivebot/.gems bundle install \
&& mkdir -p /home/archivebot/redis/data/
COPY --chown=root:root launch-root.sh /root/
COPY --chown=archivebot:archivebot launch-archivebot.sh /home/archivebot/
COPY --chown=archivebot:archivebot redis.conf /home/archivebot/redis/
VOLUME /home/archivebot/redis/data/

USER root
RUN \
# Set up CouchDB
/etc/init.d/couchdb start \
&& sleep 5 \
&& cat /opt/couchdb/var/log/couchdb.log \
&& curl -sf http://127.0.0.1:5984/ \

# Set up CouchDB
RUN /etc/init.d/couchdb start \
&& ( set +e; for i in $(seq 10); do curl -sf http://127.0.0.1:5984/ && exit 0 || sleep 1; done; echo 'CouchDB still not operational after 10 seconds'; exit 1; ) \
&& curl -sf -X PUT http://127.0.0.1:5984/_users \
&& curl -sf -X PUT http://127.0.0.1:5984/_replicator \
&& curl -sf -X PUT http://127.0.0.1:5984/archivebot \
&& grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/ignore_patterns.json | curl -sf -X PUT --data @- http://127.0.0.1:5984/archivebot/_design/ignore_patterns \
&& grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/user_agents.json | curl -sf -X PUT --data @- http://127.0.0.1:5984/archivebot/_design/user_agents

EXPOSE 4567
EXPOSE 4568
ENTRYPOINT ["/home/archivebot/launch.sh"]
ENTRYPOINT ["/root/launch-root.sh"]

+ 7
- 0
backend/README.md View File

@@ -0,0 +1,7 @@
Build:

docker build -t archivebot-backend .

Run:

docker run -d -p 80:4567 -p 4568:4568 -v abredis:/home/archivebot/redis/data/ archivebot-backend

+ 12
- 0
backend/launch-archivebot.sh View File

@@ -0,0 +1,12 @@
#!/bin/bash
export GEM_HOME=/home/archivebot/.gems
echo "Starting Redis"
(cd /home/archivebot/redis && redis-server redis.conf) &
(cd /home/archivebot/ArchiveBot/bot && bundle exec ruby bot.rb -s "ircs://irc.hackint.org:6697" -r "redis://127.0.0.1:6379/0" -c "#notarchivebot" -n Puppeteer) &
(cd /home/archivebot/ArchiveBot && export REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 && plumbing/updates-listener | plumbing/log-firehose) &
(cd /home/archivebot/ArchiveBot && bundle exec ruby dashboard/app.rb -u http://0.0.0.0:4567) &
(cd /home/archivebot/ArchiveBot && FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 plumbing/firehose-client | python3 dashboard/websocket.py) &
(cd /home/archivebot/ArchiveBot/cogs && bundle exec ruby start.rb) &
(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./analyzer) &
(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./trimmer >/dev/null) &
sleep inf

+ 7
- 0
backend/launch-root.sh View File

@@ -0,0 +1,7 @@
#!/bin/bash
set -e
echo "Starting CouchDB"
/etc/init.d/couchdb start
( set +e; for i in $(seq 10); do curl -sf http://127.0.0.1:5984/ && exit 0 || sleep 1; done; echo 'CouchDB still not operational after 10 seconds'; exit 1; )
echo "Dropping privileges"
setpriv --reuid=archivebot --regid=archivebot --init-groups /home/archivebot/launch-archivebot.sh

+ 13
- 0
backend/redis.conf View File

@@ -0,0 +1,13 @@
# Debian default is yes; restore upstream default
daemonize no

# Log to stdout (= Docker's logs)
logfile ""

# Enable append-only file for near-continuous saving
appendonly yes

# Data in Docker volume
dir ./data/
dbfilename dump.rdb
appendfilename appendonly.aof

Loading…
Cancel
Save