From 776e75ad3cee942db92902e434ea83f387ea9e02 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Thu, 4 Mar 2021 02:19:29 +0000 Subject: [PATCH] Initial commit of WIP backend Dockerfile --- Dockerfile.backend | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 Dockerfile.backend diff --git a/Dockerfile.backend b/Dockerfile.backend new file mode 100644 index 0000000..6e84ef8 --- /dev/null +++ b/Dockerfile.backend @@ -0,0 +1,70 @@ +FROM debian:buster-slim +ENV LC_ALL=C +RUN echo 'deb http://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install apt-utils \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install gnupg ca-certificates \ + && echo 'deb https://apache.bintray.com/couchdb-deb buster main' >/etc/apt/sources.list.d/couchdb.list \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8756C4F765C9AC3CB6B85D62379CE192D401AB61 \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io update \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install build-essential ruby ruby-dev bundler python3 python3-websockets redis-server couchdb=2.3.1-1~buster git libzmq5 libzmq3-dev \ + && DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t buster-backports install zstd \ + && rm -rf /var/lib/apt/lists/* +RUN groupadd -r archivebot && useradd -r -m -g archivebot archivebot +USER archivebot +RUN git clone https://github.com/ArchiveTeam/ArchiveBot.git /home/archivebot/ArchiveBot \ + && cd /home/archivebot/ArchiveBot \ + && rm Gemfile.lock \ +# && sed -i "s/analysand\.git'$/analysand.git', ref: '4aa0f03426930440fde6a6878269b9bcf6762a50'/" Gemfile \ + && sed -i "s|, '~> 3\.0\.2'||; s/gem 'webmachine', .*$/gem 'webmachine', '~> 1.2.2'/; s/gem 'reel'/gem 'reel', '~> 0.4.0'/" Gemfile \ + && GEM_HOME=/home/archivebot/.gems bundle install \ + && cd /home/archivebot/ArchiveBot/plumbing \ + && rm Gemfile.lock \ + && GEM_HOME=/home/archivebot/.gems bundle install +RUN echo '#!/bin/bash' >/home/archivebot/launch.sh \ + && echo 'echo "Starting CouchDB"' >>/home/archivebot/launch.sh \ + && echo '/etc/init.d/couchdb start' >>/home/archivebot/launch.sh \ + && echo 'sleep 5' >>/home/archivebot/launch.sh \ + && echo 'echo "Dropping privileges"' >>/home/archivebot/launch.sh \ + && echo 'setpriv --reuid=archivebot --regid=archivebot --init-groups /home/archivebot/launch2.sh' >>/home/archivebot/launch.sh \ + && chmod +x /home/archivebot/launch.sh \ + && echo '#!/bin/bash' >/home/archivebot/launch2.sh \ + && echo 'export GEM_HOME=/home/archivebot/.gems' >>/home/archivebot/launch2.sh \ + && echo 'echo "Starting Redis"' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/redis && redis-server redis.conf) &' >>/home/archivebot/launch2.sh \ +# && echo '(cd /home/archivebot/ArchiveBot/bot && bundle exec ruby bot.rb -s "ircs://irc.hackint.org:6697" -r "redis://127.0.0.1:6379/0" -c "#notarchivebot" -n Puppeteer --db-credentials admin:areallygoodpassword) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot/bot && bundle exec ruby bot.rb -s "ircs://irc.hackint.org:6697" -r "redis://127.0.0.1:6379/0" -c "#notarchivebot" -n Puppeteer) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot && export REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 && plumbing/updates-listener | plumbing/log-firehose) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot && bundle exec ruby dashboard/app.rb -u http://0.0.0.0:4567) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot && FIREHOSE_SOCKET_URL=tcp://127.0.0.1:12345 plumbing/firehose-client | python3 dashboard/websocket.py) &' >>/home/archivebot/launch2.sh \ +# && echo '(cd /home/archivebot/ArchiveBot/cogs && bundle exec ruby start.rb --db-credentials admin:areallygoodpassword) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot/cogs && bundle exec ruby start.rb) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./analyzer) &' >>/home/archivebot/launch2.sh \ + && echo '(cd /home/archivebot/ArchiveBot/plumbing && REDIS_URL=redis://127.0.0.1:6379/0 UPDATES_CHANNEL=updates ./trimmer >/dev/null) &' >>/home/archivebot/launch2.sh \ + && echo 'sleep inf' >>/home/archivebot/launch2.sh \ + && chmod +x /home/archivebot/launch2.sh \ + && mkdir /home/archivebot/redis \ + && echo 'dir ./' >/home/archivebot/redis/redis.conf +USER root +RUN \ + # Set up CouchDB +# echo 'admin = areallygoodpassword' >>/opt/couchdb/etc/local.ini \ + /etc/init.d/couchdb start \ + && sleep 5 \ + && cat /opt/couchdb/var/log/couchdb.log \ +# && curl -sf http://admin:areallygoodpassword@127.0.0.1:5984/ \ +# && curl -sf -X PUT http://admin:areallygoodpassword@127.0.0.1:5984/_users \ +# && curl -sf -X PUT http://admin:areallygoodpassword@127.0.0.1:5984/_replicator \ +# && curl -sf -X PUT http://admin:areallygoodpassword@127.0.0.1:5984/archivebot \ +# && grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/ignore_patterns.json | curl -sf -X PUT --data @- http://admin:areallygoodpassword@127.0.0.1:5984/archivebot/_design/ignore_patterns \ +# && grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/user_agents.json | curl -sf -X PUT --data @- http://admin:areallygoodpassword@127.0.0.1:5984/archivebot/_design/user_agents + && curl -sf http://127.0.0.1:5984/ \ + && curl -sf -X PUT http://127.0.0.1:5984/_users \ + && curl -sf -X PUT http://127.0.0.1:5984/_replicator \ + && curl -sf -X PUT http://127.0.0.1:5984/archivebot \ + && grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/ignore_patterns.json | curl -sf -X PUT --data @- http://127.0.0.1:5984/archivebot/_design/ignore_patterns \ + && grep -v _rev /home/archivebot/ArchiveBot/db/design_docs/user_agents.json | curl -sf -X PUT --data @- http://127.0.0.1:5984/archivebot/_design/user_agents +RUN sed -i "s~\( start_listener\)~ Dir.foreach(@path) do |filename|\n next if filename == '.' or filename == '..'\n next if not filename.end_with? '.json'\n add_set("'"#{@path}/#{filename}"'")\n end\n\1~" /home/archivebot/ArchiveBot/cogs/couchdb_doc_updater.rb +EXPOSE 4567 +EXPOSE 4568 +ENTRYPOINT ["/home/archivebot/launch.sh"]