#!/bin/bash set -exuo pipefail # https://stackoverflow.com/a/2173421 trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT if [ -z ${COUCHDB_URL+x} ]; then echo "Skipping couchdb init" else timeout 300 bash -c "while [[ \"\$(curl -s -o /dev/null -w ''%{http_code}'' ${COUCHDB_URL})\" != \"200\" ]]; do sleep 5; done" || false if [ -z ${COUCHDB_USER+x} ]; then export COUCHDB_CURL_ARGS="" else export COUCHDB_CURL_ARGS="-u \"$COUCHDB_USER:$COUCHDB_PASSWORD\"" fi pushd "db/design_docs" curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/_users" curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot" curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot_logs" grep -v _rev archive_urls.json > /tmp/archive_urls.json grep -v _rev ignore_patterns.json > /tmp/ignore_patterns.json grep -v _rev jobs.json > /tmp/jobs.json grep -v _rev user_agents.json > /tmp/user_agents.json curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot/_design/archive_urls" -d @/tmp/archive_urls.json curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot/_design/ignore_patterns" -d @/tmp/ignore_patterns.json curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot/_design/jobs" -d @/tmp/jobs.json curl -s $COUCHDB_CURL_ARGS -X PUT "$COUCHDB_URL/archivebot/_design/user_agents" -d @/tmp/user_agents.json popd fi export SHARED_WARCS_DIR="${SHARED_WARCS_DIR:-/data/}" mkdir -pv "$SHARED_WARCS_DIR/upload-queue/" case "$1" in "bot") cd bot if [ -z ${COUCHDB_USER+x} ]; then export COUCHDB_ARGS="" else export COUCHDB_ARGS="--db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\"" fi bundle exec ruby bot.rb \ -s "$IRC_URL" \ -r "$REDIS_URL" \ -c "$IRC_CHANNEL" \ -n "$IRC_NICK" \ --db "$COUCHDB_URL/archivebot" $COUCHDB_ARGS ;; "cogs") if [ -z ${COUCHDB_USER+x} ]; then export COUCHDB_ARGS="" else export COUCHDB_ARGS="--db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\" --log-db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\"" fi bundle exec ruby cogs/start.rb \ -r "$REDIS_URL" \ --db "$COUCHDB_URL/archivebot" \ --log-db "$COUCHDB_URL/archivebot_logs" $COUCHDB_ARGS ;; "firehose") export UPDATES_CHANNEL=updates export FIREHOSE_SOCKET_URL=tcp://0.0.0.0:12345 plumbing/updates-listener | plumbing/log-firehose ;; "dashboard") bundle exec ruby dashboard/app.rb -u http://0.0.0.0:8080 -r "$REDIS_URL" ;; "websocket") plumbing/firehose-client | python3 dashboard/websocket.py ;; "pipeline") cd pipeline export PIPELINE_NAME="${PIPELINE_NAME:-${PIPELINE_PREFIX}-$(hostname -s)}" export PIPELINE_NAME="${PIPELINE_NAME:0:30}" export NO_SCREEN=1 export STAGING_WARCS_DIR="$(mktemp -p "$SHARED_WARCS_DIR" -d staging-XXXXXXXXXX)" export FINISHED_WARCS_DIR="/local-staging/" sudo /usr/sbin/tcp-closer -4 --dport 443 --idle_time 21601000 --last_recv_limit 43200000 --interval 300 & sudo /usr/sbin/tcp-closer -6 --dport 443 --idle_time 21601000 --last_recv_limit 43200000 --interval 300 & /stager.sh "$FINISHED_WARCS_DIR" "$STAGING_WARCS_DIR" "$SHARED_WARCS_DIR/upload-queue/" & run-pipeline3 pipeline.py --disable-web-server \ --concurrent $PIPELINE_CONCURRENT $PIPELINE_NAME /wait-empty.sh "$FINISHED_WARCS_DIR" /wait-empty.sh "$STAGING_WARCS_DIR" rmdir "$STAGING_WARCS_DIR" ;; "uploader") python -u ./uploader/uploader.py "$SHARED_WARCS_DIR/upload-queue/" ;; "analyzer") export UPDATES_CHANNEL=updates cd plumbing ./analyzer ;; "trimmer") export UPDATES_CHANNEL=updates cd plumbing ./trimmer > /dev/null ;; esac