- #!/bin/bash
- set -exuo pipefail
-
- if [ -z ${COUCHDB_URL+x} ]; then
- echo "Skipping couchdb init"
- else
- timeout 300 bash -c "while [[ \"\$(curl -s -o /dev/null -w ''%{http_code}'' ${COUCHDB_URL})\" != \"200\" ]]; do sleep 5; done" || false
-
- if [ -z ${COUCHDB_USER+x} ]; then
- export COUCHDB_CURL_ARGS=""
- else
- export COUCHDB_CURL_ARGS="-u \"$COUCHDB_USER:$COUCHDB_PASSWORD\""
- fi
-
- pushd "db/design_docs"
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/_users
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot_logs
- grep -v _rev archive_urls.json > /tmp/archive_urls.json
- grep -v _rev ignore_patterns.json > /tmp/ignore_patterns.json
- grep -v _rev jobs.json > /tmp/jobs.json
- grep -v _rev user_agents.json > /tmp/user_agents.json
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot/_design/archive_urls -d @/tmp/archive_urls.json
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot/_design/ignore_patterns -d @/tmp/ignore_patterns.json
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot/_design/jobs -d @/tmp/jobs.json
- curl -s $COUCHDB_CURL_ARGS -X PUT $COUCHDB_URL/archivebot/_design/user_agents -d @/tmp/user_agents.json
- popd
- fi
-
- export FINISHED_WARCS_DIR=${FINISHED_WARCS_DIR:-/data/}
-
- case "$1" in
- "bot")
- cd bot
- if [ -z ${COUCHDB_USER+x} ]; then
- export COUCHDB_ARGS=""
- else
- export COUCHDB_ARGS="--db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\""
- fi
- bundle exec ruby bot.rb \
- -s "$IRC_URL" \
- -r "$REDIS_URL" \
- -c "$IRC_CHANNEL" \
- -n "$IRC_NICK" \
- --db "$COUCHDB_URL/archivebot" $COUCHDB_ARGS
- ;;
- "cogs")
- if [ -z ${COUCHDB_USER+x} ]; then
- export COUCHDB_ARGS=""
- else
- export COUCHDB_ARGS="--db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\" --log-db-credentials \"$COUCHDB_USER:$COUCHDB_PASSWORD\""
- fi
- bundle exec ruby cogs/start.rb \
- -r "$REDIS_URL" \
- --db "$COUCHDB_URL/archivebot" \
- --log-db "$COUCHDB_URL/archivebot_logs" $COUCHDB_ARGS
- ;;
- "firehose")
- export UPDATES_CHANNEL=updates
- export FIREHOSE_SOCKET_URL=tcp://0.0.0.0:12345
- plumbing/updates-listener | plumbing/log-firehose
- ;;
- "dashboard")
- bundle exec ruby dashboard/app.rb -u http://0.0.0.0:8080 -r "$REDIS_URL"
- ;;
- "websocket")
- plumbing/firehose-client | python3 dashboard/websocket.py
- ;;
- "pipeline")
- cd pipeline
- export PIPELINE_NAME="${PIPELINE_NAME:-${PIPELINE_PREFIX}-$(hostname -s)}"
- export NO_SCREEN=1
- sudo /usr/sbin/tcp-closer -4 --dport 443 --idle_time 21601000 --last_recv_limit 43200000 --interval 300 &
- sudo /usr/sbin/tcp-closer -6 --dport 443 --idle_time 21601000 --last_recv_limit 43200000 --interval 300 &
- run-pipeline3 pipeline.py --disable-web-server \
- --concurrent $PIPELINE_CONCURRENT $PIPELINE_NAME
- ;;
- "uploader")
- python ./uploader/uploader.py $FINISHED_WARCS_DIR
- ;;
- "analyzer")
- export UPDATES_CHANNEL=updates
- cd plumbing
- ./analyzer
- ;;
- "trimmer")
- export UPDATES_CHANNEL=updates
- cd plumbing
- ./trimmer > /dev/null
- ;;
- esac
|