archiveteam-megawarc-factory/pack-one

#!/bin/bash
# Feeds the upload queue with megawarcs.
# (Needs a config.sh in the working directory.)
#
#   ./pack-one
#
# 1. Grabs an item from PACKING_QUEUE_DIR
# 2. Reserves the item by moving the directory to the
#    PACKER_WORKING_CHUNKS_DIR
# 3. Makes a megawarc in the PACKER_WORKING_MEGAWARC_DIR
# 4. Removes the source files from the working directory
# 5. Moves the megawarc to the UPLOAD_QUEUE_DIR
#
# The program exits with 1 on any nontransient error.
#

WORKING_DIR="$( pwd )"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
MEGAWARC=$SCRIPT_DIR/megawarc/megawarc

if [[ ! -x $MEGAWARC ]]
then
  echo "$MEGAWARC does not exist or is not executable."
  exit 1
fi

source ./config.sh || exit 1

mkdir -p "$PACKER_WORKING_CHUNKS_DIR" || exit 1
mkdir -p "$PACKER_WORKING_MEGAWARC_DIR" || exit 1
mkdir -p "$UPLOAD_QUEUE_DIR" || exit 1


function mayicontinue {
  echo
# echo "May I continue?"
# read
# echo
}


# check if the upload queue is empty
# if [ "$( ls -A $UPLOAD_QUEUE_DIR )" ]
# then
#   echo "Upload queue not empty. Wait."
#   sleep 30
#   exit 0
# fi


mayicontinue


# try to grab a directory from the packing queue
ITEM=none
while [[ $ITEM = none ]]
do
  possible_item=$( ls -1 "$PACKING_QUEUE_DIR/" | grep -E '[0-9]{14}_[a-f0-9]{8}$' | sort | head -n 1 )
  if test -n "${possible_item}"
  then
    echo "Trying to grab $possible_item"
    if mv "$PACKING_QUEUE_DIR/$possible_item" "$PACKER_WORKING_CHUNKS_DIR/"
    then
      ITEM=$possible_item
    else
      echo "Failed to move $possible_item"
      sleep 5
    fi
  else
    date
    echo "No current item found!"
    sleep 30
    exit 0
  fi
done


mayicontinue


echo "$( date ): Starting megawarc for item $ITEM" >> packer.log

# construct a megawarc
mkdir -p $PACKER_WORKING_MEGAWARC_DIR/$ITEM
# megawarcs use relative paths
cd "$PACKER_WORKING_CHUNKS_DIR/"
$MEGAWARC --verbose pack --server $ZST_DICTIONARY_API $PACKER_WORKING_MEGAWARC_DIR/$ITEM/${FILE_PREFIX}${ITEM} $ITEM
result=$?
cd "$WORKING_DIR"

if [[ $result -ne 0 ]]
then
  date
  echo "megawarc exited with $result for $ITEM"
  exit 1
fi

echo "$( date ): Completed megawarc for item $ITEM" >> packer.log


mayicontinue


# remove files
echo "megawarc OK, removing source files"
rm -rf "$PACKER_WORKING_CHUNKS_DIR/$ITEM"
result=$?

if [[ $result -ne 0 ]]
then
  date
  echo "rm -rf source files exited with $result for $ITEM"
  exit 1
fi


echo "add to upload queue"
mv "$PACKER_WORKING_MEGAWARC_DIR/$ITEM" "$UPLOAD_QUEUE_DIR/"


exit 0