archiveteam-megawarc-factory/pack-one

#!/bin/bash
# Feeds the upload queue with megawarcs.
# (Needs a config.sh in the working directory.)
#
#   ./pack-one
#
# 1. Grabs an item from PACKING_QUEUE_DIR
# 2. Reserves the item by moving the directory to the
#    PACKER_WORKING_CHUNKS_DIR
# 3. Makes a megawarc in the PACKER_WORKING_MEGAWARC_DIR
# 4. Removes the source files from the working directory
# 5. Moves the megawarc to the UPLOAD_QUEUE_DIR
#
# The program exits with 1 on any nontransient error.
#

WORKING_DIR="$( pwd )"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
MEGAWARC="${SCRIPT_DIR}/megawarc/megawarc"

if [[ ! -x "${MEGAWARC}" ]]
then
  echo "${MEGAWARC} does not exist or is not executable."
  exit 1
fi

source ./config.sh || exit 1

mkdir -p "${PACKER_WORKING_CHUNKS_DIR}" || exit 1
mkdir -p "${PACKER_WORKING_MEGAWARC_DIR}" || exit 1
mkdir -p "${UPLOAD_QUEUE_DIR}" || exit 1


function mayicontinue {
  echo
# echo "May I continue?"
# read
# echo
}


# check if the upload queue is empty
# if [ "$( ls -A ${UPLOAD_QUEUE_DIR} )" ]
# then
#   echo "Upload queue not empty. Wait."
#   sleep 30
#   exit 0
# fi


mayicontinue


# try to grab a directory from the packing queue
ITEM=none
while [[ "${ITEM}" = none ]]
do
  possible_item=$( ls -1 "${PACKING_QUEUE_DIR}/" | grep -E '[0-9]{14}_[a-f0-9]{8}$' | sort | head -n 1 )
  if test -n "${possible_item}"
  then
    echo "Trying to grab ${possible_item}"
    if mv "${PACKING_QUEUE_DIR}/${possible_item}" "${PACKER_WORKING_CHUNKS_DIR}/"
    then
      ITEM="${possible_item}"
    else
      echo "Failed to move ${possible_item}"
      sleep 5
    fi
  else
    date
    echo "No current item found!"
    sleep 30
    exit 0
  fi
done


mayicontinue


echo "$( date ): Starting megawarc for item ${ITEM}" >> packer.log

# construct a megawarc
mkdir -p "${PACKER_WORKING_MEGAWARC_DIR}/${ITEM}"
# megawarcs use relative paths
cd "${PACKER_WORKING_CHUNKS_DIR}/"
"${MEGAWARC}" --verbose pack --server "${ZST_DICTIONARY_API}" "${PACKER_WORKING_MEGAWARC_DIR}/${ITEM}/${FILE_PREFIX}${ITEM}" "${ITEM}"
result="${?}"
cd "${WORKING_DIR}"

if [[ "${result}" -ne 0 ]]
then
  date
  echo "megawarc exited with ${result} for ${ITEM}"
  exit 1
fi

echo "$( date ): Completed megawarc for item ${ITEM}" >> packer.log


mayicontinue


# remove files
echo "megawarc OK, removing source files"
rm -rf "${PACKER_WORKING_CHUNKS_DIR}/${ITEM}"
result="${?}"

if [[ "${result}" -ne 0 ]]
then
  date
  echo "rm -rf source files exited with ${result} for ${ITEM}"
  exit 1
fi


echo "add to upload queue"
mv "${PACKER_WORKING_MEGAWARC_DIR}/${ITEM}" "${UPLOAD_QUEUE_DIR}/"


exit 0