DV Elim Stats - Bash Scripts

[Ynot]

With round 2 of the elimination game set to start on Wednesday, I thought some people would be interested in the scripts I made for the stats. These are a lot more in-depth than the first round stats, and in particular, use concurrent processing to speed up execution. It may not be the most efficient of programs (in fact, I'm sure someone will spot bits that are highly inefficient), but I'm quite proud of it none-the-less The main program is called "elim_start" This program sets up some global stuff, forks off the parallel processing work for tracking individual users' scores, then does the top-x totals and pulls everything together. For obvious reasons, I've altered the login credentials of my FTP account elim_start Spoiler for elim_start: #!/bin/bash ## VARIABLES ## #THREAD_ID="68842" THREAD_ID="0" NUM_THREADS=10 NUM_TOP_VOTERS=20 NUM_TOP_PLAYERS=5 NUM_TOP_KILLERS=5 NUM_TOP_FAVOURS=10 NUM_TOP_GRUDGES=10 EMPTY_TOP_ENTRY="- - - - - - - - - - - - - - - -" export POSTS_FILE="/tmp/dv_posts" export VOTE_TABLE="/tmp/dv_votes" export VOTE_TABLE_DELIM="::" export KILLERS_TALLY="/tmp/dv_killers" export DEAD_TALLY="/tmp/dv_dead" ## VARIABLES END ## ## FUNCTIONS ## function ftp_upload { ftp -inv my_ftp_account<<ENDFTP user my_username my_password cd /htdocs lcd /home/tony/scripts/dv_elim_2 put dv_elim.html bye ENDFTP } ## FUNCTIONS END ## ## START ## if [ -f "$VOTE_TABLE" ]; then rm "$VOTE_TABLE" fi if [ -f "$POSTS_FILE" ]; then rm "$POSTS_FILE" fi if [ -f "$KILLERS_TALLY" ]; then rm "$KILLERS_TALLY" fi if [ -f "$DEAD_TALLY" ]; then rm "$DEAD_TALLY" fi touch "$KILLERS_TALLY" touch "$DEAD_TALLY" TOP_KILLERS="" ANCHOR_NAMES="" # Start Timer TIME_START=`date +%s` BREAK=0 PAGE=1 while [ $BREAK == 0 ]; do PAGE_HTML=`wget -q -O - "http://www.dreamviews.com/community/archive/index.php/t-${THREAD_ID}-p-${PAGE}.html"` HAS_POSTS=`echo "$PAGE_HTML" | grep '<div class="post">'` if [ -z "$HAS_POSTS" ]; then BREAK=1 else if [ -z "$HTML" ]; then HTML="$PAGE_HTML" else HTML=`echo "$HTML"; echo -e "\n\n"; echo "$PAGE_HTML"` fi PAGE=`expr $PAGE + 1` fi done UPDATED=`date -u` echo "$HTML" | tr -d '\r\n' | sed 's/<div class="post">/\n<div class="post">/g' | grep '<div class="post">' | grep -e '--' > "$POSTS_FILE" LOOP_UNAMES=`echo "${HTML}" | grep -o '<div class="username">[^<]*' | sed 's/<div class="username">//g' | tr -d '[]' | tr '_' ' '` UNIQ_UNAMES=`echo "$LOOP_UNAMES" | sort -u` IFS=$'\n' for UNIQ_UNAME_FILE in $UNIQ_UNAMES; do UNIQ_UNAME_FILE=`echo "$UNIQ_UNAME_FILE" | sed 's| |__SPACE__|g'` touch "/tmp/dv_uname_$UNIQ_UNAME_FILE" done IFS=$' \t\n' # Parallel loop of users !!! /home/tony/scripts/dv_elim_2/parallel -j $NUM_THREADS -r "/home/tony/scripts/dv_elim_2/user_loop *" /tmp/dv_uname_* IFS=$'\n' # Compile Top Lists VOTERS_ONLY=`cat "$VOTE_TABLE" | grep -v "0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0$" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2${VOTE_TABLE_DELIM}\3${VOTE_TABLE_DELIM}\4|"` # Top Voters echo "Compiling Top Voters" >&2 TOTAL_VOTES_TABLE="" for VOTER in $UNIQ_UNAMES; do TOTAL_VOTES=0 VOTER_LINES=`echo "$VOTERS_ONLY" | grep "^${VOTER}${VOTE_TABLE_DELIM}"` for VOTER_LINE in $VOTER_LINES; do LINE_UP=`echo "$VOTER_LINE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` LINE_DOWN=`echo "$VOTER_LINE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\3|"` LINE_TOTAL=`echo "$LINE_UP + $LINE_DOWN" | bc` TOTAL_VOTES=`echo "$TOTAL_VOTES + $LINE_TOTAL" | bc` done if [ "${TOTAL_VOTES}" -gt 0 ]; then TOTAL_VOTES_TABLE=`echo "$TOTAL_VOTES_TABLE"; echo "${TOTAL_VOTES}${VOTE_TABLE_DELIM}${VOTER}"` fi done TOTAL_VOTES_TABLE=`echo "$TOTAL_VOTES_TABLE" | grep -v '^$' | sort -nr | head -${NUM_TOP_VOTERS}` TOP_VOTER_HTML="" COUNTER=1 for TOTAL_VOTE in $TOTAL_VOTES_TABLE; do VOTER_VOTES=`echo "$TOTAL_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\1|"` VOTER_NAME=`echo "$TOTAL_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_03 | sed "s|&!LOOP_VOTERS_NUM!&|${COUNTER}|;s|&!TOP_VOTER!&|${VOTER_NAME}|;s|&!TOP_VOTER_VOTES!&|${VOTER_VOTES}|"` TOP_VOTER_HTML=`echo "$TOP_VOTER_HTML"; echo "$HTML_LINE"` COUNTER=`expr $COUNTER + 1` done TOP_VOTER_HTML_LEN=`echo "$TOP_VOTER_HTML" | grep -v '^$' | wc -l` if [ "$TOP_VOTER_HTML_LEN" -lt ${NUM_TOP_VOTERS} ]; then NUM_NEEDED=`echo "${NUM_TOP_VOTERS} - $TOP_VOTER_HTML_LEN" | bc` FILL_NUM=`echo "$TOP_VOTER_HTML_LEN + 1" | bc` FILL_LIMIT=`expr $NUM_TOP_VOTERS + 1` while [ "$FILL_NUM" -lt $FILL_LIMIT ]; do HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_03 | sed "s|&!LOOP_VOTERS_NUM!&|${FILL_NUM}|;s|&!TOP_VOTER!& (&!TOP_VOTER_VOTES!& votes)|${EMPTY_TOP_ENTRY}|"` TOP_VOTER_HTML=`echo "$TOP_VOTER_HTML"; echo "${HTML_LINE}"` FILL_NUM=`expr $FILL_NUM + 1` done fi # Top Friendships echo "Compiling Top Favours" >&2 TOP_FRIENDS=`cat "$VOTE_TABLE" | sed "s|\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\5${VOTE_TABLE_DELIM}\2${VOTE_TABLE_DELIM}\1|" | sort -nr | head -${NUM_TOP_FAVOURS}` TOP_FRIENDS_HTML="" COUNTER=1 for FRIENDSHIP in $TOP_FRIENDS; do FRIEND_OVERALL=`echo "$FRIENDSHIP" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\1|"` if [ "$FRIEND_OVERALL" -gt 0 ]; then FRIEND_VOTER=`echo "$FRIENDSHIP" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` FRIEND_SUBJ=`echo "$FRIENDSHIP" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\3|"` HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_07 | sed "s|&!LOOP_FRIENDS_NUM!&|${COUNTER}|;s|&!TOP_FRIEND!&|${FRIEND_VOTER}|;s|&!TOP_FRIEND_R!&|${FRIEND_SUBJ}|;s|&!TOP_FRIEND_NUM!&|${FRIEND_OVERALL}|"` TOP_FRIENDS_HTML=`echo "$TOP_FRIENDS_HTML"; echo "$HTML_LINE"` fi COUNTER=`expr $COUNTER + 1` done TOP_FRIENDS_HTML_LEN=`echo "$TOP_FRIENDS_HTML" | grep -v '^$' | wc -l` if [ "$TOP_FRIENDS_HTML_LEN" -lt ${NUM_TOP_FAVOURS} ]; then NUM_NEEDED=`echo "${NUM_TOP_FAVOURS} - $TOP_FRIENDS_HTML_LEN" | bc` FILL_NUM=`echo "$TOP_FRIENDS_HTML_LEN + 1" | bc` FILL_LIMIT=`expr $NUM_TOP_FAVOURS + 1` while [ "$FILL_NUM" -lt $FILL_LIMIT ]; do HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_07 | sed "s|&!LOOP_FRIENDS_NUM!&|${FILL_NUM}|;s|&!TOP_FRIEND!& has voted &!TOP_FRIEND_R!& up &!TOP_FRIEND_NUM!& times.|${EMPTY_TOP_ENTRY}|"` TOP_FRIENDS_HTML=`echo "$TOP_FRIENDS_HTML"; echo "${HTML_LINE}"` FILL_NUM=`expr $FILL_NUM + 1` done fi # Top Grudges echo "Compiling Top Grudges" >&2 TOP_GRUDGES=`cat "$VOTE_TABLE" | sed "s|\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\5${VOTE_TABLE_DELIM}\2${VOTE_TABLE_DELIM}\1|" | sort -n | head -${NUM_TOP_GRUDGES}` TOP_GRUDGES_HTML="" COUNTER=1 for GRUDGE in $TOP_GRUDGES; do GRUDGE_OVERALL=`echo "$GRUDGE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\1|" | sed 's|^-||'` if [ "$GRUDGE_OVERALL" -gt 0 ]; then GRUDGE_VOTER=`echo "$GRUDGE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` GRUDGE_SUBJ=`echo "$GRUDGE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\3|"` HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_09 | sed "s|&!LOOP_GRUDGES_NUM!&|${COUNTER}|;s|&!TOP_GRUDGE!&|${GRUDGE_VOTER}|;s|&!TOP_GRUDGE_R!&|${GRUDGE_SUBJ}|;s|&!TOP_GRUDGE_NUM!&|${GRUDGE_OVERALL}|"` TOP_GRUDGES_HTML=`echo "$TOP_GRUDGES_HTML"; echo "$HTML_LINE"` fi COUNTER=`expr $COUNTER + 1` done TOP_GRUDGES_HTML_LEN=`echo "$TOP_GRUDGES_HTML" | grep -v '^$' | wc -l` if [ "$TOP_GRUDGES_HTML_LEN" -lt ${NUM_TOP_GRUDGES} ]; then NUM_NEEDED=`echo "${NUM_TOP_GRUDGES} - $TOP_GRUDGES_HTML_LEN" | bc` FILL_NUM=`echo "$TOP_GRUDGES_HTML_LEN + 1" | bc` FILL_LIMIT=`expr $NUM_TOP_GRUDGES + 1` while [ "$FILL_NUM" -lt $FILL_LIMIT ]; do HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_09 | sed "s|&!LOOP_GRUDGES_NUM!&|${FILL_NUM}|;s|&!TOP_GRUDGE!& has voted &!TOP_GRUDGE_R!& down &!TOP_GRUDGE_NUM!& times.|${EMPTY_TOP_ENTRY}|"` TOP_GRUDGES_HTML=`echo "$TOP_GRUDGES_HTML"; echo "${HTML_LINE}"` FILL_NUM=`expr $FILL_NUM + 1` done fi # Top Killers echo "Compiling Top Killers" >&2 TOP_KILLERS=`cat "$KILLERS_TALLY" | sort -nr | head -${NUM_TOP_KILLERS}` TOP_KILLER_HTML="" COUNTER=1 for KILLER in $TOP_KILLERS; do NUM_KILLED=`echo "$KILLER" | sed "s|^\([0-9]\+\)${VOTE_TABLE_DELIM}\(.*\)$|\1|"` KILLER_NAME=`echo "$KILLER" | sed "s|^\([0-9]\+\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_05 | sed "s|&!LOOP_KILLERS_NUM!&|${COUNTER}|;s|&!TOP_KILLER!&|${KILLER_NAME}|;s|&!TOP_KILLER_NUM!&|${NUM_KILLED}|"` TOP_KILLER_HTML=`echo "$TOP_KILLER_HTML"; echo "$HTML_LINE"` COUNTER=`expr $COUNTER + 1` done TOP_KILLER_HTML_LEN=`echo "$TOP_KILLER_HTML" | grep -v '^$' | wc -l` if [ "$TOP_KILLER_HTML_LEN" -lt ${NUM_TOP_KILLERS} ]; then NUM_NEEDED=`echo "${NUM_TOP_KILLERS} - $TOP_KILLER_HTML_LEN" | bc` FILL_NUM=`echo "$TOP_KILLER_HTML_LEN + 1" | bc` FILL_LIMIT=`expr $NUM_TOP_KILLERS + 1` while [ "$FILL_NUM" -lt $FILL_LIMIT ]; do HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_05 | sed "s|&!LOOP_KILLERS_NUM!&|${FILL_NUM}|;s|&!TOP_KILLER!& has killed &!TOP_KILLER_NUM!& people.|${EMPTY_TOP_ENTRY}|"` TOP_KILLER_HTML=`echo "$TOP_KILLER_HTML"; echo "${HTML_LINE}"` FILL_NUM=`expr $FILL_NUM + 1` done fi # Top Players echo "Compiling Top Players" >&2 TOP_PLAYERS="" TOP_PLAYERS_HTML="" ALL_VOTED_PLAYERS=`cat "$VOTE_TABLE" | grep -v "0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0$" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\1|" | sort -u` for VOTED_PLAYER in $ALL_VOTED_PLAYERS; do IS_DEAD=`cat "$DEAD_TALLY" | grep "${VOTED_PLAYER}"` if [ -z "$IS_DEAD" ]; then ALL_PLAYER_VOTES=`cat "$VOTE_TABLE" | grep "^${VOTED_PLAYER}" | grep -v "0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0$"` TOTAL_VOTES=0 for PLAYER_VOTE in $ALL_PLAYER_VOTES; do LINE_UP=`echo "$PLAYER_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\3|"` LINE_DOWN=`echo "$PLAYER_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\4|"` TOTAL_LINE=`echo "$LINE_UP + $LINE_DOWN" | bc` TOTAL_VOTES=`echo "$TOTAL_VOTES + $TOTAL_LINE" | bc` done TOP_PLAYERS=`echo "$TOP_PLAYERS"; echo "${TOTAL_VOTES}${VOTE_TABLE_DELIM}${VOTED_PLAYER}"` fi done TOP_PLAYERS=`echo "$TOP_PLAYERS" | sort -nr | head -${NUM_TOP_PLAYERS}` COUNTER=1 for TOP_PLAYER in $TOP_PLAYERS; do TOP_PLAYER_VOTES=`echo "$TOP_PLAYER" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\1|"` TOP_PLAYER_NAME=`echo "$TOP_PLAYER" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` TOP_PLAYERS_HTML=`echo "$TOP_PLAYERS_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_16 | sed "s|&!LOOP_PLAYERS_NUM!&|${COUNTER}|;s|&!TOP_PLAYER!&|${TOP_PLAYER_NAME}|;s|&!TOP_PLAYER_VOTES!&|${TOP_PLAYER_VOTES}|"` COUNTER=`expr $COUNTER + 1` done TOP_PLAYERS_HTML_LEN=`echo "$TOP_PLAYERS_HTML" | grep -v '^$' | wc -l` if [ "$TOP_PLAYERS_HTML_LEN" -lt ${NUM_TOP_PLAYERS} ]; then NUM_NEEDED=`echo "${NUM_TOP_PLAYERS} - $TOP_PLAYERS_HTML_LEN" | bc` FILL_NUM=`echo "$TOP_PLAYERS_HTML_LEN + 1" | bc` FILL_LIMIT=`expr $NUM_TOP_PLAYERS + 1` while [ "$FILL_NUM" -lt $FILL_LIMIT ]; do HTML_LINE=`cat /home/tony/scripts/dv_elim_2/HTML_INC_16 | sed "s|&!LOOP_PLAYERS_NUM!&|${FILL_NUM}|;s|&!TOP_PLAYER!& (&!TOP_PLAYER_VOTES!& total votes).|${EMPTY_TOP_ENTRY}|"` TOP_PLAYERS_HTML=`echo "$TOP_PLAYERS_HTML"; echo "${HTML_LINE}"` FILL_NUM=`expr $FILL_NUM + 1` done fi # User Tables echo "Generating Per/User Tables" >&2 USER_TABLES_HTML="" ANCHOR_LIST="0 a b c d e f g h i j k l m n o p q r s t u v w x y z" LAST_USERNAME_FIRST="" COUNTER=0 for LOOP_USER in $UNIQ_UNAMES; do IS_IN_VOTE_TABLE=`cat "$VOTE_TABLE" | grep "^$LOOP_USER" | grep -v "0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0$"` if [ -n "$IS_IN_VOTE_TABLE" ]; then U_USERNAME_FIRST=`echo ${LOOP_USER:0:1} | tr '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' '0000000000abcdefghijklmnopqrstuvwxyz'` if [ "$U_USERNAME_FIRST" != "$LAST_USERNAME_FIRST" ]; then if [ -n "$ANCHOR_LIST" ]; then ANCHORS_TO_WRITE=`echo "$ANCHOR_LIST" | grep -o "^.*${U_USERNAME_FIRST}"` ANCHOR_LIST=`echo "$ANCHOR_LIST" | sed "s|^.*${U_USERNAME_FIRST} *||"` IFS=$' \t\n' for ANCHOR_WRITE in $ANCHORS_TO_WRITE; do USER_TABLES_HTML=`echo "$USER_TABLES_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_11 | sed "s|&!ANCHOR_NAME!&|${ANCHOR_WRITE}|"` done fi fi IFS=$'\n' U_PICKS=`cat "$VOTE_TABLE" | grep "^${LOOP_USER}" | sed "s|\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\5${VOTE_TABLE_DELIM}\2|" | sort -nr` if [ -n "$U_PICKS" ]; then BEST_NUM=`echo "$U_PICKS" | head -1 | grep -o "^.*${VOTE_TABLE_DELIM}" | sed "s|${VOTE_TABLE_DELIM}||"` WORST_NUM=`echo "$U_PICKS" | tail -1 | grep -o "^.*${VOTE_TABLE_DELIM}" | sed "s|${VOTE_TABLE_DELIM}||"` if [ "$BEST_NUM" -gt 0 ]; then U_FRIENDS=`echo "$U_PICKS" | grep "^${BEST_NUM}" | sed "s|.*${VOTE_TABLE_DELIM}\(.*\)|\1|" | tr '\n' ',' | sed 's|,$||' | sed 's|,|, |g'` else U_FRIENDS="N/A" fi if [ "$WORST_NUM" -lt 0 ]; then U_ENEMIES=`echo "$U_PICKS" | grep "^${WORST_NUM}" | sed "s|.*${VOTE_TABLE_DELIM}\(.*\)|\1|" | tr '\n' ',' | sed 's|,$||' | sed 's|,|, |g'` else U_ENEMIES="N/A" fi else U_FRIENDS="N/A" U_ENEMIES="N/A" fi USER_TABLES_HTML=`echo "$USER_TABLES_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_12 | sed "s|&!U_USERNAME!&|${LOOP_USER}|;s|&!U_FRIENDS!&|${U_FRIENDS}|;s|&!U_ENEMY!&|${U_ENEMIES}|;s|&!LOOP_1_NUM!&|${COUNTER}|g"` ALL_VOTES=`cat "$VOTE_TABLE" | grep "^${LOOP_USER}" | grep -v "0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0$"` for IND_VOTE in $ALL_VOTES; do VOTER_USERNAME=`echo "$IND_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\2|"` TOT_UP=`echo "$IND_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\3|"` TOT_DOWN=`echo "$IND_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\4|"` TOT_OVERALL=`echo "$IND_VOTE" | sed "s|^\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)${VOTE_TABLE_DELIM}\(.*\)$|\5|"` if [ "$TOT_OVERALL" -gt 0 ]; then TOT_OVERALL="Friend" elif [ "$TOT_OVERALL" -eq 0 ]; then TOT_OVERALL="Neutral" elif [ "$TOT_OVERALL" -lt 0 ]; then TOT_OVERALL="Foe" fi USER_TABLES_HTML=`echo "$USER_TABLES_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_13 | sed "s|&!V_USERNAME!&|${VOTER_USERNAME}|;s|&!V_UP!&|${TOT_UP}|;s|&!V_DOWN!&|${TOT_DOWN}|;s|&!V_OVERALL!&|${TOT_OVERALL}|"` done USER_TABLES_HTML=`echo "$USER_TABLES_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_14` LAST_USERNAME_FIRST="$U_USERNAME_FIRST" COUNTER=`expr $COUNTER + 1` fi done IFS=$' \t\n' for ANCHOR_WRITE in $ANCHOR_LIST; do USER_TABLES_HTML=`echo "$USER_TABLES_HTML"; cat /home/tony/scripts/dv_elim_2/HTML_INC_11 | sed "s|&!ANCHOR_NAME!&|${ANCHOR_WRITE}|"` done IFS=$'\n' # Stop Timer TIME_END=`date +%s` RUNTIME_S=`echo "$TIME_END - $TIME_START" | bc` RUNTIME_M="00" RUNTIME_H="00" while [ "$RUNTIME_S" -gt 59 ]; do RUNTIME_M=`expr $RUNTIME_S / 60` RUNTIME_SUB=`expr $RUNTIME_M \* 60` RUNTIME_S=`expr $RUNTIME_S - $RUNTIME_SUB` done while [ "$RUNTIME_M" -gt 59 ]; do RUNTIME_H=`expr $RUNTIME_M / 60` RUNTIME_SUB=`expr $RUNTIME_H \* 60` RUNTIME_M=`expr $RUNTIME_M - $RUNTIME_SUB` done if [ "$RUNTIME_S" -lt 10 ]; then RUNTIME_S="0${RUNTIME_S}" RUNTIME_S=`echo "$RUNTIME_S" | sed 's|^0\+|0|;s|^0$|00|'` fi if [ "$RUNTIME_M" -lt 10 ]; then RUNTIME_M="0${RUNTIME_M}" RUNTIME_M=`echo "$RUNTIME_M" | sed 's|^0\+|0|;s|^0$|00|'` fi if [ "$RUNTIME_H" -lt 10 ]; then RUNTIME_H="0${RUNTIME_H}" RUNTIME_H=`echo "$RUNTIME_H" | sed 's|^0\+|0|;s|^0$|00|'` fi RUNTIME_NICE="${RUNTIME_H}:${RUNTIME_M}:${RUNTIME_S}" # Put everything together. OUTPUT=`cat /home/tony/scripts/dv_elim_2/HTML_INC_01 | sed "s|&!UPDATED!&|${UPDATED}|;s|&!RUNTIME!&|${RUNTIME_NICE}|"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_02` OUTPUT=`echo "$OUTPUT"; echo "$TOP_VOTER_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_17` OUTPUT=`echo "$OUTPUT"; echo "$TOP_PLAYERS_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_04` OUTPUT=`echo "$OUTPUT"; echo "$TOP_KILLER_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_06` OUTPUT=`echo "$OUTPUT"; echo "$TOP_FRIENDS_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_08` OUTPUT=`echo "$OUTPUT"; echo "$TOP_GRUDGES_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_10` OUTPUT=`echo "$OUTPUT"; echo "$USER_TABLES_HTML"` OUTPUT=`echo "$OUTPUT"; cat /home/tony/scripts/dv_elim_2/HTML_INC_15` OUTPUT=`echo "$OUTPUT" | grep -v '^$'` echo "$OUTPUT" > /home/tony/scripts/dv_elim_2/dv_elim.html COUNTER=0 UPLOAD_CHECK="" while [ "${UPLOAD_CHECK}" != "${UPDATED}" ]; do if [ "$COUNTER" -gt 10 ]; then echo "Upload Error - Failed 10 times" >&2 exit 1 fi ftp_upload sleep 10 UPLOAD_CHECK=`wget -q -O - "http://www.snoopy.force9.co.uk/dv_elim.html" | grep -o '<p class="info">Last updated: [^<]*</p>' | sed 's|<p class="info">Last updated: \([^<]*\)</p>|\1|'` COUNTER=`expr $COUNTER + 1` done # # # echo "DONE" >&2 The parallel processing is governed by this wrapper script parallel Spoiler for parallel: #!/bin/bash NUM=0 QUEUE="" MAX_NPROC=2 # default REPLACE_CMD=0 # no replacement by default USAGE="A simple wrapper for running processes in parallel. Usage: `basename $0` [-h] [-r] [-j nb_jobs] command arg_list -h Shows this help -r Replace asterix * in the command string with argument -j nb_jobs Set number of simultanious jobs [2] Examples: `basename $0` somecommand arg1 arg2 arg3 `basename $0` -j 3 \"somecommand -r -p\" arg1 arg2 arg3 `basename $0` -j 6 -r \"convert -scale 50% * small/small_*\" *.jpg" function queue { QUEUE="$QUEUE $1" NUM=$(($NUM+1)) } function regeneratequeue { OLDREQUEUE=$QUEUE QUEUE="" NUM=0 for PID in $OLDREQUEUE do if [ -d /proc/$PID ] ; then QUEUE="$QUEUE $PID" NUM=$(($NUM+1)) fi done } function checkqueue { OLDCHQUEUE=$QUEUE for PID in $OLDCHQUEUE do if [ ! -d /proc/$PID ] ; then regeneratequeue # at least one PID has finished break fi done } # parse command line if [ $# -eq 0 ]; then # must be at least one arg echo "$USAGE" >&2 exit 1 fi while getopts j:rh OPT; do # "j:" waits for an argument "h" doesnt case $OPT in h) echo "$USAGE" exit 0 ;; j) MAX_NPROC=$OPTARG ;; r) REPLACE_CMD=1 ;; \?) # getopts issues an error message echo "$USAGE" >&2 exit 1 ;; esac done # Main program #echo Using $MAX_NPROC parallel threads shift `expr $OPTIND - 1` # shift input args, ignore processed args COMMAND=$1 shift for INS in $* # for the rest of the arguments do # DEFINE COMMAND if [ $REPLACE_CMD -eq 1 ]; then CMD=${COMMAND//"*"/$INS} else CMD="$COMMAND $INS" #append args fi #echo "Running $CMD" $CMD & # DEFINE COMMAND END PID=$! queue $PID while [ $NUM -ge $MAX_NPROC ]; do checkqueue sleep 0.4 done done wait # wait for all processes to finish before exit And finally, the per-user tracking script user_loop Spoiler for user_loop: #!/bin/bash VOTE_TABLE_SPLIT="/tmp/dv_votes.$$" POSTS=`cat "$POSTS_FILE"` function clean_up { if [ -f "$VOTE_TABLE_SPLIT" ]; then rm "$VOTE_TABLE_SPLIT" fi echo "PID $$ ending. Stop tracking $U_USERNAME" >&2 } function commit_stuff { if [ -f "$VOTE_TABLE_SPLIT" ]; then for VOTE_TABLE_SPLIT_LINE in `cat "$VOTE_TABLE_SPLIT"`; do echo "$VOTE_TABLE_SPLIT_LINE" >> "$VOTE_TABLE" done fi clean_up } function update_table { U_USER="$1" V_USER="$2" CH_UP="$3" CH_DOWN="$4" if [ ! -f "$VOTE_TABLE_SPLIT" ]; then touch "$VOTE_TABLE_SPLIT" fi TARGET_LINE=`cat "$VOTE_TABLE_SPLIT" | grep "^${U_USER}${VOTE_TABLE_DELIM}${V_USER}"` if [ -z "$TARGET_LINE" ]; then TARGET_LINE="${U_USER}${VOTE_TABLE_DELIM}${V_USER}${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0${VOTE_TABLE_DELIM}0" echo "$TARGET_LINE" >> "$VOTE_TABLE_SPLIT" fi OLD_UP=`echo "$TARGET_LINE" | grep -o "${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}.*$" | grep -o "^${VOTE_TABLE_DELIM}[0-9]\+" | grep -o '[0-9]\+$'` OLD_DOWN=`echo "$TARGET_LINE" | grep -o "${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}.*$" | grep -o "^${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}[0-9]\+" | grep -o '[0-9]\+$'` if [ "$CH_UP" -gt 0 ]; then NEW_UP=`echo "$OLD_UP + $CH_UP" | bc` else NEW_UP="$OLD_UP" fi if [ "$CH_DOWN" -gt 0 ]; then NEW_DOWN=`echo "$OLD_DOWN + $CH_DOWN" | bc` else NEW_DOWN="$OLD_DOWN" fi DIG_OVERALL=`echo "$NEW_UP - $NEW_DOWN" | bc` sed -i "s|${U_USER}${VOTE_TABLE_DELIM}${V_USER}${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}[0-9]\+${VOTE_TABLE_DELIM}[0-9]\+|${U_USER}${VOTE_TABLE_DELIM}${V_USER}${VOTE_TABLE_DELIM}${NEW_UP}${VOTE_TABLE_DELIM}${NEW_DOWN}${VOTE_TABLE_DELIM}${DIG_OVERALL}|" "$VOTE_TABLE_SPLIT" } # BEGIN USER LOOP U_USERNAME=`basename "$1" | sed 's|^dv_uname_||;s|__SPACE__| |g'` rm "$1" IFS=$'\n' U_USERNAME=`echo "$U_USERNAME" | sed 's|&#[0-9]\+;||g;s|\[||g;s|\]||g'` echo "PID $$ - Tracking $U_USERNAME" >&2 LAST_VALUE="" # BEGIN POST LOOP # Loop through posts, tracking user value for POST in $POSTS; do V_USERNAME=`echo "$POST" | grep -o '<div class="username">[^<]*' | sed 's/<div class="username">//' | tr -d '[]' | tr '_' ' '` if [ "$V_USERNAME" != "$U_USERNAME" ]; then ## VOTE MUNGE ## ## EDIT TO SUIT ## CURRENT_VALUE=`echo "$POST" | sed 's/<br \/>/\n/g' | grep -i "$U_USERNAME" | grep -v '<div class="post">' | grep -o -e '--[ ]*[0-9][0-9]*' | sed 's/^--//g' | sed 's/^/=/g' | tr -d '\n' | grep -o '=[0-9][0-9]*$' | sed 's/=//'` if [ -n "$CURRENT_VALUE" ]; then # IS PARENT STILL RUNNING? # if not, exit PARENT_RUNNING=`ps -e h | grep -o '^[ ]*[0-9]\+' | sed 's|^ *||' | grep "$PPID"` if [ -z "$PARENT_RUNNING" ]; then clean_up exit 1 fi if [ -n "$LAST_VALUE" ]; then VALUE_CHANGE=`expr "$CURRENT_VALUE" - "$LAST_VALUE"` else VALUE_CHANGE=0 fi if [ "$VALUE_CHANGE" -gt 0 ]; then UP_CHANGE=`echo "$VALUE_CHANGE" | grep -o '[0-9]\+$'` DOWN_CHANGE=0 fi if [ "$VALUE_CHANGE" -lt 0 ]; then UP_CHANGE=0 DOWN_CHANGE=`echo "$VALUE_CHANGE" | grep -o '[0-9]\+$'` fi if [ "$VALUE_CHANGE" -eq 0 ]; then UP_CHANGE=0 DOWN_CHANGE=0 fi update_table "$U_USERNAME" "$V_USERNAME" "$UP_CHANGE" "$DOWN_CHANGE" ## TOP KILLERS RUNNING CALCS if [ "$CURRENT_VALUE" -eq 0 ]; then # USER KILLED !!! echo "$U_USERNAME killed by $V_USERNAME" >&2 # is V_USERNAME in TOP_KILLERS? IN_TOP_KILLERS=`cat "$KILLERS_TALLY" | grep "$V_USERNAME"` if [ -z "$IN_TOP_KILLERS" ]; then echo "1${VOTE_TABLE_DELIM}${V_USERNAME}" >> "$KILLERS_TALLY" else KILL_VAL=`echo "$IN_TOP_KILLERS" | grep -o '^[0-9]\+'` KILL_VAL=`expr $KILL_VAL + 1` sed -i "s|^[0-9]\+${VOTE_TABLE_DELIM}${V_USERNAME}|${KILL_VAL}${VOTE_TABLE_DELIM}${V_USERNAME}|" "$KILLERS_TALLY" fi echo "$U_USERNAME" >> "$DEAD_TALLY" commit_stuff exit 0 fi LAST_VALUE="$CURRENT_VALUE" fi fi done # END OF POST LOOP commit_stuff # END OF USER LOOP *edit* The finished product can be seen here (obviously, it's blank until the game actually starts) http://www.snoopy.force9.co.uk/dv_elim.html