#!/bin/bash
set -e
export PYTHONUNBUFFERED=yes     # make sure the python programs don't buffer

# Only tesla-M is supported at the moment.
# Only English is currently supported.

# stdin format: 
# line-number candidate translation

# The text is assumed to be tokenized and lower-cased.

function usage()
{
    echo Usage: `basename $0` 'number-of-references reference_txt < candidate > score' >&2
    echo >&2
    exit 1
}

D=`dirname $0`
P=$$
LANGUAGE=en
TOOLS=$D/3rd-party
OPENNLP=${TOOLS}/opennlp-tools-1.4.3

export CLASSPATH=${OPENNLP}/output/opennlp-tools-1.4.3.jar:${OPENNLP}/lib/trove.jar:${OPENNLP}/lib/maxent-2.5.2.jar

NUMBER_REF=$1
REF_TXT=$2

[ -z "$NUMBER_REF" -o -z "$REF_TXT" ] && usage

echo Number of references per sentence: $NUMBER_REF >&2
echo Reference text: $REF_TXT >&2

echo Removing non-alphanum tokens from the reference.. >&2
$D/scripts/inline-python "
def keep_tok(tok):
    tok = tok.split('_')[0]
    if not any([c.isalnum() for c in tok]):
        return False
    # ok, at least one alnum
    return tok.upper() not in ['-LRB-', '-RRB-', '-LSB-', '-RSB-', '-LCB-', '-RCB-']

for line in open('$REF_TXT'):
    str = ' '.join([tok for tok in line.split() if keep_tok(tok)])
    if len(str) == 0:
        str = '.'       # our pipeline doesn't like completely empty lines.
    print str
" > temp$P.ref.alphanum

echo POS tagging the reference.. >&2
$D/scripts/pos_tagger.sh $LANGUAGE temp$P.ref.alphanum temp$P.ref.pos

echo Lemmatizing the reference.. >&2
$D/scripts/lemmatize.sh $LANGUAGE temp$P.ref.pos temp$P.ref.lem

# 20-bmng-wnms-mrp-func_x0.1
echo Generating reference BMNG.. >&2
cat temp$P.ref.lem |\
$D/scripts/count_monotext_ngrams.py |\
$D/scripts/discount_function.py -d 0.1 > temp$P.ref.bmng

# 20-bmng-pos-mrp-func_x0.1
echo Generating reference POS-BMNG.. >&2
cat temp$P.ref.lem |\
$D/scripts/keep_nth.py 1 |\
$D/scripts/count_monotext_ngrams.py |\
$D/scripts/discount_function.py -p0 -d 0.1 > temp$P.ref.pos-bmng

##### reference processing done. Now we process the stdin line by line #####

# Remove non-alphanum tokens from candidate
mkfifo temp$P.fifo.alphanum.in
mkfifo temp$P.fifo.alphanum.out

$D/scripts/inline-python "
import sys

def keep_tok(tok):
    tok = tok.split('_')[0]
    if not any([c.isalnum() for c in tok]):
        return False
    # ok, at least one alnum
    return tok.upper() not in ['-LRB-', '-RRB-', '-LSB-', '-RSB-', '-LCB-', '-RCB-']

fin = open('temp$P.fifo.alphanum.in')
fout = open('temp$P.fifo.alphanum.out', 'w')

while True:
    line = fin.readline()
    if line == '':
        break

    str = ' '.join([tok for tok in line.split() if keep_tok(tok)])
    if len(str) == 0:
        str = '.'       # our pipeline doesn't like completely empty lines.
    print >> fout, str
    fout.flush()
" &

# POS tag the candidate
mkfifo temp$P.fifo.pos_tagger.in
mkfifo temp$P.fifo.pos_tagger.out

cat temp$P.fifo.pos_tagger.in |\
java opennlp.tools.lang.english.PosTagger \
    -d $OPENNLP/models/tagdict \
    $OPENNLP/models/tag.bin.gz |\
$D/scripts/pos_format.py opennlp \
> temp$P.fifo.pos_tagger.out &

# Lemmatizing the candidate
mkfifo temp$P.fifo.lem.in
mkfifo temp$P.fifo.lem.out
$D/scripts/lemmatize.sh en temp$P.fifo.lem.in temp$P.fifo.lem.out &

# 20-bmng-wnms-mrp-func_x0.1
mkfifo temp$P.fifo.bmng.in
mkfifo temp$P.fifo.bmng.out
cat temp$P.fifo.bmng.in |\
$D/scripts/count_monotext_ngrams.py |\
$D/scripts/discount_function.py -d 0.1 > temp$P.fifo.bmng.out &

# 20-bmng-wnms-mrp-func_x0.1
# matching BMNGs 
mkfifo temp$P.fifo.bmng-scored.in
mkfifo temp$P.fifo.bmng-scored.lineids
mkfifo temp$P.fifo.bmng-scored.out
# for every line of input, we generate NUMBER_REF lines here
$D/scripts/pos_ngrams_mrp_wordnet.py \
    --streaming --refcount=$NUMBER_REF --lineids=temp$P.fifo.bmng-scored.lineids \
    --format=word_pos_stem --separator=_ \
    --simmethod=maxsim temp$P.ref.bmng temp$P.fifo.bmng-scored.in temp$P.fifo.bmng-scored.out &


# 20-bmng-pos-mrp-func_x0.1
mkfifo temp$P.fifo.bmng-pos.in
mkfifo temp$P.fifo.bmng-pos.out
cat temp$P.fifo.bmng-pos.in |\
$D/scripts/keep_nth.py 1 |\
$D/scripts/count_monotext_ngrams.py |\
$D/scripts/discount_function.py -p0 -d 0.1 > temp$P.fifo.bmng-pos.out &


# scoring BMNGs
mkfifo temp$P.fifo.bmng-mrp.in
mkfifo temp$P.fifo.bmng-mrp.out
$D/scripts/inline-python "
import sys
inf = open('temp$P.fifo.bmng-mrp.in')
while True:
    line = inf.readline()
    if line == '':
        break

    m1, r1, p1, m2, r2, p2, m3, r3, p3, m4, r4, p4 = map(float, line.split())
    for (m, r, p) in [(m1, r1, p1), (m2, r2, p2), (m3, r3, p3)]:
        try:
            precision = m / p
            recall = m / r
            f = precision * recall / (0.8 * precision + 0.2 * recall)
        except ZeroDivisionError:
            f = 0.0
        print f,
    print
" > temp$P.fifo.bmng-mrp.out &


# 20-bmng-pos-mrp-func_x0.1
# matching BMNGs 
mkfifo temp$P.fifo.bmng-pos-scored.in
mkfifo temp$P.fifo.bmng-pos-scored.lineids
mkfifo temp$P.fifo.bmng-pos-scored.out
# for every line of input, we generate NUMBER_REF lines here
$D/scripts/pos_ngrams_mrp_wordnet.py \
    --streaming --refcount=$NUMBER_REF --lineids=temp$P.fifo.bmng-pos-scored.lineids \
    --format=word --separator=_ \
    --simmethod=surface temp$P.ref.pos-bmng temp$P.fifo.bmng-pos-scored.in temp$P.fifo.bmng-pos-scored.out &
 

# scoring BMNGs
mkfifo temp$P.fifo.bmng-pos-mrp.in
mkfifo temp$P.fifo.bmng-pos-mrp.out
$D/scripts/inline-python "
import sys
inf = open('temp$P.fifo.bmng-pos-mrp.in')
while True:
    line = inf.readline()
    if line == '':
        break

    m1, r1, p1, m2, r2, p2, m3, r3, p3, m4, r4, p4 = map(float, line.split())
    for (m, r, p) in [(m1, r1, p1), (m2, r2, p2), (m3, r3, p3)]:
        try:
            precision = m / p
            recall = m / r
            f = precision * recall / (0.8 * precision + 0.2 * recall)
        except ZeroDivisionError:
            f = 0.0
        print f,
    print
" > temp$P.fifo.bmng-pos-mrp.out &


# tesla-M
# 1 line of output for every NUMBER_REF lines of input
mkfifo temp$P.fifo.tesla-m.in1
mkfifo temp$P.fifo.tesla-m.in2
mkfifo temp$P.fifo.tesla-m.out
$D/scripts/inline-python "
import sys
lines = []

file1 = open('temp$P.fifo.tesla-m.in1')
file2 = open('temp$P.fifo.tesla-m.in2')

def process_lines():
    global lines
    if len(lines) == 0:
        return
    assert len(lines) == 2 * $NUMBER_REF
    scores = map(float, ' '.join(lines).split())
    print sum(scores) / len(scores)
    sys.stdout.flush()
    lines = []

while True:
    line1 = file1.readline()
    line2 = file2.readline()
    if line1 == '' or line2 == '':
        break
    lines.append(line1)
    lines.append(line2)
    if len(lines) >= 2 * $NUMBER_REF:
        process_lines()

process_lines()
" > temp$P.fifo.tesla-m.out &


# driver python
echo "Launching drivers." >&2
$D/scripts/pipe_driver.py temp$P.fifo.alphanum.out \
temp$P.fifo.pos_tagger.in &

$D/scripts/pipe_driver.py temp$P.fifo.pos_tagger.out \
temp$P.fifo.lem.in &

$D/scripts/pipe_driver.py temp$P.fifo.lem.out \
temp$P.fifo.bmng.in temp$P.fifo.bmng-pos.in &

$D/scripts/pipe_driver.py temp$P.fifo.bmng.out \
temp$P.fifo.bmng-scored.in &
 
$D/scripts/pipe_driver.py temp$P.fifo.bmng-scored.out \
temp$P.fifo.bmng-mrp.in &

$D/scripts/pipe_driver.py temp$P.fifo.bmng-mrp.out \
temp$P.fifo.tesla-m.in1 &

$D/scripts/pipe_driver.py temp$P.fifo.bmng-pos.out \
temp$P.fifo.bmng-pos-scored.in &
 
$D/scripts/pipe_driver.py temp$P.fifo.bmng-pos-scored.out \
temp$P.fifo.bmng-pos-mrp.in &

$D/scripts/pipe_driver.py temp$P.fifo.bmng-pos-mrp.out \
temp$P.fifo.tesla-m.in2 &

python -c "
import sys

source = open('temp$P.fifo.alphanum.in', 'w')
idsource = open('temp$P.fifo.bmng-scored.lineids', 'w')
idsource2 = open('temp$P.fifo.bmng-pos-scored.lineids', 'w')
sink = open('temp$P.fifo.tesla-m.out')

print >> sys.stderr, 'Entering main driver loop..'

while True:
    line = sys.stdin.readline()
    if line == '':
        break

    # print >> sys.stderr, 'Processing line: ', line

    lineid, sent = line.split(' ', 1)
    print >> source, sent.strip()
    source.flush()
    print >> idsource, int(lineid)
    idsource.flush()
    print >> idsource2, int(lineid)
    idsource2.flush()

    line = sink.readline().strip()
    print 'tesla-m=', line
    sys.stdout.flush()
"

wait

rm -f temp$P.*
