Add forgotten unicode punctuation normalization to get_ende_bleu.

PiperOrigin-RevId: 191758943
tensorflow · Apr 5, 2018 · fc9335c · fc9335c
1 parent bca81be
commit fc9335c
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/tensor2tensor/utils/get_ende_bleu.sh b/tensor2tensor/utils/get_ende_bleu.sh
@@ -5,8 +5,11 @@ tok_gold_targets=newstest2013.tok.de
 
 decodes_file=$1
 
+# Replace unicode.
+perl $mosesdecoder/scripts/tokenizer/replace-unicode-punctuation.perl -l de  < $decodes_file > $decodes_file.n
+
 # Tokenize.
-perl $mosesdecoder/scripts/tokenizer/tokenizer.perl -l de < $decodes_file > $decodes_file.tok
+perl $mosesdecoder/scripts/tokenizer/tokenizer.perl -l de < $decodes_file.n > $decodes_file.tok
 
 # Put compounds in ATAT format (comparable to papers like GNMT, ConvS2S).
 # See https://nlp.stanford.edu/projects/nmt/ :