From f5f1328ff78dba9b30d87de5ad8270be50e7bcf9 Mon Sep 17 00:00:00 2001 From: su Date: Sat, 18 Aug 2018 11:26:13 +0800 Subject: [PATCH 1/3] [feature]fix OOM of QAnet (#17) --- reader/qanet/layers.py | 14 +++++++++----- reader/qanet/rnn_reader.py | 4 ++-- reader/qanet/vector.py | 2 ++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/reader/qanet/layers.py b/reader/qanet/layers.py index 179f08f..49001dc 100644 --- a/reader/qanet/layers.py +++ b/reader/qanet/layers.py @@ -125,7 +125,8 @@ def forward(self,ori_x, x, x_mask): class CQattn(nn.Module): def __init__(self, dim): super().__init__() - self.linear = nn.Linear(dim*3, 1, bias=False) + self.linear1 = nn.Linear(dim, dim, bias=False) + self.linear2 = nn.Linear(dim, dim, bias=False) def forward(self, x1, x2, x1_mask, x2_mask): """ @@ -135,10 +136,13 @@ def forward(self, x1, x2, x1_mask, x2_mask): :param x2_mask: b x m """ # bxnxmxd - x1_aug = x1.unsqueeze(2).expand(x1.size(0), x1.size(1), x2.size(1), x1.size(2)) - x2_aug = x2.unsqueeze(1).expand(x1.size(0), x1.size(1), x2.size(1), x2.size(2)) - x_input = torch.cat([x1_aug, x2_aug, x1_aug * x2_aug], dim=3) - similarity = self.linear(x_input).squeeze(3) + #x1_aug = x1.unsqueeze(2).expand(x1.size(0), x1.size(1), x2.size(1), x1.size(2)) + #x2_aug = x2.unsqueeze(1).expand(x1.size(0), x1.size(1), x2.size(1), x2.size(2)) + #x_input = torch.cat([x1_aug, x2_aug, x1_aug * x2_aug], dim=3) + #similarity = self.linear(x_input).squeeze(3) + x1_aug = F.relu(self.linear1(x1)) + x2_aug = F.relu(self.linear2(x2)) + similarity = x1_aug.bmm(x2_aug.transpose(1, 2)) # bxnxm x2_mask = x2_mask.unsqueeze(1).expand_as(similarity) similarity.data.masked_fill_(x2_mask.data, -2e20) diff --git a/reader/qanet/rnn_reader.py b/reader/qanet/rnn_reader.py index a94e022..3ba7761 100644 --- a/reader/qanet/rnn_reader.py +++ b/reader/qanet/rnn_reader.py @@ -108,9 +108,9 @@ def pp(na,n): fusion = self.compose(fusion) M1 = self.modelenc1(x1, fusion, x1_mask) M2 = self.modelenc2(x1, M1, x1_mask) - M3 = self.modelenc3(x1, M2, x1_mask) + # M3 = self.modelenc3(x1, M2, x1_mask) starts = self.linears(torch.cat([M1,M2],dim=2)).squeeze(2) - ends = self.lineare(torch.cat([M1,M3],dim=2)).squeeze(2) + ends = self.lineare(torch.cat([M1,M2],dim=2)).squeeze(2) if self.training: start_scores = F.log_softmax(starts,dim=1) end_scores = F.log_softmax(ends, dim=1) diff --git a/reader/qanet/vector.py b/reader/qanet/vector.py index 35bc116..52facd2 100644 --- a/reader/qanet/vector.py +++ b/reader/qanet/vector.py @@ -97,6 +97,7 @@ def batchify(batch): questions_char = [ex[3] for ex in batch] # Batch documents and features max_length = max([d.size(0) for d in docs]) + #max_length = 400 x1 = torch.LongTensor(len(docs), max_length).zero_() x1_c = torch.LongTensor(len(docs), max_length, word_len).zero_() x1_mask = torch.ByteTensor(len(docs), max_length).fill_(1) @@ -107,6 +108,7 @@ def batchify(batch): # Batch questions max_length = max([q.size(0) for q in questions]) + #max_length = 20 x2 = torch.LongTensor(len(questions), max_length).zero_() x2_c = torch.LongTensor(len(questions), max_length, word_len).zero_() x2_mask = torch.ByteTensor(len(questions), max_length).fill_(1) From 5cf4e83725b1e23e46e6a10938bf12a5668b0f29 Mon Sep 17 00:00:00 2001 From: su Date: Sat, 18 Aug 2018 11:50:09 +0800 Subject: [PATCH 2/3] Dev (#19) * [feature]fix OOM of QAnet * [feature]move bash scripts into folder --- predict.sh => runs/predict.sh | 2 +- train_bidaf.sh => runs/train_bidaf.sh | 1 + train_bidaf_sogou.sh => runs/train_bidaf_sogou.sh | 1 + train_docqa.sh => runs/train_docqa.sh | 1 + train_docqa_sogou.sh => runs/train_docqa_sogou.sh | 1 + train_drqa.sh => runs/train_drqa.sh | 1 + train_drqa_sogou.sh => runs/train_drqa_sogou.sh | 2 +- train_fusionnet.sh => runs/train_fusionnet.sh | 1 + train_qanet.sh => runs/train_qanet.sh | 1 + train_rnet.sh => runs/train_rnet.sh | 1 + train_rnet_sogou.sh => runs/train_rnet_sogou.sh | 1 + train_slqa.sh => runs/train_slqa.sh | 1 + 12 files changed, 12 insertions(+), 2 deletions(-) rename predict.sh => runs/predict.sh (97%) rename train_bidaf.sh => runs/train_bidaf.sh (98%) rename train_bidaf_sogou.sh => runs/train_bidaf_sogou.sh (98%) rename train_docqa.sh => runs/train_docqa.sh (98%) rename train_docqa_sogou.sh => runs/train_docqa_sogou.sh (98%) rename train_drqa.sh => runs/train_drqa.sh (98%) rename train_drqa_sogou.sh => runs/train_drqa_sogou.sh (98%) rename train_fusionnet.sh => runs/train_fusionnet.sh (98%) rename train_qanet.sh => runs/train_qanet.sh (98%) rename train_rnet.sh => runs/train_rnet.sh (98%) rename train_rnet_sogou.sh => runs/train_rnet_sogou.sh (98%) rename train_slqa.sh => runs/train_slqa.sh (98%) diff --git a/predict.sh b/runs/predict.sh similarity index 97% rename from predict.sh rename to runs/predict.sh index 8717a84..0bc2404 100644 --- a/predict.sh +++ b/runs/predict.sh @@ -1,7 +1,7 @@ #!/bin/bash export PYTHONPATH=`pwd`:$PYTHONPATH - +cd `pwd` DATA=test python3 scripts/reader/converter.py data/sogou/${DATA}.json data/sogou/${DATA}_std_all.json test python3 scripts/reader/predict.py data/sogou/${DATA}_std_all.json --model data/models/docqa_sogou.mdl --tokenizer jieba --out-dir data/sogou diff --git a/train_bidaf.sh b/runs/train_bidaf.sh similarity index 98% rename from train_bidaf.sh rename to runs/train_bidaf.sh index 68745a1..e3d91b8 100644 --- a/train_bidaf.sh +++ b/runs/train_bidaf.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_bidaf.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name bidaf_cove diff --git a/train_bidaf_sogou.sh b/runs/train_bidaf_sogou.sh similarity index 98% rename from train_bidaf_sogou.sh rename to runs/train_bidaf_sogou.sh index 332819d..7653aac 100644 --- a/train_bidaf_sogou.sh +++ b/runs/train_bidaf_sogou.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/converter.py data/sogou/train.json data/sogou/train_std.json train #python3 scripts/reader/converter.py data/sogou/dev.json data/sogou/dev_std.json train # diff --git a/train_docqa.sh b/runs/train_docqa.sh similarity index 98% rename from train_docqa.sh rename to runs/train_docqa.sh index a69e716..1abae66 100644 --- a/train_docqa.sh +++ b/runs/train_docqa.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_docqa.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name doc_test_self diff --git a/train_docqa_sogou.sh b/runs/train_docqa_sogou.sh similarity index 98% rename from train_docqa_sogou.sh rename to runs/train_docqa_sogou.sh index f23a29d..16f14cf 100644 --- a/train_docqa_sogou.sh +++ b/runs/train_docqa_sogou.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/converter.py data/sogou/train.json data/sogou/train_std.json train #python3 scripts/reader/converter.py data/sogou/dev.json data/sogou/dev_std.json train # diff --git a/train_drqa.sh b/runs/train_drqa.sh similarity index 98% rename from train_drqa.sh rename to runs/train_drqa.sh index 1915aea..592d6e1 100644 --- a/train_drqa.sh +++ b/runs/train_drqa.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_drqa.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name drqa_drop diff --git a/train_drqa_sogou.sh b/runs/train_drqa_sogou.sh similarity index 98% rename from train_drqa_sogou.sh rename to runs/train_drqa_sogou.sh index 4464e2e..6c1a17e 100644 --- a/train_drqa_sogou.sh +++ b/runs/train_drqa_sogou.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH - +cd `pwd` #python3 scripts/reader/converter.py data/sogou/train.json data/sogou/train_std.json train #python3 scripts/reader/converter.py data/sogou/dev.json data/sogou/dev_std.json train # diff --git a/train_fusionnet.sh b/runs/train_fusionnet.sh similarity index 98% rename from train_fusionnet.sh rename to runs/train_fusionnet.sh index 315c6f9..7469f76 100644 --- a/train_fusionnet.sh +++ b/runs/train_fusionnet.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_fusionnet.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name debug_fusionnet_test diff --git a/train_qanet.sh b/runs/train_qanet.sh similarity index 98% rename from train_qanet.sh rename to runs/train_qanet.sh index 3389030..7e7ab71 100644 --- a/train_qanet.sh +++ b/runs/train_qanet.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_qanet.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name qanet_debug diff --git a/train_rnet.sh b/runs/train_rnet.sh similarity index 98% rename from train_rnet.sh rename to runs/train_rnet.sh index 8ee5d62..4e5c8cc 100644 --- a/train_rnet.sh +++ b/runs/train_rnet.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_rnet.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name rnet_std diff --git a/train_rnet_sogou.sh b/runs/train_rnet_sogou.sh similarity index 98% rename from train_rnet_sogou.sh rename to runs/train_rnet_sogou.sh index b2e2113..a084bd0 100644 --- a/train_rnet_sogou.sh +++ b/runs/train_rnet_sogou.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/converter.py data/sogou/train.json data/sogou/train_std.json train #python3 scripts/reader/converter.py data/sogou/dev.json data/sogou/dev_std.json train # diff --git a/train_slqa.sh b/runs/train_slqa.sh similarity index 98% rename from train_slqa.sh rename to runs/train_slqa.sh index 47fa891..d61ba1d 100644 --- a/train_slqa.sh +++ b/runs/train_slqa.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex export PYTHONPATH=`pwd`:$PYTHONPATH +cd `pwd` #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-train --tokenizer spacy #python3 scripts/reader/preprocess.py data/datasets/ data/datasets/ --split SQuAD-v1.1-dev --tokenizer spacy python3 scripts/reader/train_slqa.py --data-dir data/datasets --restrict-vocab 0 --train-file SQuAD-v1.1-train-processed-spacy.txt --dev-file SQuAD-v1.1-dev-processed-spacy.txt --dev-json SQuAD-v1.1-dev.json --model-dir data/models --model-name slqa1 From 3ac943e2cb68f42fc4b50c02876aca2320d136df Mon Sep 17 00:00:00 2001 From: sulixin Date: Sat, 18 Aug 2018 12:05:02 +0800 Subject: [PATCH 3/3] [feature]add simpleqa --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b191265..cecb1b2 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ DrQA | 68.39 | 77.90 QAnet | ... | ... SLQA | 67.09 | 76.67 FusionNet | 68.27 | 77.79 - +SimpleQA | ... | ... ## Current progress ### [Rnet](https://www.microsoft.com/en-us/research/wp-content/uploads/2017/05/r-net.pdf)