Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

keyword_vectors empty case added #5

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

keyword_vectors empty case added #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4132a50
keyword_vectors empty case added
Ai-Marshal Jun 12, 2022
add9875
Update lbl2vec.py
Ai-Marshal Jun 12, 2022
a872381
empty keyword_vectors case
Ai-Marshal Jun 12, 2022
4e8f64c
empty keywordword_vectors case
Ai-Marshal Jun 12, 2022
8be9479
IndexError: list index out of range
Ai-Marshal Jul 21, 2022
c25340a
test
Ai-Marshal Jul 21, 2022
95ab7e6
removed print statement
Ai-Marshal Jul 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added .DS_Store
View file
Open in desktop
Binary file not shown.
17 changes: 14 additions & 3 deletions lbl2vec/lbl2vec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,10 +431,12 @@ def predict_new_docs(
# ToDo: set swifter.progress_bar(self.verbose) again instead of swifter.progress_bar(False) when swifter resolved the TQDM progress bar issues
# get document vectors of new documents
if multiprocessing:

labeled_docs['doc_vec'] = labeled_docs['doc_word_tokens'].swifter.progress_bar(
False).apply(lambda row: self.doc2vec_model.infer_vector(doc_words=row))

else:

labeled_docs['doc_vec'] = labeled_docs['doc_word_tokens'].apply(
lambda row: self.doc2vec_model.infer_vector(doc_words=row))

Expand Down Expand Up @@ -623,16 +625,25 @@ def _get_similar_documents(
# the list
keywordword_vectors = [doc2vec_model.wv[word]
for word in cleaned_keywords_list]
similar_docs = doc2vec_model.dv.most_similar(
positive=keywordword_vectors, topn=num_docs)
if keywordword_vectors != []:
similar_docs = doc2vec_model.dv.most_similar(
positive=keywordword_vectors, topn=num_docs)
else:
return_docs_keys = [0] * num_docs
return_docs_similarity_scores = [0] * num_docs
logger.warning("No keywords found in this document")
return pd.Series([return_docs_keys, return_docs_similarity_scores], index=[
'doc_keys', 'doc_similarity_scores'])

except KeyError as error:
error.args = (
error.args[0] + " in trained Doc2Vec model. Either replace the keyword from the 'keywords_list' parameter or train a new Doc2Vec model that knows the keyword.",) + error.args[1:]
raise


doc_keys = [docs[0] for docs in similar_docs]
doc_scores = [docs[1] for docs in similar_docs]

# print(doc_scores, min_num_docs)
# add number of min_num_docs documents if too few documents are
# chosen by simiarilty threshold alone
if min_num_docs is not None and doc_scores[min_num_docs] < similarity_threshold and len(
Expand Down