Merge pull request #470 from shtdbb/master

Update negative sampling in hn_mine to fix issue #464

Author: staoxiao

FlagEmbedding/baai_general_embedding/finetune/hn_mine.py

@@ -100,7 +100,9 @@ def find_knn_neg(model, input_file, candidate_pool, output_file, sample_range, n
     with open(output_file, 'w') as f:
         for data in train_data:
             if len(data['neg']) < negative_number:
-                data['neg'].extend(random.sample(corpus, negative_number - len(data['neg'])))
+                samples = random.sample(corpus, negative_number - len(data['neg']) + len(data['pos']))
+                samples = [sent for sent in samples if sent not in data['pos']]
+                data['neg'].extend(samples[: negative_number - len(data['neg'])])
             f.write(json.dumps(data, ensure_ascii=False) + '\n')