Add docstrings and cleanup

Author: OXPHOS

.gitignore

@@ -5,3 +5,7 @@ src/preprocessing/.ipynb_checkpoints/*
 .idea/*
 src/__pycache__/*
 *.npy
+Combine_result/*
+tmp_output/*
+ExampleCode.Python.zip
+exps

src/generate_vsm.py

@@ -14,18 +14,33 @@
 
 
 def reduce_dimension(vectors):
+    """
+    Perform PCA transformation on given vectors, n_components=0.95
+    
+    :param vectors: the matrix to be transformed
+    :return: the transformed matrix
+    """
     pca = PCA(n_components=0.95)
     return pca.fit_transform(vectors)
 
 
-def averaging_vectors(namelist, default_value=True, fill_na=False):
+def averaging_vectors(namelist, default_value=True):
+    """
+    Merge a list of word vectors into a single vector by averaging
+    
+    :param namelist: The list of word vectors
+    :param default_value: whether to replace OOV word with unknown
+    :return: averaged vectors
+    """
     if default_value:
         unk = w2v_model.get_vector('unknown')
         vectors = np.array([unk for _ in range(len(namelist))])
     else:
         vectors = np.array([[np.nan] * vector_size for _ in range(len(namelist))])
+
     i = 0
     for e in namelist:
+        # Remove stop words
         name = ''.join(filter(whitelist.__contains__, e.replace('-', ' ')))
         vec = np.zeros(vector_size)
         count = 0
@@ -44,10 +59,18 @@ def averaging_vectors(namelist, default_value=True, fill_na=False):
 
 
 def fixed_length_vectors(namelist, embedding_size=entity_embedding_size):
+    """
+    Turn the input word vector groups into vector with fixed length: embedding_size
+    
+    :param namelist: The list of word vectors
+    :param embedding_size: the length of the embedding
+    :return: embedded vectors
+    """
     vectors = np.zeros((len(namelist), embedding_size, vector_size))
-    i = 0
+    i = 0  # Tracks the number of word vector group
     for e in namelist:
         name = ''.join(filter(whitelist.__contains__, e.replace('-', ' ')))
+        # If the input word is too long, averaging the neighboring word vectors into one vector first
         if len(name.split()) > embedding_size:
             tmp = list()
             for w in name.split():
@@ -56,10 +79,12 @@ def fixed_length_vectors(namelist, embedding_size=entity_embedding_size):
                 elif w.lower() in w2v_model.vocab:
                     tmp.append(w2v_model.get_vector(w.lower()))
             avg_factor = np.ceil(len(tmp) / embedding_size).astype(int)
+
+            # Concat the averaged vectors
             for k in range(0, len(tmp), avg_factor):
                 vectors[i][k//avg_factor] = np.mean(tmp[k:k+avg_factor])
         else:
-            j = 0
+            j = 0  # Tracks the counting within a word vector group
             for w in name.split():
                 if w in w2v_model.vocab:
                     vectors[i][j] = w2v_model.get_vector(w)
@@ -76,9 +101,20 @@ def weighted_vectors():
 
 
 def fixed_length_vectors_by_text(names):
+    """
+    Turn the input word vector groups, with context entity word vector groups, into vector with fixed length.
+    Each entity (word group) takes entity_embedding_size rows, 
+    and the total embedding result takes context_embedding_size rows.
+    
+    :param names: the DataFrame with entities information 
+    :return: DataFrame with embedded vectors with context entity information.
+             Columns: ['text_id', 'vec']
+    """
     # print(names.groupby('text_id').count())
+
     names_by_text = names.groupby('text_id').aggregate(lambda x: set(x))
     names_by_text['vec'] = None
+
     # for idx in names_by_text.index:
     #     tmp = {}
     #     for n in names_by_text.name[idx]:
@@ -96,6 +132,9 @@ def fixed_length_vectors_by_text(names):
 
 
 def process_biotope_dict():
+    """
+    Convert biotope dictionary words to vectors by averaging individual word in each dict term respectively
+    """
     ref = parse_biotope_dict()
 
     vectors = averaging_vectors(ref.name)
@@ -106,23 +145,43 @@ def process_biotope_dict():
 
 
 def process_entity_and_label_table(tablename):
+    """
+    Generate embedded entity and label vectors with native methods.
+    entity vectors: fixed_length_vectors
+    label vectors: averaging vectors
+    Save the numpy array to local
+    
+    :param tablename: entity_and_label table generated from generate_tables.py
+    """
+    # Get "train" or "test" or "dev"
     prefix = tablename.split('_', -1)[-1]
+
     names_and_labels = parse_entity_and_label_table(tablename)
     names_vec = fixed_length_vectors(names_and_labels.name)
     labels_vec = averaging_vectors(names_and_labels.dict_name)
+
+    # Save to local
     names_and_labels.to_csv(os.path.join(path, '%s_names_and_labels.tsv' %prefix), sep='\t')
     np.save(os.path.join(path, '%s_names_vectors.npy' %prefix), names_vec)
     np.save(os.path.join(path, '%s_labels_vectors.npy' %prefix), labels_vec)
 
 
 def generated_normalized_dict_and_labels():
+    """
+    Run PCA on reference word space, n_component=0.95 (default), and turn the space to 139 dimensions.
+    Generate embedded label vectors by looking-up PCA-ed reference word vectors
+    Generate embedded entity and label vectors with fixed length methods.
+    Save the numpy array to local
+    """
+    # Reduce dimensions of random vectors
     ref = parse_biotope_dict()
     vectors = averaging_vectors(ref.name)
     vectors = reduce_dimension(vectors)
     np.save(os.path.join(path, 'OBT_VSM_norm.npy'), vectors)
     ref['vec'] = list(vectors)
     ref.to_csv(os.path.join(path, 'OBT_VSM_norm.tsv'), sep='\t')
 
+    # Parse entity_and_label tables
     for tablename in ['entity_and_label_list_BioNLP-OST-2019_BB-norm_train.tsv',
                       'entity_and_label_list_BioNLP-OST-2019_BB-norm_dev.tsv']:
         labels_id_and_labels = parse_entity_and_label_table(tablename)
@@ -132,28 +191,53 @@ def generated_normalized_dict_and_labels():
 
 
 def generate_context_entity_list(tablename):
+    """
+    Include entities appeared in the same article as input information.
+    Target entity: with size of embedding_vector_size*200
+    Other context entities: each with size of embedding_vector_size*200
+    Total input: padded to size of context_vector_size*200
+    
+    :param tablename: entity_and_label table generated from generate_tables.py
+    """
+    # target entity vectors
     names_and_labels = parse_entity_and_label_table(tablename)
     names_vec = fixed_length_vectors(names_and_labels.name)
 
+    # context entity vectors
     names_by_text = fixed_length_vectors_by_text(names_and_labels[['text_id', 'name']])
     concat_vec = np.stack(names_by_text.loc[names_and_labels.text_id, 'vec'], axis=0)
     names_vec = np.concatenate((names_vec, concat_vec), axis=1)
-#     print(names_vec.shape)
-    names_and_labels.to_csv(os.path.join(path, '%s_names_and_labels_with_context.tsv' %tablename.split('_', -1)[-1]), sep='\t')
-    np.save(os.path.join(path, '%s_names_vectors_with_context.npy' %tablename.split('_', -1)[-1]), names_vec)
+
+    # save
+    names_and_labels.to_csv(os.path.join(path, '%s_names_and_labels_with_context.tsv'
+                                         % tablename.split('_', -1)[-1]), sep='\t')
+    np.save(os.path.join(path, '%s_names_vectors_with_context.npy' % tablename.split('_', -1)[-1]), names_vec)
 
 
 def generate_five_fold_dataset(prediction=False):
+    """
+    Randomly select a percentage of data as test dataset (0.17 for training, and use the real test dataset for testing)
+    Generate embedded label vectors by looking-up PCA-ed reference word vectors
+    Generate embedded entity and label vectors with fixed length methods.
+    
+    :param prediction: if True: using pre-assigned text data set
+                       if False: randomly select test data set, frac=0.17
+    :return: 
+    """
+    # generate PCA-ed reference vectors
     ref = parse_biotope_dict()
     vectors = averaging_vectors(ref.name)
     vectors = reduce_dimension(vectors)
     np.save(os.path.join(path, 'OBT_VSM_norm.npy'), vectors)
     ref['vec'] = list(vectors)
     ref.to_csv(os.path.join(path, 'OBT_VSM_norm.tsv'), sep='\t')
 
+    # get total entity_and_label_list
     names_and_labels = parse_entity_and_label_table('entity_and_label_list_BioNLP-OST-2019_BB-norm_train.tsv')
     names_and_labels = pd.concat([names_and_labels,
                                  parse_entity_and_label_table('entity_and_label_list_BioNLP-OST-2019_BB-norm_dev.tsv')])
+
+    # if prediction: no label for test data set
     if prediction:
         training_size = len(names_and_labels)
         names_and_labels = pd.concat([names_and_labels,
@@ -183,14 +267,24 @@ def generate_five_fold_dataset(prediction=False):
 if __name__=="__main__":
     w2v_model = gensim.models.KeyedVectors.load_word2vec_format(
        '../input_data/wikipedia-pubmed-and-PMC-w2v.bin', binary=True)
-    vector_size = w2v_model.vector_size
+    vector_size = w2v_model.vector_size # 200
+
+
     whitelist = set('abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+
     path = os.path.join(os.getcwd(), '../input_data/vsm/')
 
+    """
+    Note: different methods use different methods to generate word vectors.
+    One only need to select desired subgroups of methods to generate dictionary, entity and label word vectors
+    """
     # process_biotope_dict(default_value=False)
     # process_entity_and_label_table('entity_and_label_list_BioNLP-OST-2019_BB-norm_train.tsv')
     # process_entity_and_label_table('entity_and_label_list_BioNLP-OST-2019_BB-norm_dev.tsv')
+
     # generated_normalized_dict_and_labels()
+
     # generate_context_entity_list('entity_and_label_list_BioNLP-OST-2019_BB-norm_train.tsv')
     # generate_context_entity_list('entity_and_label_list_BioNLP-OST-2019_BB-norm_dev.tsv')
-    generate_five_fold_dataset(prediction=True)
+
+    generate_five_fold_dataset(prediction=False)