Add documentation

build_embeddings.py

@@ -1,3 +1,21 @@
+"""
+Module which builds embeddings for issues and pull requests
+
+The module is designed to be run from the command line and takes the following arguments:
+
+        --input_filename: The name of the file containing the issues and pull requests
+        --model_id: The name of the sentence transformer model to use
+        --issue_type: The type of issue to embed (either "issue" or "pull")
+        --n_issues: The number of issues to embed
+        --update: Whether to update the existing embeddings
+
+The module saves the embeddings to a file called <issue_type>_embeddings.npy and the index to a file called
+embedding_index_to_<issue_type>.json
+
+The index provides a mapping from the index of the embedding to the issue or pull request number.
+
+"""
+
 import argparse
 import json
 import logging

build_issue_dict.py

@@ -1,3 +1,7 @@
+"""
+Module which builds a dictionary keyed by issue number from a json file
+"""
+
 import argparse
 import json
 

defaults.py

@@ -4,3 +4,4 @@ OWNER = "huggingface"
 REPO = "transformers"
 TOKEN = os.environ.get("GITHUB_TOKEN")
 GITHUB_API_VERSION = "2022-11-28"
+ISSUE_JSON_FILE = "issues.json"

fetch.py

@@ -1,9 +1,20 @@
 """
 Script to fetch issues from the transformers repo and save them to a json file
+
+The script can be run from the command line with the following arguments:
+            --update: Whether to update the existing file. If True the script will fetch
+                the most recent issues and append them to the file
+            --overwrite: Whether to overwrite the existing file
+            --output_filename: The name of the output file
+            --github_api_version: The version of the GitHub API to use
+            --owner: The owner of the repo
+            --repo: The name of the repo
+            --token: The GitHub token to use
+            --n_pages: The number of pages to fetch. Useful for testing
+
 """
 
 import argparse
-import datetime
 import logging
 import json
 import os
@@ -11,17 +22,13 @@ import os
 import requests
 import numpy as np
 
-from defaults import OWNER, REPO, GITHUB_API_VERSION, TOKEN
+from defaults import OWNER, REPO, GITHUB_API_VERSION, TOKEN, ISSUE_JSON_FILE
 
 logging.basicConfig(level=logging.INFO)
-
 logger = logging.getLogger(__name__)
 
-today = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
-
-JSON_FILE = "issues.json"
-UPDATE_FILE = False
-OVERWRITE_FILE = True
+UPDATE_FILE = True
+OVERWRITE_FILE = False
 
 
 def get_last_entry(file_path):
@@ -41,7 +48,7 @@ def get_last_issue_number(file_path):
 def get_issues(
     overwrite=OVERWRITE_FILE,
     update=UPDATE_FILE,
-    output_filename=JSON_FILE,
+    output_filename=ISSUE_JSON_FILE,
     github_api_version=GITHUB_API_VERSION,
     owner=OWNER,
     repo=REPO,
@@ -125,9 +132,9 @@ def get_issues(
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--update", action="store_true", default=True)
-    parser.add_argument("--overwrite", action="store_true", default=False)
-    parser.add_argument("--output_filename", type=str, default=JSON_FILE)
+    parser.add_argument("--update", action="store_true", default=UPDATE_FILE)
+    parser.add_argument("--overwrite", action="store_true", default=OVERWRITE_FILE)
+    parser.add_argument("--output_filename", type=str, default=ISSUE_JSON_FILE)
     parser.add_argument("--github_api_version", type=str, default=GITHUB_API_VERSION)
     parser.add_argument("--owner", type=str, default=OWNER)
     parser.add_argument("--repo", type=str, default=REPO)

retrieval.py

@@ -1,3 +1,8 @@
+"""
+Module which contains functionality to retrieve the most similar issues for a given query
+"""
+
+
 import argparse
 import json
 

update_stored_issues.py

@@ -1,5 +1,14 @@
 """
-Module which updates any of the issues to reflect changes in the issue state
+Module which updates any of the issues to reflect changes in the issue state e.g. new comments
+
+The module can be run from the command line using the following arguments:
+            --input_filename: The name of the input file containing the issues
+            --output_filename: The name of the output file to save the updated issues
+            --github_api_version: The version of the GitHub API to use
+            --owner: The owner of the repo
+            --repo: The name of the repo
+            --token: The GitHub token to use
+            --n_pages: The number of pages to fetch. Useful for testing
 """
 import argparse
 import json
@@ -9,7 +18,7 @@ import os
 import numpy as np
 import requests
 
-from defaults import TOKEN, OWNER, REPO, GITHUB_API_VERSION
+from defaults import TOKEN, OWNER, REPO, GITHUB_API_VERSION, ISSUE_JSON_FILE
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -18,8 +27,8 @@ JSON_FILE = "issues.json"
 
 
 def update_issues(
-    input_filename=JSON_FILE,
-    output_filename=JSON_FILE,
+    input_filename=ISSUE_JSON_FILE,
+    output_filename=ISSUE_JSON_FILE,
     github_api_version=GITHUB_API_VERSION,
     owner=OWNER,
     repo=REPO,