performance optimization using aiohttp and change api endpoint

Pipfile

@@ -9,6 +9,7 @@ requests = "*"
 networkx = "*"
 pyvis = {editable = true, path = "./pyvis"}
 aiohttp = {extras = ["speedups"], version = "*"}
+pybase64 = "*"
 
 [dev-packages]
 black = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "e010c7fdd6165010e7a392341982c75ee2953e99593bbebab054d9388f6cf9d5"
+            "sha256": "f46d041868e3a17ffed758287363ab28b314e33d2594c5880248d6571cf69cf0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -193,28 +193,6 @@
             ],
             "version": "==0.2.0"
         },
-        "backports.zoneinfo": {
-            "hashes": [
-                "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf",
-                "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328",
-                "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546",
-                "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6",
-                "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570",
-                "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9",
-                "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7",
-                "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987",
-                "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722",
-                "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582",
-                "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc",
-                "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b",
-                "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1",
-                "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08",
-                "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac",
-                "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"
-            ],
-            "markers": "python_version >= '3.6' and python_version < '3.9'",
-            "version": "==0.2.1"
-        },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
@@ -591,14 +569,6 @@
             "markers": "python_version >= '3.7'",
             "version": "==4.11.4"
         },
-        "importlib-resources": {
-            "hashes": [
-                "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3",
-                "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"
-            ],
-            "markers": "python_version < '3.9'",
-            "version": "==5.7.1"
-        },
         "ipykernel": {
             "hashes": [
                 "sha256:6f42070a5d87ecbf4a2fc27a7faae8d690fd3794825a090ddf6b00b9678a5b69",
@@ -1140,6 +1110,96 @@
             "markers": "python_version >= '3.7'",
             "version": "==8.0.0"
         },
+        "pybase64": {
+            "hashes": [
+                "sha256:002402c6e32b8325410a7fe7633c94f7ec265ec5c164d931853a5c278d47d177",
+                "sha256:03aeffad37d282cf8af76304176bb3df7a8567bbaf9aebc3766f0229672500fa",
+                "sha256:06c9750db1881f8ea5568715871c04788c8f9f01f55f7d466b7aa2dd9461436b",
+                "sha256:074b5915c80317a69044c64d6521754242093f40454af75404072ef4b763a476",
+                "sha256:083bd31391501e77789cdef6989c1ac85d605f12dfaec58ecf003b56fcecd8a1",
+                "sha256:086d9a151e165ba5ac3b23f69927e8404edcd196b5f10ef5601a029fec21dcc3",
+                "sha256:115b486095ead72a788f7d9ff43d73457813755eab05846ad575ae0d9361e218",
+                "sha256:138c791a53f58b3293bed0251d61f7e42f95b03abf5048b06b1b5ffc8ba64c1c",
+                "sha256:14f06c398a9c95f62075ec54592da7dccab813cc68f27c23960c40a220a7d64c",
+                "sha256:17927f60dd0688062e39fcc1db31a00c90b6073d2f0718cdfb44b30bcfc3a5f4",
+                "sha256:1d4be51b4455697b1f99a15e42f95cb37f383832993d6d4e2f12df3264db066b",
+                "sha256:1d53038944833d1ed7f06651fd0947cddd0f98d508809f5a4028fd9966c607e8",
+                "sha256:1eb2bc17d3bf9523ec92640c62332b0e11192fe2fead2dd5f0861593719e4e2b",
+                "sha256:223fa4e6951d71e6b857b4b67c99496f1aa7e688c07798256d6082d623cfd4ec",
+                "sha256:227d86fbc8aae760277c08c893abfebbf95e49edcb55e8dc0941121d82e49f40",
+                "sha256:2ba652cf1c9c82bb5f2772c7d69e50d99d62c57312c1c0637411d0f70fa3abe3",
+                "sha256:2be7f47ae45b9aeba342773c6044ca03338620812a5257b2e221c1bb9665a7c8",
+                "sha256:2e7b3dc73cc149a7e6543cda0d006e3b366ccad42efaa4fa3df2d2ebbc5b2e35",
+                "sha256:4289b8149ea83fad9f0d198c91542585ba76ae1e6533997c63e0115d4d1ab79b",
+                "sha256:428a80f731732334d2cc48779c6b411014afcfcc54b048af2c41cededba3fe05",
+                "sha256:469020014cc82eb95bcb227f1a3b41cdfd3a90147b83757cc2c939f3ed722d03",
+                "sha256:4908de8e71a4ed7319d49482424bf51dca41b4bb729538388878325d1cee4eac",
+                "sha256:4a8c9960ae2944c69bedff36dac03af8beab7cce4f1b7a9708abdd48fb919277",
+                "sha256:5128d4dc733406777c968da211ae07255e213cd960a0b2e9fdb9d18948b6c198",
+                "sha256:520af879876b3b4d3808e1a1af3825d98cc7653330b44a1f7dc8806aceec2cb5",
+                "sha256:52a5abec54b4871d829b9fed0ae1d23ce9285e309d190c7e999b67c522bb63f5",
+                "sha256:553cb711ad3933ccc5ae6674254793cd7ba9b978b851cbc2716c47b16a108f67",
+                "sha256:5f85bd8e1a133552e6fb94cee60b69d337469d294f4708e7e2ce504cd013d81c",
+                "sha256:60f002841f8f02883ad92d3c8e8e556a567b6745c56fc3ce39079eb42fb206e6",
+                "sha256:60f15e1fa0664f4b2fafdf2208b624207a5c76bd928b47202003f4c71132a2f5",
+                "sha256:6245c3182dd5d8563e0d7a17946db6ccf0327c4302cc4100b4b2b4c7f81f2a9e",
+                "sha256:6310792de5ce27ffc420a0f0ea90a2b3b4d6d025061f608192533549bc03d124",
+                "sha256:631424e78da38b90cd09905abb7d8b8b9968924a061067a86cbde4e059102ae8",
+                "sha256:6a663cfba11ae2d860a716548f0a17b2d9403a5817e769c10368894239afafb6",
+                "sha256:6b8f126849ff3446ea1a0f2b6c92bf9486447144cf1e02c2b612e2267a4dcec1",
+                "sha256:71fe6a13add14c2a7ef0d800d1b0d73e1711dc0a4fb45a29c5a3cf586eb78a34",
+                "sha256:74754b0b6cc32970b43d1eb45494d4d44730a43bf3e4ee12a9ad8c737837f83e",
+                "sha256:76978e23187754fda63cbbbc824dc66ad945cb8e141b57e1970e1c885fd09b35",
+                "sha256:77741ca9d24a3cc43be7370be9d70fef335b646452b23a0601327e09b1acbcc7",
+                "sha256:779347de5125b16611d5410612e660aa7974a52cb14a10acab1cb7b123560b16",
+                "sha256:7ab7bfc4c0f4182d2142ed15e6afa17536c9258954ad1d23dc936ed411ac16ee",
+                "sha256:7c7e7ddeb9ba2c962e2eed139f91073605f6133f071fb522df977d10d0ca415d",
+                "sha256:7e399994df61d6e0dfabd06fe2a4b461e815e62ef5ebfc60e43419906e31b17f",
+                "sha256:7eeaabee388db02890746022b106834476a332f0b718de90b775bd5fe1140609",
+                "sha256:7fbafe9c33f484a97418ec06e3e46b98ff3f8e7ca65848c84fc4b6c3833acb1f",
+                "sha256:80c8dedda291c2967242de4c9bdddaa5c8151b238a2d83a4623d48393a2b31fa",
+                "sha256:8252c0d3116734484889728eebe125e6c5d2f3f1cc7cb8dbd110342df079ef94",
+                "sha256:84dcbdc19e45eacb7917afe0e76db70c5336e7375314792a9fa69c37ca5bbf56",
+                "sha256:8e067f9ca6e658151fb8c0a6405aff98df0e6d4a9726b65e8a1390326fd6c38c",
+                "sha256:8f1f2e29e7a17b9ec4a272beba4e98e924ae2011ed13a832462c592ef4180a00",
+                "sha256:90ebc76a1874c53c5bd7831ce2b5884dee113feaeca7b4cf40c958e5e7bad832",
+                "sha256:92cbd5b13f70658f9d130a39d67f4196bcf0cf83ed724420e96149874b568c63",
+                "sha256:95e0e62c35acd8117f7f779c03bab4e12d4b0cc0249a57a2a2895996263a82d1",
+                "sha256:983104702075c91af22dbdefa2ec4b99699126934f366c41e7bc05ad5e89e2e1",
+                "sha256:994195cf1cbdb3f8658304cea731d93a2d5cae83612cddb90e6415953e03c459",
+                "sha256:9a1168dec66aa4fb73062ca9a031ca0e028e90429bd5333e307e2d7c41ab7f69",
+                "sha256:9b47d2d0f0a0e2c1c74f4bff681dd4cc922ae9a4f6967444cfb3c80ac9fe2d54",
+                "sha256:9ccc61ea23deac82fa96c0ab0005e88451241bd00775c21f1d334a93485e945a",
+                "sha256:a08c5cc6df5a667f261ac1d2c23ff59fd095b7e9d5b39525eaf8245ff3f8b202",
+                "sha256:aa9e94fe7809584c5bd6c2cf1971e0be37065c518a491fd2f038e91d1697f5ac",
+                "sha256:ad9aa7a054386a2b7d6eb109f538bce9759a53341262e8b0879b62f79d9d8da7",
+                "sha256:aede141563e1ba320d400e1e74d0b3807e7fe04153e69a9606de19b7897307ea",
+                "sha256:b27b86544c87c9a876c70d8aa1c780ed1c6a80e24842bcf45d1f2cd2d5216e8e",
+                "sha256:b6e51e25fd8e83e2028d6d874af8059c1685f496520e137b87de3aded9d0055a",
+                "sha256:b7fe9f93c250a74e82de61cce67acb4d5cff5a73db60c6707efe22193ad987a4",
+                "sha256:b8c21cedb8271d9ebfa9787e3341cca282f96f00993f59062afa3da1782c8a31",
+                "sha256:bdbf00344e81169b6b89cf4952dcea45b92a2e521e4ec9f6c35e221d3868045d",
+                "sha256:bdf41018265b43495e5b709a13c1f09b18024c2a2d1ef5355a1c761ea2caa6c2",
+                "sha256:befd983a5a7b6551355b6abe56723f7f9f12c580e02f126b7883b075bebff25c",
+                "sha256:bf56503363852a744f85ad08fd20a0a286d2616aa4feb3c1f197e34d07c872bf",
+                "sha256:d3fb7a2836cdcabb00ca8cb5faf9ed93b26d45865745e32eb70e404faf8a6ccf",
+                "sha256:d505433a8b94dd8fff53d56b7f9e3f739e8a648bea204c487da08b69d681c160",
+                "sha256:d5c29a701502a1a26a5bae69397f616c3510f6c3e478a32c4ae1dd8f7b6b515b",
+                "sha256:d6d0b6b08933f3f908404a74a6befeff9d0c3fa145ed424e346f3bb09266e360",
+                "sha256:d77d7b67421915a082fd41274dd94cef9c82c9dce03e4f06636543f16c399407",
+                "sha256:db98ffd03ab10cf80844d7adee5c00ab73aa8ddc5c9685c705dcaeee2f85381a",
+                "sha256:e88827ec2fd12db9d3f93dd692c8881f292e065a3e50892ac67d0b0da7489f7c",
+                "sha256:ecae5643460762f4665d04087b59719e0cb43a1b8a407ca7c1b4c5d9e419416f",
+                "sha256:edd27bb6cf81c4a29362b98965906fd2534d57cc37efd3b1d8364224ffaee4f2",
+                "sha256:ee79b48c3a19e3f5ddf0c85e46f6c15c65d623b581dd0786c214d188e9539cad",
+                "sha256:ef8a18c78f1e5e7f37ed6c08e6564216c5b19564f5bb212438f1eba0f19c9607",
+                "sha256:f41afa7779e71d7176e174225298aeeeff0f5c871bc3e693070842b99ef2d935",
+                "sha256:f420da3258085992dfbc0e050f145d0f1b601bfb1013d3ad8377c158a20a2d9d",
+                "sha256:fa58addd890b20b922a2037169f530a91ed004b5c5d258fe3f896f4a6d57515f"
+            ],
+            "index": "pypi",
+            "version": "==1.2.2"
+        },
         "pycares": {
             "hashes": [
                 "sha256:03490be0e7b51a0c8073f877bec347eff31003f64f57d9518d419d9369452837",
@@ -1345,7 +1405,7 @@
                 "sha256:4c586de507202505346f3e32d1363eb9ed6932f0c2f63184dea88983ff4971e2",
                 "sha256:d2bbd99c320a2532ac71ff6a3164867884357da3e3301f0240090c5d2fdac7ec"
             ],
-            "markers": "python_version < '4' and python_full_version >= '3.6.3'",
+            "markers": "python_full_version >= '3.6.3' and python_full_version < '4.0.0'",
             "version": "==12.4.4"
         },
         "semver": {
@@ -1494,7 +1554,7 @@
                 "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
                 "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
             ],
-            "markers": "python_version < '3.9'",
+            "markers": "python_version >= '3.7'",
             "version": "==4.2.0"
         },
         "tzdata": {
@@ -1518,7 +1578,7 @@
                 "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14",
                 "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0'",
             "version": "==1.26.9"
         },
         "validators": {
@@ -1632,7 +1692,7 @@
                 "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
                 "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
             ],
-            "markers": "python_version < '3.10'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.8.0"
         }
     },
@@ -1890,7 +1950,7 @@
                 "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
                 "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
             ],
-            "markers": "python_version < '3.9'",
+            "markers": "python_version >= '3.7'",
             "version": "==4.2.0"
         }
     }

README.md

@@ -49,8 +49,13 @@ input to one GitHub public repo's URL(input)
   - [x] Visualize a basic `networkx` graph using `pyvis`
 - [x] Build a `streamlit` app
 - [ ] Performance optimization
-  - [x] `requests` to `aiohttp`: reduce latency 8-10x in my laptop(MacBook Pro)
+  - [x] `requests` to `aiohttp`
+  - [x] change fetching GitHub repository content api from preventing querying multiple times
+  - [ ] ...
+- [ ] Network analysis
+- [ ] Enhance network visualization UI
 ## References
 
 - [How to set up a perfect Python project](https://sourcery.ai/blog/python-best-practices/)
-- [Benchmark of popular graph/network packages](https://www.timlrx.com/blog/benchmark-of-popular-graph-network-packages)
+- [Benchmark of popular graph/network packages](https://www.timlrx.com/blog/benchmark-of-popular-graph-network-packages)
+- [Asynchronous HTTP Requests in Python with aiohttp and asyncio](https://www.twilio.com/blog/asynchronous-http-requests-in-python-with-aiohttp)

app.py

@@ -22,19 +22,16 @@ st.markdown(
 
 owner = st.text_input("Fill in the GitHub username", value="cyyeh")
 repo = st.text_input("Fill in the GitHib repository", value="py-code-analyzer")
-path = st.text_input(
-    "Fill in the target directory path. Default: the root directory",
+tree_sha = st.text_input(
+    "Fill in sha, ex: 2f387d0adea72a7b4c99a5e8fc3e4fd83b5469b8",
 )
-ref = st.text_input(
-    "Fill in the name of the commit/branch/tag. Default: the repository's default branch",
-)
-clicked_ok_button = st.button("OK")
+clicked_ok_button = st.button("OK", disabled=not owner or not repo or not tree_sha)
 st.markdown("---")
 
 
 @conditonal_decorator(time_function, DEV)
-def get_python_files(owner, repo, path, ref=""):
-    return CodeFetcher().get_python_files(owner, repo, path, ref=ref)
+def get_python_files(owner, repo, tree_sha):
+    return CodeFetcher().get_python_files(owner, repo, tree_sha)
 
 
 @conditonal_decorator(time_function, DEV)
@@ -56,13 +53,13 @@ def read_graph_visualization_file():
 
 if clicked_ok_button and owner and repo:
     with st.spinner("Getting python files..."):
-        python_files = get_python_files(owner, repo, path, ref=ref)
+        python_files = get_python_files(owner, repo, tree_sha)
 
     with st.spinner("Parsing python files and generating imports graph..."):
         imports_graph = generate_imports_graph(python_files)
 
     with st.spinner("Generating graph visualization file..."):
-        generate_graph_visualization_file(imports_graph, f"{owner}/{repo}/{path}")
+        generate_graph_visualization_file(imports_graph, f"{owner}/{repo}")
 
     with st.spinner("Showing the graph..."):
         graph_visualization_file = read_graph_visualization_file()

py_code_analyzer/code_fetcher.py

@@ -1,14 +1,24 @@
 """CodeFetcher deals with every detail of
 how to get all python files in the given directory
 """
-import os
-
 import requests
 
-# to increase api rate limiting
-# https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
-USER = os.environ.get("USER", "")
-PERSONAL_ACCESS_TOKEN = os.environ.get("PERSONAL_ACCESS_TOKEN", "")
+
+def construct_api_url(owner, repo, tree_sha, recursive):
+    import os
+
+    # to increase api rate limiting
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+    USER = os.environ.get("USER", "")
+    PERSONAL_ACCESS_TOKEN = os.environ.get("PERSONAL_ACCESS_TOKEN", "")
+
+    api_url = f"api.github.com/repos/{owner}/{repo}/git/trees/{tree_sha}"
+    if USER and PERSONAL_ACCESS_TOKEN:
+        api_url = f"{USER}:{PERSONAL_ACCESS_TOKEN}@{api_url}"
+    if recursive:
+        api_url += "?recursive=1"
+
+    return "https://" + api_url
 
 
 class CodeFetcher:
@@ -17,36 +27,29 @@ class CodeFetcher:
         cls,
         owner: str,
         repo: str,
-        path: str = "",
-        ref: str = "",
+        tree_sha: str,
         recursive: bool = True,
     ):
-        """https://docs.github.com/en/rest/repos/contents#get-repository-content"""
+        """https://docs.github.com/en/rest/git/trees#get-a-tree"""
+        # TODO: deal with truncated api results
 
-        api_url = (
-            f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
-            if not USER or not PERSONAL_ACCESS_TOKEN
-            else f"https://{USER}:{PERSONAL_ACCESS_TOKEN}@api.github.com/repos/{owner}/{repo}/contents/{path}"
-        )
-        if ref:
-            api_url += f"?ref={ref}"
+        api_url = construct_api_url(owner, repo, tree_sha, recursive)
 
-        python_files = []
-        api_results = requests.get(
+        response = requests.get(
             api_url, headers={"Accept": "application/vnd.github.v3+json"}
-        ).json()
-
-        for result in api_results:
-            if type(result) is dict:
-                if result["type"] == "file" and result["name"].endswith(".py"):
-                    python_files.append(result)
-                elif (
-                    recursive
-                    and result["type"] == "dir"
-                    and not result["name"].startswith(".")
-                ):
-                    python_files += cls.get_python_files(
-                        owner, repo, path=result["path"], ref=ref, recursive=recursive
-                    )
-
-        return python_files
+        )
+
+        api_results = response.json()
+        if response.status_code == requests.codes.ok:
+            python_files = [
+                result
+                for result in api_results["tree"]
+                if type(result) is dict
+                and result["type"] == "blob"
+                and result["path"].endswith(".py")
+            ]
+            return python_files
+        else:
+            print(api_results)
+
+        return []

py_code_analyzer/code_imports_analyzer.py

@@ -2,12 +2,44 @@
 to get what modules are imported in given python files, then uses networkx to generate imports graph
 """
 import ast
+import asyncio
 
 import aiohttp
+import pybase64
 
 from .graph_analyzer import GraphAnalyzer
 
 
+def construct_api_url(api_url):
+    import os
+
+    # to increase api rate limiting
+    # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+    USER = os.environ.get("USER", "")
+    PERSONAL_ACCESS_TOKEN = os.environ.get("PERSONAL_ACCESS_TOKEN", "")
+
+    api_urls = api_url.split("://")
+    if USER and PERSONAL_ACCESS_TOKEN:
+        api_url = f"{USER}:{PERSONAL_ACCESS_TOKEN}@{api_urls[-1]}"
+
+    return f"{api_urls[0]}://{api_url}"
+
+
+async def get_program_text(session, python_file):
+    async with session.get(
+        construct_api_url(python_file["url"]),
+        headers={"Accept": "application/vnd.github.v3+json"},
+    ) as response:
+        data = await response.json()
+        print(data)
+        if data["encoding"] == "base64":
+            return data["content"]
+        else:
+            print(
+                f"WARNING: {python_file}'s encoding is {data['encoding']}, not base64"
+            )
+
+
 class CodeImportsAnalyzer:
     class _NodeVisitor(ast.NodeVisitor):
         def __init__(self, imports):
@@ -35,21 +67,24 @@ class CodeImportsAnalyzer:
 
     async def analyze(self):
         async with aiohttp.ClientSession() as session:
+            tasks = []
             for python_file in self.python_files:
-                async with session.get(
-                    python_file["download_url"],
-                    headers={"Accept": "application/vnd.github.v3+json"},
-                ) as response:
-                    program = await response.text()
-                    tree = ast.parse(program)
-                    self.python_imports += [
-                        {
-                            "file_name": python_file["name"],
-                            "file_path": python_file["path"],
-                            "imports": [],
-                        }
-                    ]
-                    self._node_visitor.visit(tree)
+                self.python_imports += [
+                    {
+                        "file_name": python_file["path"].split("/")[-1],
+                        "file_path": python_file["path"],
+                        "imports": [],
+                    }
+                ]
+                tasks.append(
+                    asyncio.ensure_future(get_program_text(session, python_file))
+                )
+
+            base64_program_texts = await asyncio.gather(*tasks)
+            for base64_program_text in base64_program_texts:
+                program = pybase64.b64decode(base64_program_text)
+                tree = ast.parse(program)
+                self._node_visitor.visit(tree)
 
     def generate_imports_graph(self):
         # TODO: thought on how to improve the graph generation logic