External libraries

app.py

@@ -17,7 +17,13 @@ HfFolder.save_token(HF_TOKEN)
 datasets = {
     "stars": load_dataset("open-source-metrics/stars").sort('dates'),
     "issues": load_dataset("open-source-metrics/issues").sort('dates'),
-    "pip": load_dataset("open-source-metrics/pip").sort('day')
+    "pip": load_dataset("open-source-metrics/pip").sort('day'),
+}
+
+external_datasets = {
+    "stars": load_dataset("open-source-metrics/stars-external").sort('dates'),
+    "issues": load_dataset("open-source-metrics/issues-external").sort('dates'),
+    "pip": load_dataset("open-source-metrics/pip-external").sort('day')
 }
 
 val = 0
@@ -45,11 +51,18 @@ def _ignore_org_members(e):
 
     return e
 
+
 stars = {}
 for k, v in datasets['stars'].items():
     stars[k] = v.map(_range)
     val = 0
 
+stars_external = {}
+for k, v in external_datasets['stars'].items():
+    stars_external[k] = v.map(_range)
+    val = 0
+
+
 issues = {}
 for k, v in datasets['issues'].items():
     issues[k] = v.map(_range)
@@ -57,15 +70,18 @@ for k, v in datasets['issues'].items():
     issues[k] = issues[k].map(_ignore_org_members)
     val = 0
 
-datasets['stars'] = DatasetDict(**stars)
-datasets['issues'] = DatasetDict(**issues)
 
+issues_external = {}
+for k, v in external_datasets['issues'].items():
+    issues_external[k] = v.map(_range)
+    val = 0
+    issues_external[k] = issues_external[k].map(_ignore_org_members)
+    val = 0
 
-# datasets = {
-#     k1: DatasetDict({
-#         k2: v2.select(range(0, len(v2), max(1, int(len(v2) / 1000)))) for k2, v2 in v1.items()
-#     }) for k1, v1 in datasets.items()
-# }
+datasets['stars'] = DatasetDict(**stars)
+datasets['issues'] = DatasetDict(**issues)
+external_datasets['stars'] = DatasetDict(**stars_external)
+external_datasets['issues'] = DatasetDict(**issues_external)
 
 
 def link_values(library_names, returned_values):
@@ -108,25 +124,54 @@ class RequestHandler(SimpleHTTPRequestHandler):
         if self.path.startswith("/initialize"):
             dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
             dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
+
+            external_dataset_keys = {k: set(v.keys()) for k, v in external_datasets.items()}
+            external_dataset_with_most_splits = max([d for d in external_dataset_keys.values()], key=len)
+
             warnings = []
 
             for k, v in dataset_keys.items():
                 if len(v) < len(dataset_with_most_splits):
-                    warnings.extend(f"The {k} dataset does not contain all splits. Missing: {dataset_with_most_splits - v}")
+                    warnings.append(
+                        f"The {k} dataset does not contain all splits. Missing: {dataset_with_most_splits - v}."
+                        f"\nSelecting that split to show the pip install numbers will not work."
+                    )
+
+            for k, v in external_dataset_keys.items():
+                if len(v) < len(external_dataset_with_most_splits):
+                    warnings.append(
+                        f"The {k} dataset does not contain all splits. Missing: {external_dataset_with_most_splits - v}"
+                        f".\nSelecting that split to show the pip install numbers will not work."
+                    )
 
-            # TODO: Send and display warnings
             dataset_with_most_splits = list(dataset_with_most_splits)
             dataset_with_most_splits.sort()
 
-            return self.response(list(dataset_with_most_splits))
+            external_dataset_with_most_splits = list(external_dataset_with_most_splits)
+            external_dataset_with_most_splits.sort()
+
+            return self.response({
+                'internal': list(dataset_with_most_splits),
+                'external': external_dataset_with_most_splits,
+                'warnings': warnings
+            })
 
         if self.path.startswith("/retrievePipInstalls"):
+            errors = []
             library_names, options = parse_name_and_options(self.path)
 
             if '1' in options:
                 returned_values = {}
                 for library_name in library_names:
-                    for i in datasets['pip'][library_name]:
+                    ds = None
+                    if library_name in datasets['pip']:
+                        ds = datasets['pip'][library_name]
+                    elif library_name in external_datasets['pip']:
+                        ds = external_datasets['pip'][library_name]
+                    else:
+                        errors.append(f"No {library_name} found in internal or external datasets.")
+
+                    for i in ds:
                         if i['day'] in returned_values:
                             returned_values[i['day']]['Cumulated'] += i['num_downloads']
                         else:
@@ -137,7 +182,16 @@ class RequestHandler(SimpleHTTPRequestHandler):
             else:
                 returned_values = {}
                 for library_name in library_names:
-                    for i in datasets['pip'][library_name]:
+
+                    if library_name in datasets['pip']:
+                        ds = datasets['pip'][library_name]
+                    elif library_name in external_datasets['pip']:
+                        ds = external_datasets['pip'][library_name]
+                    else:
+                        errors.append(f"No {library_name} found in internal or external datasets for pip.")
+                        return {'errors': errors}
+
+                    for i in ds:
                         if i['day'] in returned_values:
                             returned_values[i['day']][library_name] = i['num_downloads']
                         else:
@@ -155,13 +209,21 @@ class RequestHandler(SimpleHTTPRequestHandler):
             return self.response(output)
 
         if self.path.startswith("/retrieveStars"):
+            errors = []
             library_names, options = parse_name_and_options(self.path)
             returned_values = {}
             dataset_dict = datasets['stars']
+            external_dataset_dict = external_datasets['stars']
             week_over_week = '1' in options
 
             for library_name in library_names:
-                dataset = dataset_dict[library_name]
+                if library_name in dataset_dict:
+                    dataset = dataset_dict[library_name]
+                elif library_name in external_dataset_dict:
+                    dataset = external_dataset_dict[library_name]
+                else:
+                    errors.append(f"No {library_name} found in internal or external datasets for stars.")
+                    return {'errors': errors}
 
                 last_value = 0
                 last_week = dataset[0]['week']
@@ -188,6 +250,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
 
 
         if self.path.startswith("/retrieveIssues"):
+            errors = []
             library_names, options = parse_name_and_options(self.path)
 
             exclude_org_members = '1' in options
@@ -195,10 +258,17 @@ class RequestHandler(SimpleHTTPRequestHandler):
 
             returned_values = {}
             dataset_dict = datasets['issues']
+            external_dataset_dict = external_datasets['issues']
             range_id = 'range' if not exclude_org_members else 'range_non_org'
 
             for library_name in library_names:
-                dataset = dataset_dict[library_name]
+                if library_name in dataset_dict:
+                    dataset = dataset_dict[library_name]
+                elif library_name in external_dataset_dict:
+                    dataset = external_dataset_dict[library_name]
+                else:
+                    errors.append(f"No {library_name} found in internal or external datasets for stars.")
+                    return {'errors': errors}
 
                 last_value = 0
                 last_week = dataset[0]['week']

index.html

@@ -10,6 +10,7 @@
     </head>
 
     <body>
+        <div id="warnings"></div>
         <div id="loading"></div>
         <div class="graphs">
             <div id="library-selector"></div>

index.js

@@ -82,7 +82,9 @@ const createButton = (title, libraries, methods) => {
         document.getElementById('issue-graph').innerHTML = ''
         const e = load()
         document.body.appendChild(e)
-        const selectedLibraries = libraries.filter(e => document.querySelector(`#${e}Checkbox`).checked);
+        const selectedInternalLibraries = libraries.internal.filter(e => document.querySelector(`#${e}Checkbox`).checked);
+        const selectedExternalLibraries = libraries.external.filter(e => document.querySelector(`#${e}Checkbox`).checked);
+        const selectedLibraries = selectedInternalLibraries.concat(selectedExternalLibraries);
 
         const relevantOptions = getCheckedOptions();
 
@@ -102,8 +104,11 @@ const createButton = (title, libraries, methods) => {
 
 const initialize = async () => {
     const inferResponse = await fetch(`initialize`);
+    console.log(inferResponse);
     const inferJson = await inferResponse.json();
-    // const graphsDiv = document.getElementsByClassName('graphs')[0];
+    console.log(inferJson);
+
+    const warnings = document.getElementById("warnings")
     const librarySelector = document.getElementById('library-selector');
     const graphSelector = document.getElementById('graph-selector');
     const selectorSubmit = document.getElementById('selector-submit');
@@ -116,7 +121,41 @@ const initialize = async () => {
     graphSpan.textContent = "Select graphs to display"
     graphSelector.appendChild(graphSpan);
 
-    for (const element of inferJson) {
+    if (inferJson.warnings.length > 0) {
+        const div = document.createElement('div');
+        div.classList.add('warning-div')
+
+        for (const warning of inferJson.warnings) {
+            const labelSpan = document.createElement('span');
+            labelSpan.textContent = `Warning: ${warning}`;
+
+            div.appendChild(labelSpan);
+        }
+        warnings.appendChild(div);
+    }
+
+    for (const element of inferJson.internal) {
+        const div = document.createElement('div');
+        const checkBox = document.createElement('input');
+        checkBox.type = 'checkbox'
+        checkBox.id = `${element}Checkbox`;
+
+        const checkBoxLabel = document.createElement('label');
+        const labelSpan = document.createElement('span')
+
+        labelSpan.textContent = element.charAt(0).toUpperCase() + element.slice(1)
+        checkBoxLabel.appendChild(checkBox)
+        checkBoxLabel.appendChild(labelSpan)
+
+        div.appendChild(checkBoxLabel)
+        librarySelector.appendChild(div)
+    }
+
+    const externalLibs = document.createElement("h3")
+    externalLibs.textContent = "External Libraries"
+    librarySelector.appendChild(externalLibs);
+
+    for (const element of inferJson.external) {
         const div = document.createElement('div');
         const checkBox = document.createElement('input');
         checkBox.type = 'checkbox'

style.css

@@ -52,6 +52,15 @@ button:hover {
     margin-left: 20px;
 }
 
+.warning-div {
+    background-color: rgb(255, 230, 164);
+    border-radius: 10px;
+    border-bottom: 3px solid rgb(235, 210, 144);
+    border-right: 3px solid rgb(235, 210, 144);
+    margin: 10px;
+    padding: 20px;
+}
+
 .submit {
     margin-bottom: 50px;
 }