HF-QA-Demo / data /get_hf_repositories_urls.py
KonradSzafer's picture
initial commit
c69cba4
import json
import argparse
import requests
from typing import List
def get_repositories_names(token: str, min_stars: int) -> List[str]:
repos_per_page = 100
repo_names = []
i = 0
while True:
url = \
f'https://api.github.com/orgs/huggingface/repos?' \
f'per_page={repos_per_page}&page={i}'
headers = {'Authorization': f'token {token}'}
response = requests.get(url, headers=headers)
if response.status_code == 200:
repos = json.loads(response.content)
repo_names += [
repo['full_name'] for repo in repos
if repo['stargazers_count'] >= min_stars
]
if len(repos) < repos_per_page:
break
i += 1
else:
return 'Error: '+str(response.status_code)
return list(set(repo_names))
def save_repositories_urls(repositories_names: List[str], output_filename: str):
urls = [f'https://github.com/{repo_name}' for repo_name in repositories_names]
data = {'urls': urls}
with open(output_filename, 'w') as f:
json.dump(data, f, indent=4)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--token', type=str)
parser.add_argument('--stars', type=str)
args = parser.parse_args()
repositories = get_repositories_names(token=args.token, min_stars=int(args.stars))
repositories += [
'huggingface/hf-endpoints-documentation',
'gradio-app/gradio'
]
print(f'Found {len(repositories)} repositories with at least {args.stars} stars')
save_repositories_urls(repositories, 'datasets/hf_repositories_urls_scraped.json')