Spaces:
Running
Running
hugobarauna
commited on
Commit
•
882e475
1
Parent(s):
0f53df6
Add github stars app example
Browse files- public-apps/github_stars.livemd +240 -0
public-apps/github_stars.livemd
ADDED
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<!-- livebook:{"app_settings":{"access_type":"public","auto_shutdown_ms":60000,"output_type":"rich","show_source":true,"slug":"github-stars"}} -->
|
2 |
+
|
3 |
+
# Github Repo Star History
|
4 |
+
|
5 |
+
```elixir
|
6 |
+
Mix.install([
|
7 |
+
{:req, "~> 0.3.9"},
|
8 |
+
{:kino, github: "livebook-dev/kino", override: true},
|
9 |
+
{:kino_vega_lite, "~> 0.1.7"},
|
10 |
+
{:kino_explorer, "~> 0.1.7"}
|
11 |
+
])
|
12 |
+
```
|
13 |
+
|
14 |
+
## About this notebook
|
15 |
+
|
16 |
+
```elixir
|
17 |
+
Kino.Markdown.new("""
|
18 |
+
## What is this?
|
19 |
+
|
20 |
+
This is a simple example of a [Livebook](https://livebook.dev/?utm_source=github-stars-app&utm_medium=livebook-apps) app that connects to an API, gets some data, and displays that data in a chart.
|
21 |
+
|
22 |
+
It uses Github's API to get data about the number of stars of repo had over time and plots that data.
|
23 |
+
|
24 |
+
### What are Livebook apps?
|
25 |
+
|
26 |
+
A Livebook app is a notebook-based app written in Elixir and using [Livebook](https://livebook.dev/?utm_source=github-stars-app&utm_medium=livebook-apps).
|
27 |
+
If you want to know more about Livebook apps, read or watch that [blog post](https://news.livebook.dev/deploy-notebooks-as-apps-quality-of-life-upgrades---launch-week-1---day-1-2OTEWI) or [video](https://www.youtube.com/watch?v=q7T6ue7cw1Q).
|
28 |
+
|
29 |
+
No HTML or Javascript was written for that app, only Elixir. 😉
|
30 |
+
|
31 |
+
You can view the source code by clicking the icon at the top corner.
|
32 |
+
""")
|
33 |
+
```
|
34 |
+
|
35 |
+
## Github API client
|
36 |
+
|
37 |
+
This notebook uses GitHub API to retrieve repository metadata. It needs a personal access token acess the API, otherwise it would easily rit the rate limit for unauthenticated requests.
|
38 |
+
|
39 |
+
Here's the URL to generate a Github API token: https://github.com/settings/tokens/new
|
40 |
+
|
41 |
+
```elixir
|
42 |
+
defmodule Github do
|
43 |
+
def get_star_dates(repo_name) do
|
44 |
+
case Req.get!(base_req(), url: "/repos/#{repo_name}/stargazers?per_page=100") do
|
45 |
+
%Req.Response{status: 200, headers: headers} ->
|
46 |
+
last_page = get_last_page_number(headers)
|
47 |
+
star_dates = concurret_paginate(repo_name, last_page)
|
48 |
+
{:ok, star_dates}
|
49 |
+
|
50 |
+
%Req.Response{status: 404, body: body} ->
|
51 |
+
{:error, body["message"]}
|
52 |
+
end
|
53 |
+
end
|
54 |
+
|
55 |
+
defp base_req() do
|
56 |
+
Req.new(
|
57 |
+
base_url: "https://api.github.com",
|
58 |
+
auth: {:bearer, github_token()},
|
59 |
+
headers: [
|
60 |
+
accept: "application/vnd.github.star+json",
|
61 |
+
"X-GitHub-Api-Version": "2022-11-28"
|
62 |
+
]
|
63 |
+
)
|
64 |
+
end
|
65 |
+
|
66 |
+
defp github_token do
|
67 |
+
System.fetch_env!("LB_GITHUB_TOKEN")
|
68 |
+
end
|
69 |
+
|
70 |
+
defp get_last_page_number(headers) do
|
71 |
+
link_header =
|
72 |
+
headers
|
73 |
+
|> Enum.find(fn {key, _value} -> key == "link" end)
|
74 |
+
|> elem(1)
|
75 |
+
|
76 |
+
last_link =
|
77 |
+
link_header
|
78 |
+
|> String.split(",")
|
79 |
+
|> Enum.map(fn link ->
|
80 |
+
[url, rel] = String.split(link, ";")
|
81 |
+
[url] = Regex.run(~r/<(.*)>/, url, capture: :all_but_first)
|
82 |
+
[_, rel] = String.split(rel, "=")
|
83 |
+
rel = String.trim(rel)
|
84 |
+
[url, rel]
|
85 |
+
end)
|
86 |
+
|> Enum.find(fn [_url, rel] -> rel == "\"last\"" end)
|
87 |
+
|
88 |
+
if last_link == nil do
|
89 |
+
""
|
90 |
+
else
|
91 |
+
[page, _] = last_link
|
92 |
+
|
93 |
+
%{"page_number" => page_number} =
|
94 |
+
Regex.named_captures(~r/.*&page=(?<page_number>\d+)/, page)
|
95 |
+
|
96 |
+
String.to_integer(page_number)
|
97 |
+
end
|
98 |
+
end
|
99 |
+
|
100 |
+
defp concurret_paginate(repo_name, last_page) do
|
101 |
+
1..last_page
|
102 |
+
|> Task.async_stream(
|
103 |
+
fn page ->
|
104 |
+
response =
|
105 |
+
Req.get!(base_req(), url: "/repos/#{repo_name}/stargazers?per_page=100&page=#{page}")
|
106 |
+
|
107 |
+
if response.status != 200, do: IO.inspect("BAM!")
|
108 |
+
parse_star_dates(response.body)
|
109 |
+
end,
|
110 |
+
max_concurrency: 60
|
111 |
+
)
|
112 |
+
|> Enum.reduce([], fn {:ok, star_dates}, star_dates_acc ->
|
113 |
+
[star_dates | star_dates_acc]
|
114 |
+
end)
|
115 |
+
|> List.flatten()
|
116 |
+
end
|
117 |
+
|
118 |
+
defp parse_star_dates(body) do
|
119 |
+
body
|
120 |
+
|> Enum.map(fn %{"starred_at" => date} ->
|
121 |
+
{:ok, datetime, _} = DateTime.from_iso8601(date)
|
122 |
+
DateTime.to_date(datetime)
|
123 |
+
end)
|
124 |
+
end
|
125 |
+
end
|
126 |
+
```
|
127 |
+
|
128 |
+
## Data processing
|
129 |
+
|
130 |
+
```elixir
|
131 |
+
process_data = fn star_dates ->
|
132 |
+
star_dates
|
133 |
+
|> List.flatten()
|
134 |
+
|> Enum.group_by(& &1)
|
135 |
+
|> Enum.map(fn {date, dates} -> {date, Enum.count(dates)} end)
|
136 |
+
|> List.keysort(0, {:asc, Date})
|
137 |
+
|> Enum.reduce(%{date: [], stars: []}, fn {date, stars}, data ->
|
138 |
+
%{date: dates_acc, stars: stars_acc} = data
|
139 |
+
|
140 |
+
cumulative_stars =
|
141 |
+
if List.first(stars_acc) == nil do
|
142 |
+
0 + stars
|
143 |
+
else
|
144 |
+
List.first(stars_acc) + stars
|
145 |
+
end
|
146 |
+
|
147 |
+
%{date: [date | dates_acc], stars: [cumulative_stars | stars_acc]}
|
148 |
+
end)
|
149 |
+
end
|
150 |
+
```
|
151 |
+
|
152 |
+
## Chart generation
|
153 |
+
|
154 |
+
```elixir
|
155 |
+
generate_chart = fn data, repo_name ->
|
156 |
+
VegaLite.new(
|
157 |
+
width: 850,
|
158 |
+
height: 550,
|
159 |
+
title: [text: "⭐️ Github Stars History for #{repo_name}", font_size: 20]
|
160 |
+
)
|
161 |
+
|> VegaLite.data_from_values(data, only: ["date", "stars"])
|
162 |
+
|> VegaLite.mark(:line, tooltip: true)
|
163 |
+
|> VegaLite.encode_field(:x, "date",
|
164 |
+
type: :temporal,
|
165 |
+
axis: [
|
166 |
+
label_expr:
|
167 |
+
"[timeFormat(datum.value, '%b'), timeFormat(datum.value, '%m') == '01' ? timeFormat(datum.value, '%Y') : ['']]",
|
168 |
+
grid_dash: [
|
169 |
+
condition: [test: [field: "value", time_unit: "month", equal: 1], value: []],
|
170 |
+
value: [2, 2]
|
171 |
+
],
|
172 |
+
tick_dash: [
|
173 |
+
condition: [test: [field: "value", time_unit: "month", equal: 1], value: []],
|
174 |
+
value: [2, 2]
|
175 |
+
]
|
176 |
+
]
|
177 |
+
)
|
178 |
+
|> VegaLite.encode_field(:y, "stars", type: :quantitative)
|
179 |
+
end
|
180 |
+
```
|
181 |
+
|
182 |
+
## UI
|
183 |
+
|
184 |
+
```elixir
|
185 |
+
display_data = fn frame, %{data: data, repo_name: repo_name}, to: origin ->
|
186 |
+
chart = generate_chart.(data, repo_name)
|
187 |
+
|
188 |
+
Kino.Frame.render(
|
189 |
+
frame,
|
190 |
+
Kino.Layout.tabs(
|
191 |
+
Chart: chart,
|
192 |
+
Table: Explorer.DataFrame.new(data)
|
193 |
+
),
|
194 |
+
to: origin
|
195 |
+
)
|
196 |
+
end
|
197 |
+
|
198 |
+
display_error = fn frame, message, to: origin ->
|
199 |
+
Kino.Frame.render(
|
200 |
+
frame,
|
201 |
+
Kino.Markdown.new("<p style='font-style:italic; color:red'>#{message}</p"),
|
202 |
+
to: origin
|
203 |
+
)
|
204 |
+
end
|
205 |
+
```
|
206 |
+
|
207 |
+
```elixir
|
208 |
+
form =
|
209 |
+
Kino.Control.form(
|
210 |
+
[
|
211 |
+
repo_name: Kino.Input.text("Github full repo name", default: "livebook-dev/livebook")
|
212 |
+
],
|
213 |
+
submit: "Generate Chart"
|
214 |
+
)
|
215 |
+
|
216 |
+
frame = Kino.Frame.new()
|
217 |
+
|
218 |
+
Kino.listen(form, fn %{data: %{repo_name: repo_name}, origin: origin} ->
|
219 |
+
Kino.Frame.clear(frame, to: origin)
|
220 |
+
|
221 |
+
if repo_name == "" or repo_name == nil do
|
222 |
+
display_error.(frame, "repo can't be blank", to: origin)
|
223 |
+
else
|
224 |
+
Kino.Frame.append(frame, "Collecting stars from Github API...", to: origin)
|
225 |
+
|
226 |
+
case Github.get_star_dates(repo_name) do
|
227 |
+
{:ok, star_dates} ->
|
228 |
+
data = process_data.(star_dates)
|
229 |
+
display_data.(frame, %{data: data, repo_name: repo_name}, to: origin)
|
230 |
+
|
231 |
+
{:error, "Not Found"} ->
|
232 |
+
display_error.(frame, "repo not found", to: origin)
|
233 |
+
end
|
234 |
+
end
|
235 |
+
end)
|
236 |
+
|
237 |
+
Kino.Layout.grid([form, frame], boxed: true)
|
238 |
+
```
|
239 |
+
|
240 |
+
<!-- livebook:{"offset":6577,"stamp":{"token":"QTEyOEdDTQ.rWfWLINAy1PfVNfbUpFdFcjlCS5ZNA0Ra5urU6W-uAdotpTYFGcq3lLeWDI.dToRryKxCtcsE31r.7KlM22UTAeP1ouFC-5GmrBKtqEGoWYGDwiukixdaxWv208qQYdWZ8v_pNfPFTv_y.lT_DCnW7lnzkLodYV1jEhQ","version":1}} -->
|