Spaces:
Running
Running
| <!-- livebook:{"app_settings":{"access_type":"public","output_type":"rich","show_source":true,"slug":"github-stars"}} --> | |
| # Github Repo Star History | |
| ```elixir | |
| Mix.install([ | |
| {:req, "~> 0.3.9"}, | |
| {:kino, github: "livebook-dev/kino", override: true}, | |
| {:kino_vega_lite, "~> 0.1.7"}, | |
| {:kino_explorer, "~> 0.1.7"} | |
| ]) | |
| ``` | |
| ## About this notebook | |
| ```elixir | |
| Kino.Markdown.new(""" | |
| ## What is this? | |
| This is a simple example of a [Livebook](https://livebook.dev/?utm_source=github-stars-app&utm_medium=livebook-apps) app that connects to an API, gets some data, and displays that data in a chart. | |
| It uses Github's API to get data about the number of stars of repo had over time and plots that data. | |
| ### What are Livebook apps? | |
| A Livebook app is a notebook-based app written in Elixir and using [Livebook](https://livebook.dev/?utm_source=github-stars-app&utm_medium=livebook-apps). | |
| If you want to know more about Livebook apps, read or watch that [blog post](https://news.livebook.dev/deploy-notebooks-as-apps-quality-of-life-upgrades---launch-week-1---day-1-2OTEWI) or [video](https://www.youtube.com/watch?v=q7T6ue7cw1Q). | |
| No HTML or Javascript was written for that app, only Elixir. 😉 | |
| You can view the source code by clicking the icon at the top corner. | |
| """) | |
| ``` | |
| ## Github API client | |
| This notebook uses GitHub API to retrieve repository metadata. It needs a personal access token acess the API, otherwise it would easily rit the rate limit for unauthenticated requests. | |
| Here's the URL to generate a Github API token: https://github.com/settings/tokens/new | |
| ```elixir | |
| defmodule Github do | |
| def get_star_dates(repo_name) do | |
| case Req.get!(base_req(), url: "/repos/#{repo_name}/stargazers?per_page=100") do | |
| %Req.Response{status: 200, headers: headers} -> | |
| last_page = get_last_page_number(headers) | |
| star_dates = concurret_paginate(repo_name, last_page) | |
| {:ok, star_dates} | |
| %Req.Response{status: 404, body: body} -> | |
| {:error, body["message"]} | |
| end | |
| end | |
| defp base_req() do | |
| Req.new( | |
| base_url: "https://api.github.com", | |
| auth: {:bearer, github_token()}, | |
| headers: [ | |
| accept: "application/vnd.github.star+json", | |
| "X-GitHub-Api-Version": "2022-11-28" | |
| ] | |
| ) | |
| end | |
| defp github_token do | |
| System.fetch_env!("LB_GITHUB_TOKEN") | |
| end | |
| defp get_last_page_number(headers) do | |
| link_header = | |
| headers | |
| |> Enum.find(fn {key, _value} -> key == "link" end) | |
| |> elem(1) | |
| last_link = | |
| link_header | |
| |> String.split(",") | |
| |> Enum.map(fn link -> | |
| [url, rel] = String.split(link, ";") | |
| [url] = Regex.run(~r/<(.*)>/, url, capture: :all_but_first) | |
| [_, rel] = String.split(rel, "=") | |
| rel = String.trim(rel) | |
| [url, rel] | |
| end) | |
| |> Enum.find(fn [_url, rel] -> rel == "\"last\"" end) | |
| if last_link == nil do | |
| "" | |
| else | |
| [page, _] = last_link | |
| %{"page_number" => page_number} = | |
| Regex.named_captures(~r/.*&page=(?<page_number>\d+)/, page) | |
| String.to_integer(page_number) | |
| end | |
| end | |
| defp concurret_paginate(repo_name, last_page) do | |
| 1..last_page | |
| |> Task.async_stream( | |
| fn page -> | |
| response = | |
| Req.get!(base_req(), url: "/repos/#{repo_name}/stargazers?per_page=100&page=#{page}") | |
| if response.status != 200, do: IO.inspect("BAM!") | |
| parse_star_dates(response.body) | |
| end, | |
| max_concurrency: 60 | |
| ) | |
| |> Enum.reduce([], fn {:ok, star_dates}, star_dates_acc -> | |
| [star_dates | star_dates_acc] | |
| end) | |
| |> List.flatten() | |
| end | |
| defp parse_star_dates(body) do | |
| body | |
| |> Enum.map(fn %{"starred_at" => date} -> | |
| {:ok, datetime, _} = DateTime.from_iso8601(date) | |
| DateTime.to_date(datetime) | |
| end) | |
| end | |
| end | |
| ``` | |
| ## Data processing | |
| ```elixir | |
| process_data = fn star_dates -> | |
| star_dates | |
| |> List.flatten() | |
| |> Enum.group_by(& &1) | |
| |> Enum.map(fn {date, dates} -> {date, Enum.count(dates)} end) | |
| |> List.keysort(0, {:asc, Date}) | |
| |> Enum.reduce(%{date: [], stars: []}, fn {date, stars}, data -> | |
| %{date: dates_acc, stars: stars_acc} = data | |
| cumulative_stars = | |
| if List.first(stars_acc) == nil do | |
| 0 + stars | |
| else | |
| List.first(stars_acc) + stars | |
| end | |
| %{date: [date | dates_acc], stars: [cumulative_stars | stars_acc]} | |
| end) | |
| end | |
| ``` | |
| ## Chart generation | |
| ```elixir | |
| generate_chart = fn data, repo_name -> | |
| VegaLite.new( | |
| width: 850, | |
| height: 550, | |
| title: [text: "⭐️ Github Stars History for #{repo_name}", font_size: 20] | |
| ) | |
| |> VegaLite.data_from_values(data, only: ["date", "stars"]) | |
| |> VegaLite.mark(:line, tooltip: true) | |
| |> VegaLite.encode_field(:x, "date", | |
| type: :temporal, | |
| axis: [ | |
| label_expr: | |
| "[timeFormat(datum.value, '%b'), timeFormat(datum.value, '%m') == '01' ? timeFormat(datum.value, '%Y') : ['']]", | |
| grid_dash: [ | |
| condition: [test: [field: "value", time_unit: "month", equal: 1], value: []], | |
| value: [2, 2] | |
| ], | |
| tick_dash: [ | |
| condition: [test: [field: "value", time_unit: "month", equal: 1], value: []], | |
| value: [2, 2] | |
| ] | |
| ] | |
| ) | |
| |> VegaLite.encode_field(:y, "stars", type: :quantitative) | |
| end | |
| ``` | |
| ## UI | |
| ```elixir | |
| display_data = fn frame, %{data: data, repo_name: repo_name}, to: origin -> | |
| chart = generate_chart.(data, repo_name) | |
| Kino.Frame.render( | |
| frame, | |
| Kino.Layout.tabs( | |
| Chart: chart, | |
| Table: Explorer.DataFrame.new(data) | |
| ), | |
| to: origin | |
| ) | |
| end | |
| display_error = fn frame, message, to: origin -> | |
| Kino.Frame.render( | |
| frame, | |
| Kino.Markdown.new("<p style='font-style:italic; color:red'>#{message}</p"), | |
| to: origin | |
| ) | |
| end | |
| ``` | |
| ```elixir | |
| form = | |
| Kino.Control.form( | |
| [ | |
| repo_name: Kino.Input.text("Github full repo name", default: "livebook-dev/livebook") | |
| ], | |
| submit: "Generate Chart" | |
| ) | |
| frame = Kino.Frame.new() | |
| Kino.listen(form, fn %{data: %{repo_name: repo_name}, origin: origin} -> | |
| Kino.Frame.clear(frame, to: origin) | |
| if repo_name == "" or repo_name == nil do | |
| display_error.(frame, "repo can't be blank", to: origin) | |
| else | |
| Kino.Frame.append(frame, "Collecting stars from Github API...", to: origin) | |
| case Github.get_star_dates(repo_name) do | |
| {:ok, star_dates} -> | |
| data = process_data.(star_dates) | |
| display_data.(frame, %{data: data, repo_name: repo_name}, to: origin) | |
| {:error, "Not Found"} -> | |
| display_error.(frame, "repo not found", to: origin) | |
| end | |
| end | |
| end) | |
| Kino.Layout.grid([form, frame], boxed: true) | |
| ``` | |