use remote git repo

Dockerfile

@@ -5,7 +5,7 @@ FROM ubuntu:22.04
 # Install Node.js 22 - https://github.com/nodesource/distributions?tab=readme-ov-file#installation-instructions-deb
 # And python3
 RUN apt update \
-    && apt install -y curl python3 python3-pip \
+    && apt install -y git curl python3 python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
 RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \
@@ -18,8 +18,9 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \
 RUN pip install --upgrade "huggingface_hub[cli]"
 
 # Build the app
+WORKDIR /tmp
+RUN git clone https://github.com/severo/discourse-dashboard.git app
 WORKDIR /tmp/app
-COPY app/ ./
 RUN npm ci && npm rebuild && npm run build
 
 # The site space name must be passed as an environment variable

app/.gitignore

@@ -1,4 +0,0 @@
-.DS_Store
-/dist/
-node_modules/
-yarn-error.log

app/README.md

@@ -1,53 +0,0 @@
-# Observable Forum Dashboard
-
-This is an [Observable Framework](https://observablehq.com/framework) app. To start the local preview server, run:
-
-```
-npm run dev
-```
-
-Then visit <http://localhost:3000> to preview your app.
-
-For more, see <https://observablehq.com/framework/getting-started>.
-
-## Project structure
-
-A typical Framework project looks like this:
-
-```ini
-.
-├─ src
-│  ├─ components
-│  │  └─ timeline.js           # an importable module
-│  ├─ data
-│  │  ├─ launches.csv.js       # a data loader
-│  │  └─ events.json           # a static data file
-│  ├─ example-dashboard.md     # a page
-│  ├─ example-report.md        # another page
-│  └─ index.md                 # the home page
-├─ .gitignore
-├─ observablehq.config.js      # the app config file
-├─ package.json
-└─ README.md
-```
-
-**`src`** - This is the “source root” — where your source files live. Pages go here. Each page is a Markdown file. Observable Framework uses [file-based routing](https://observablehq.com/framework/routing), which means that the name of the file controls where the page is served. You can create as many pages as you like. Use folders to organize your pages.
-
-**`src/index.md`** - This is the home page for your app. You can have as many additional pages as you’d like, but you should always have a home page, too.
-
-**`src/data`** - You can put [data loaders](https://observablehq.com/framework/loaders) or static data files anywhere in your source root, but we recommend putting them here.
-
-**`src/components`** - You can put shared [JavaScript modules](https://observablehq.com/framework/javascript/imports) anywhere in your source root, but we recommend putting them here. This helps you pull code out of Markdown files and into JavaScript modules, making it easier to reuse code across pages, write tests and run linters, and even share code with vanilla web applications.
-
-**`observablehq.config.js`** - This is the [app configuration](https://observablehq.com/framework/config) file, such as the pages and sections in the sidebar navigation, and the app’s title.
-
-## Command reference
-
-| Command           | Description                                              |
-| ----------------- | -------------------------------------------------------- |
-| `npm install`            | Install or reinstall dependencies                        |
-| `npm run dev`        | Start local preview server                               |
-| `npm run build`      | Build your static site, generating `./dist`              |
-| `npm run deploy`     | Deploy your app to Observable                            |
-| `npm run clean`      | Clear the local data loader cache                        |
-| `npm run observable` | Run commands like `observable help`                      |

app/observablehq.config.js

@@ -1,37 +0,0 @@
-// See https://observablehq.com/framework/config for documentation.
-export default {
-  // The app’s title; used in the sidebar and webpage titles.
-  title: "Observable Forum Dashboard",
-
-  // The pages and sections in the sidebar. If you don’t specify this option,
-  // all pages will be listed in alphabetical order. Listing pages explicitly
-  // lets you organize them into sections and have unlisted pages.
-  // pages: [
-  //   {
-  //     name: "Examples",
-  //     pages: [
-  //       {name: "Dashboard", path: "/example-dashboard"},
-  //       {name: "Report", path: "/example-report"}
-  //     ]
-  //   }
-  // ],
-
-  // Content to add to the head of the page, e.g. for a favicon:
-  head: '<link rel="icon" href="observable.png" type="image/png" sizes="32x32">',
-
-  // The path to the source root.
-  root: "src",
-
-  // Some additional configuration options and their defaults:
-  // theme: "default", // try "light", "dark", "slate", etc.
-  // header: "", // what to show in the header (HTML)
-  // footer: "Built with Observable.", // what to show in the footer (HTML)
-  // sidebar: true, // whether to show the sidebar
-  // toc: true, // whether to show the table of contents
-  // pager: true, // whether to show previous & next links in the footer
-  // output: "dist", // path to the output root for build
-  // search: true, // activate search
-  // linkify: true, // convert URLs in Markdown to links
-  // typographer: false, // smart quotes and other typographic improvements
-  // cleanUrls: true, // drop .html from URLs
-};

app/package.json

@@ -1,23 +0,0 @@
-{
-  "type": "module",
-  "private": true,
-  "scripts": {
-    "clean": "rimraf src/.observablehq/cache",
-    "build": "observable build",
-    "dev": "observable preview",
-    "deploy": "observable deploy",
-    "observable": "observable"
-  },
-  "dependencies": {
-    "@observablehq/framework": "^1.11.0",
-    "d3-array": "^3.2.4",
-    "d3-dsv": "^3.0.1",
-    "d3-time-format": "^4.1.0"
-  },
-  "devDependencies": {
-    "rimraf": "^5.0.5"
-  },
-  "engines": {
-    "node": ">=18"
-  }
-}

app/src/.gitignore

@@ -1 +0,0 @@
-/.observablehq/cache/

app/src/components/fetch-and-retry.js

@@ -1,29 +0,0 @@
-const sleep = async (delay) =>
-  await new Promise((resolve) => setTimeout(resolve, delay));
-
-export const fetchAndRetry = async (
-  url,
-  options,
-  { fetchAfterMs = 200, retriesAfterMs = [200, 5 * 1000, 60 * 1000] } = {}
-) => {
-  let error;
-  for (let retryAfterMs of retriesAfterMs) {
-    try {
-      await sleep(fetchAfterMs);
-      const response = await fetch(url, options);
-      if (!response.ok) {
-        console.log(
-          `Failed to fetch ${url}: ${response.status} ${
-            response.statusText
-          } - Headers: ${JSON.stringify(response.headers)}`
-        );
-        await sleep(response.headers.get("Retry-After") ?? retryAfterMs);
-        continue;
-      }
-      return response;
-    } catch (e) {
-      error = e;
-    }
-  }
-  throw error;
-};

app/src/data/categories.csv.js

@@ -1,18 +0,0 @@
-import { csvFormat } from "d3-dsv";
-import setup from "./setup.json" with { type: "json" };
-import {fetchAndRetry} from "../components/fetch-and-retry.js";
-
-const url = setup.base_url + "/categories.json";
-const response = await fetchAndRetry(url);
-const json = await response.json();
-const categories = json.category_list.categories.map((d) => ({
-  id: d.id,
-  name: d.name,
-  color: d.color,
-  // description: d.description,
-  // description_text: d.description_text,
-  // subcategory_ids: d.subcategory_ids,
-}));
-
-// Write out csv formatted data.
-process.stdout.write(csvFormat(categories));

app/src/data/posts.csv.js

@@ -1,40 +0,0 @@
-import { csvFormat } from "d3-dsv";
-import setup from "./setup.json" with { type: "json" };
-import {fetchAndRetry} from "../components/fetch-and-retry.js";
-
-const postsChunks = [];
-const MAX_REQUESTS = 10000;
-let before = undefined;
-let i = 0;
-while (i++ < MAX_REQUESTS) {
-  const url =
-    setup.base_url + "/posts.json" + (before
-      ? "?" + new URLSearchParams({
-          before,
-        }).toString()
-      : "");
-  const response = await fetchAndRetry(url);
-
-  const json = await response.json();
-  const newPosts = json.latest_posts
-    .map((d) => ({
-      id: d.id,
-      topic_id: d.topic_id,
-      username: d.username,
-      avatar_template: d.avatar_template,
-      created_at: d.created_at,
-      incoming_link_count: d.incoming_link_count,
-      reads: d.reads,
-      category_id: d.category_id,
-      accepted_answer: d.accepted_answer,
-    }))
-    .filter((d) => d.id !== before);
-  postsChunks.push(newPosts);
-  if (!newPosts.length) {
-    break;
-  }
-  before = newPosts[newPosts.length - 1].id;
-}
-
-// Write out csv formatted data.
-process.stdout.write(csvFormat(postsChunks.flat()));

app/src/data/setup.json

@@ -1,3 +0,0 @@
-{
-    "base_url": "https://discuss.huggingface.co/"
-}

app/src/index.md

@@ -1,383 +0,0 @@
----
-theme: dashboard
-toc: false
----
-
-# Forum Dashboard
-
-<!-- Load and transform the data -->
-
-```js
-import * as d3 from "d3-array";
-const setup = await FileAttachment("data/setup.json").json();
-const url = setup.base_url;
-const posts = await FileAttachment("data/posts.csv").csv({ typed: true });
-const categoriesRaw = await FileAttachment("data/categories.csv").csv({
-  typed: true,
-});
-const topics = [
-  ...d3
-    .rollup(
-      posts,
-      (v) => ({
-        topic_id: v[0].topic_id,
-        category_id: v[0].category_id,
-        posts: v,
-        users: new Set(v.map((d) => d.username)),
-      }),
-      (d) => d.topic_id
-    )
-    .values(),
-];
-const users = d3.rollup(
-  posts,
-  (v) => ({ username: v[0].username, avatar_template: v[0].avatar_template }),
-  (d) => d.username
-);
-
-const topicsByCategory = d3.rollup(
-  topics,
-  (v) => v.length,
-  (d) => d.category_id
-);
-const categories = categoriesRaw.map((d) => ({
-  ...d,
-  topics: topicsByCategory.get(d.id) || 0,
-}));
-
-const tenTopUsers = d3
-  .rollups(
-    posts,
-    (v) => v.length,
-    (d) => d.username
-  )
-  .sort((a, b) => d3.descending(a[1], b[1]))
-  .slice(0, 10)
-  .map((d) => ({
-    username: d[0],
-    posts: d[1],
-  }));
-
-const tenTopAcceptedUsers = d3
-  .rollups(
-    posts.filter((d) => d.accepted_answer),
-    (v) => v.length,
-    (d) => d.username
-  )
-  .sort((a, b) => d3.descending(a[1], b[1]))
-  .slice(0, 10)
-  .map((d) => ({
-    username: d[0],
-    posts: d[1],
-  }));
-
-const NUM_USERS = 3;
-const topAcceptedUsersPerYear = d3
-  .rollups(
-    posts.filter((d) => d.accepted_answer),
-    (v) => v.length,
-    (d) => d.created_at.getFullYear(),
-    (d) => d.username
-  )
-  .flatMap(([year, users_stats]) => {
-    const top_usernames = users_stats
-      .sort(([_, posts_count_a], [__, posts_count_b]) =>
-        d3.descending(posts_count_a, posts_count_b)
-      )
-      .slice(0, NUM_USERS)
-      .map(([username]) => username);
-    return top_usernames.map((username, i) => ({
-      rank: i + 1,
-      year,
-      username,
-      src: url + users.get(username).avatar_template.replace("{size}", "400"),
-    }));
-  });
-
-const intervals = { month: "Month", year: "Year", day: "Day", week: "Week" };
-const interval = "month";
-const intervalLabel = intervals[interval];
-
-const color = {
-  users: "#e36209",
-  posts: "#3b5fc0",
-  accepted: "green",
-};
-
-const years = d3.extent(posts, (d) => d.created_at.getFullYear());
-```
-
-## Trends over time
-
-<!-- Cards with big numbers -->
-
-<div class="grid grid-cols-4">
-<div class="card">
-    <h2>Years</h2>
-    <span class="big">${years[0]}—${years[1]}</span>
-  </div>
-  <div class="card">
-    <h2>Topics</h2>
-    <span class="big">${topics.length.toLocaleString("en-US")}</span>
-    <p>Topics are the forum's questions, or threads.</p>
-  </div>
-  <div class="card">
-    <h2>Posts</h2>
-    <span class="big">${posts.length.toLocaleString("en-US")}</span>
-    <p>The posts are comments in a thread, ie the answers to a question. The topics are not included.</p>
-  </div>
-  <!-- <div class="card">
-    <h2>Posts per topic</h2>
-    <span class="big">${(posts.length / topics.size).toLocaleString("en-US", {
-    minimumFractionDigits: 2,
-    maximumFractionDigits: 2,
-  })}</span>
-  </div>
-  <div class="card">
-    <h2>Users per topic</h2>
-    <span class="big">${(d3.sum(topics, d => d[1].users.size) / topics.size).toLocaleString("en-US", {
-    minimumFractionDigits: 2,
-    maximumFractionDigits: 2,
-  })}</span>
-  </div> -->
-  <!-- <div class="card">
-    <h2>Categories</h2>
-    <span class="big">${categories.length.toLocaleString("en-US")}</span>
-  </div> -->
-  <div class="card">
-    <h2>Users</h2>
-    <span class="big">${users.size.toLocaleString("en-US")}</span>
-  </div>
-</div>
-
-<!-- Plot of monthly active users -->
-
-```js
-function postsMAU(data, { width } = {}) {
-  return Plot.plot({
-    title: `Monthly active users`,
-    width,
-    height: 300,
-    y: { grid: true, label: `users` },
-    // color: {...color, legend: true},
-    marks: [
-      Plot.lineY(
-        data,
-        Plot.binX(
-          { y: "distinct" },
-          {
-            x: "created_at",
-            y: "username",
-            stroke: color.users,
-            interval: "month",
-            tip: true,
-          }
-        )
-      ),
-      Plot.ruleY([0]),
-    ],
-  });
-}
-```
-
-<!-- Plot of posts history -->
-
-```js
-function postsTimeline(data, { width } = {}) {
-  return Plot.plot({
-    title: `Posts created every ${interval}`,
-    width,
-    height: 300,
-    y: { grid: true, label: "posts" },
-    // color: {...color, legend: true},
-    marks: [
-      Plot.lineY(
-        data,
-        Plot.binX(
-          { y: "count" },
-          { x: "created_at", stroke: color.posts, interval, tip: true }
-        )
-      ),
-      Plot.ruleY([0]),
-    ],
-  });
-}
-```
-
-<div class="grid grid-cols-2">
-  <div class="card">
-    ${resize((width) => postsMAU(posts, {width}))}
-  </div>
-  <div class="card">
-    ${resize((width) => postsTimeline(posts, {width}))}
-  </div>
-</div>
-
-## Topics
-
-<!-- Plot of topics per category -->
-
-```js
-function categoriesChart(data, { width }) {
-  return Plot.plot({
-    title: "Most active categories",
-    width,
-    height: 300,
-    marginTop: 0,
-    marginLeft: 150,
-    x: { grid: true, label: "Topics" },
-    y: { label: null },
-    marks: [
-      Plot.barX(data, {
-        x: "topics",
-        y: "name",
-        fill: (d) => "#" + d.color,
-        tip: true,
-        sort: { y: "-x" },
-      }),
-      Plot.ruleX([0]),
-    ],
-  });
-}
-```
-
-<!-- Posts per topic -->
-
-```js
-function answersPerTopicChart(data, { width }) {
-  return Plot.plot({
-    title: "Answers per topic",
-    width,
-    height: 300,
-    marginTop: 0,
-    marginLeft: 150,
-    x: { grid: true, label: "Proportion (%)", percent: true },
-    y: { label: "Answers", reverse: true },
-    marks: [
-      Plot.rectX(
-        data,
-        Plot.binY(
-          { x: "proportion" },
-          {
-            y: {
-              value: (d) => d.posts.length - 1,
-              thresholds: d3.range(-0.5, 10.5),
-            },
-            fill: (d) => (d.posts.length === 1 ? "#AAA" : "#DDD"),
-            tip: true,
-          }
-        )
-      ),
-      Plot.ruleX([0]),
-    ],
-  });
-}
-```
-
-<div class="grid grid-cols-2">
-  <div class="card">
-    ${resize((width) => categoriesChart(categories, {width}))}
-  </div>
-  <div class="card">
-    ${resize((width) => answersPerTopicChart(topics, {width}))}
-  </div>
-</div>
-
-## Users
-
-<!-- Top users -->
-
-```js
-function topUsersChart(data, { width }) {
-  return Plot.plot({
-    title: "Top posters",
-    width,
-    height: 300,
-    marginTop: 0,
-    marginLeft: 150,
-    x: { grid: true, label: "Posts" },
-    y: { label: null },
-    marks: [
-      Plot.barX(data, {
-        x: "posts",
-        y: "username",
-        fill: color.posts,
-        tip: true,
-        sort: { y: "-x" },
-      }),
-      Plot.ruleX([0]),
-    ],
-  });
-}
-
-function topAcceptedUsersChart(data, { width }) {
-  return Plot.plot({
-    title: "Users with most accepted answers",
-    width,
-    height: 300,
-    marginTop: 0,
-    marginLeft: 150,
-    x: { grid: true, label: "Posts" },
-    y: { label: null },
-    marks: [
-      Plot.barX(data, {
-        x: "posts",
-        y: "username",
-        fill: color.accepted,
-        tip: true,
-        sort: { y: "-x" },
-      }),
-      Plot.ruleX([0]),
-    ],
-  });
-}
-
-function topAcceptedUsersPerYearChart(data, { width }) {
-  return Plot.plot({
-    title: "User with most accepted answers per year",
-    width,
-    height: 300,
-    marginTop: 0,
-    marginLeft: 50,
-    marginRight: 50,
-    x: { grid: false, label: "Year" },
-    y: { grid: false, ticks: false, label: null, domain: [4, 0] },
-    color: { domain: [1, 2, 3], range: ["#FFD700", "#C0C0C0", "#CD7F32"] },
-    marks: [
-      Plot.image(data, {
-        x: (d) => new Date(d.year + "-01-01"),
-        y: "rank",
-        src: "src",
-        tip: true,
-        r: 20,
-        preserveAspectRatio: "xMidYMin slice",
-        title: (d) => `${d.username} - rank ${d.rank} (${d.year})`,
-      }),
-      Plot.dot(data, {
-        x: (d) => new Date(d.year + "-01-01"),
-        y: "rank",
-        r: 20,
-        stroke: "rank",
-        strokeWidth: 2,
-      }),
-      Plot.ruleY([4]),
-    ],
-  });
-}
-```
-
-<div class="grid grid-cols-2">
-  <div class="card">
-    ${resize((width) => topUsersChart(tenTopUsers, {width}))}
-  </div>
-  <!-- 
-  <div class="card">
-    ${resize((width) => topAcceptedUsersChart(tenTopAcceptedUsers, {width}))}
-  </div>
-  -->
-  <div class="card">
-    ${resize((width) => topAcceptedUsersPerYearChart(topAcceptedUsersPerYear, {width}))}
-  </div>
-</div>
-
-Data: ${url} activity from ${d3.min(posts, d => d.created_at).getFullYear()} to ${d3.max(posts, d => d.created_at).getFullYear()} downloaded using the [Discourse API](https://docs.discourse.org/).