{desc}
" if desc else "" + course["notebooks"] = [ + { + "title": get_notebook_title(root / course_id / nb) + or re.sub(r"^\d+_", "", nb.replace(".py", "")).replace("_", " ").title(), + "html_path": f"{course_id}/{nb.replace('.py', '.html')}", + "local_html_path": nb.replace(".py", ".html"), + } + for nb in course.get("notebooks", []) + ] + index_md = root / course_id / "index.md" + post = frontmatter.load(index_md) + course["body_html"] = md.markdown(post.content, extensions=["fenced_code", "tables"]) + return data + + +def render(template, path, **kwargs): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(template.render(**kwargs)) + + +def main(): + parser = argparse.ArgumentParser(description="Generate static site from lesson data") + parser.add_argument("--root", required=True, help="Project root directory") + parser.add_argument("--output", required=True, help="Output directory") + parser.add_argument("--data", required=True, help="Path to lessons JSON file") + args = parser.parse_args() + + root = Path(args.root) + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + + lessons = transform_lessons(json.loads(Path(args.data).read_text()), root) + env = Environment(loader=FileSystemLoader(root / "templates")) + current_year = datetime.date.today().year + + render( + env.get_template("index.html"), + output / "index.html", + courses=lessons, + current_year=current_year, + root_path="", + ) + + assets_src = root / "assets" + if assets_src.exists(): + shutil.copytree(assets_src, output / "assets", dirs_exist_ok=True) + + for course_id, lesson in lessons.items(): + render( + env.get_template("lesson.html"), + output / course_id / "index.html", + lesson=lesson, + current_year=current_year, + root_path="../", + ) + + page_template = env.get_template("page.html") + for page_src in sorted((root / "pages").glob("*.md")): + post = frontmatter.load(page_src) + render( + page_template, + output / page_src.stem / "index.html", + title=post.get("title", page_src.stem), + body_html=md.markdown(post.content, extensions=["fenced_code", "tables"]), + current_year=current_year, + root_path="../", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_empty_cells.py b/bin/check_empty_cells.py similarity index 99% rename from scripts/check_empty_cells.py rename to bin/check_empty_cells.py index d03bf27d5a234d399a44f33fb302399be8e0b14c..90208c51f9f97392051a5f769cb8ffa55e945947 100644 --- a/scripts/check_empty_cells.py +++ b/bin/check_empty_cells.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """ Script to detect empty cells in marimo notebooks. @@ -15,7 +15,6 @@ This script will: """ import os -import re import sys from pathlib import Path from typing import List, Tuple @@ -136,4 +135,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/bin/check_missing_titles.py b/bin/check_missing_titles.py new file mode 100644 index 0000000000000000000000000000000000000000..bea2f33400e50525d2d5a259350d31289aa1df87 --- /dev/null +++ b/bin/check_missing_titles.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Report marimo notebooks that are missing an H1 title.""" + +import sys +from pathlib import Path + +from utils import get_notebook_title + + +def main(): + root = Path(__file__).parent.parent + notebooks = sorted(root.glob("*/[0-9]*.py")) + missing = [nb for nb in notebooks if get_notebook_title(nb) is None] + if missing: + for nb in missing: + print(nb.relative_to(root)) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/bin/check_notebook_packages.py b/bin/check_notebook_packages.py new file mode 100644 index 0000000000000000000000000000000000000000..f5787b28d72b2b9057588cf21cd7bb7883af72f3 --- /dev/null +++ b/bin/check_notebook_packages.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +"""Check that marimo notebooks in the same lesson directory agree on package versions. + +It is acceptable for different notebooks in a directory to specify different packages, +but if two or more notebooks specify the same package, their version constraints must +be identical. +""" + +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path + + +# Regex to extract the inline script metadata block (PEP 723) +SCRIPT_BLOCK_RE = re.compile(r"^# /// script\s*\n((?:#[^\n]*\n)*?)# ///", re.MULTILINE) +DEPENDENCY_LINE_RE = re.compile(r'^#\s+"([^"]+)",?\s*$') + + +def parse_script_header(text: str) -> list[str]: + """Return the list of dependency strings from a PEP 723 script header, or [].""" + match = SCRIPT_BLOCK_RE.search(text) + if not match: + return [] + block = match.group(1) + deps: list[str] = [] + in_deps = False + for raw_line in block.splitlines(): + line = raw_line.lstrip("#").strip() + if line.startswith("dependencies"): + in_deps = True + continue + if in_deps: + if line.startswith("]"): + break + # strip surrounding quotes and comma: e.g. ' "polars==1.0",' -> 'polars==1.0' + stripped = line.strip().strip('"\'').rstrip(",").strip('"\'') + if stripped: + deps.append(stripped) + return deps + + +def package_name(dep: str) -> str: + """Extract the bare package name from a PEP 508 dependency string. + + Examples: + "polars==1.22.0" -> "polars" + "pandas>=2.0,<3" -> "pandas" + "marimo" -> "marimo" + """ + return re.split(r"[><=!;\s\[]", dep, maxsplit=1)[0].lower() + + +def check_directory(lesson_dir: Path, only: set[str]) -> list[str]: + """Return a list of error messages for version inconsistencies among *only* in lesson_dir.""" + # Map package name -> {version_spec: [notebook_path, ...]} + seen: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list)) + + for nb in sorted(lesson_dir.glob("*.py")): + if nb.name not in only: + continue + try: + text = nb.read_text(encoding="utf-8") + except IOError: + continue + if "marimo.App" not in text: + continue + for dep in parse_script_header(text): + name = package_name(dep) + seen[name][dep].append(nb.name) + + errors: list[str] = [] + for name, specs in sorted(seen.items()): + if len(specs) > 1: + errors.append(f" Package '{name}' has conflicting specifications:") + for spec, files in sorted(specs.items()): + errors.append(f" {spec!r} in: {', '.join(files)}") + return errors + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("notebooks", nargs="+", metavar="NOTEBOOK", + help="notebook files to check (grouped by directory)") + args = parser.parse_args() + + dir_filter: dict[Path, set[str]] = defaultdict(set) + for nb_path in (Path(p) for p in args.notebooks): + dir_filter[nb_path.parent].add(nb_path.name) + + total_errors = 0 + for lesson_dir, only in sorted(dir_filter.items()): + errors = check_directory(lesson_dir, only=only) + if errors: + print(f"\n{lesson_dir}/") + for msg in errors: + print(msg) + total_errors += len(errors) + + if total_errors: + print(f"\nFound package version inconsistencies in {total_errors} package(s).") + sys.exit(1) + else: + print("All package version specifications are consistent.") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/bin/create_sql_lab.sql b/bin/create_sql_lab.sql new file mode 100644 index 0000000000000000000000000000000000000000..1a787b94c5cc7c2d149094459ce99274078627c6 --- /dev/null +++ b/bin/create_sql_lab.sql @@ -0,0 +1,22 @@ +create table job ( + name text not null, + credits real not null +); + +create table work ( + person text not null, + job text not null +); + +insert into job values +('calibrate', 1.5), +('clean', 0.5); + +insert into work values +('Amal', 'calibrate'), +('Amal', 'clean'), +('Amal', 'complain'), +('Gita', 'clean'), +('Gita', 'clean'), +('Gita', 'complain'), +('Madhi', 'complain'); diff --git a/bin/create_sql_penguins.py b/bin/create_sql_penguins.py new file mode 100644 index 0000000000000000000000000000000000000000..4f06490d23aac562f95555fdec0da9ec57742eb5 --- /dev/null +++ b/bin/create_sql_penguins.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import csv +import sqlite3 +import sys + + +SCHEMA = """ +CREATE TABLE penguins ( + species text, + island text, + bill_length_mm real, + bill_depth_mm real, + flipper_length_mm real, + body_mass_g real, + sex text +); +""" + +def main(): + infile = sys.argv[1] + outfile = sys.argv[2] + + con = sqlite3.connect(outfile) + con.execute(SCHEMA) + + with open(infile, newline="") as f: + reader = csv.DictReader(f) + rows = [ + ( + row["species"], + row["island"], + float(row["bill_length_mm"]) if row["bill_length_mm"] else None, + float(row["bill_depth_mm"]) if row["bill_depth_mm"] else None, + float(row["flipper_length_mm"]) if row["flipper_length_mm"] else None, + float(row["body_mass_g"]) if row["body_mass_g"] else None, + row["sex"] if row["sex"] else None, + ) + for row in reader + ] + + con.executemany( + "INSERT INTO penguins VALUES (?, ?, ?, ?, ?, ?, ?)", rows + ) + con.commit() + con.close() + + +if __name__ == "__main__": + main() diff --git a/bin/create_sql_survey.py b/bin/create_sql_survey.py new file mode 100644 index 0000000000000000000000000000000000000000..e0340e694824530d9e8f7f011d1c7c81cb428c1d --- /dev/null +++ b/bin/create_sql_survey.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +import datetime +import faker +import itertools +import random +import sqlite3 +import sys + + +LOCALE = "es" + +NUM_PERSONS = 6 + +DATE_START = datetime.date(2025, 9, 1) +DATE_END = datetime.date(2025, 12, 31) +DATE_DURATION = 7 + +NUM_MACHINES = 5 + +CREATE_PERSONS = """\ +create table person( + person_id text not null primary key, + personal text not null, + family text not null, + supervisor_id text, + foreign key(supervisor_id) references person(person_id) +); +""" +INSERT_PERSONS = """\ +insert into person values (:person_id, :personal, :family, :supervisor_id); +""" + +CREATE_SURVEYS = """\ +create table survey( + survey_id text not null primary key, + person_id text not null, + start_date text, + end_date text, + foreign key(person_id) references person(person_id) +); +""" +INSERT_SURVEYS = """\ +insert into survey values(:survey_id, :person_id, :start, :end); +""" + +CREATE_MACHINES = """\ +create table machine( + machine_id text not null primary key, + machine_type text not null +); +""" +INSERT_MACHINES = """\ +insert into machine values(:machine_id, :machine_type); +""" + +CREATE_RATINGS = """\ +create table rating( + person_id text not null, + machine_id text not null, + level integer, + foreign key(person_id) references person(person_id), + foreign key(machine_id) references machine(machine_id) +); +""" +INSERT_RATINGS = """\ +insert into rating values(:person_id, :machine_id, :level); +""" + +def main(): + db_name = sys.argv[1] + seed = int(sys.argv[2]) + random.seed(seed) + + persons_counter = itertools.count() + next(persons_counter) + persons = gen_persons(NUM_PERSONS, persons_counter) + + supers = gen_persons(int(NUM_PERSONS / 2), persons_counter) + for p in persons: + p["supervisor_id"] = random.choice(supers)["person_id"] + if len(supers) > 1: + supers[0]["supervisor_id"] = supers[-1]["person_id"] + + surveys = gen_surveys(persons + supers[0:int(len(supers)/2)]) + surveys[int(len(surveys)/2)]["start"] = None + + cnx = sqlite3.connect(db_name) + cur = cnx.cursor() + + everyone = persons + supers + random.shuffle(everyone) + cur.execute(CREATE_PERSONS) + cur.executemany(INSERT_PERSONS, everyone) + + cur.execute(CREATE_SURVEYS) + cur.executemany(INSERT_SURVEYS, surveys) + + machines = gen_machines() + cur.execute(CREATE_MACHINES) + cur.executemany(INSERT_MACHINES, machines) + + ratings = gen_ratings(everyone, machines) + cur.execute(CREATE_RATINGS) + cur.executemany(INSERT_RATINGS, ratings) + + cnx.commit() + cnx.close() + + +def gen_machines(): + adjectives = "hydraulic rotary modular industrial automated".split() + nouns = "press conveyor generator actuator compressor".split() + machines = set() + while len(machines) < NUM_MACHINES: + candidate = f"{random.choice(adjectives)} {random.choice(nouns)}" + if candidate not in machines: + machines.add(candidate) + counter = itertools.count() + next(counter) + return [ + {"machine_id": f"M{next(counter):04d}", "machine_type": m} + for m in machines + ] + + +def gen_persons(num, counter): + fake = faker.Faker(LOCALE) + fake.seed_instance(random.randint(0, 1_000_000)) + return [ + { + "person_id": f"P{next(counter):03d}", + "personal": fake.first_name(), + "family": fake.last_name(), + "supervisor_id": None, + } + for _ in range(num) + ] + + +def gen_ratings(persons, machines): + temp = {} + while len(temp) < int(len(persons) * len(machines) / 4): + p = random.choice(persons)["person_id"] + m = random.choice(machines)["machine_id"] + if (p, m) in temp: + continue + temp[(p, m)] = random.choice([None, 1, 2, 3]) + return [ + {"person_id": p, "machine_id": m, "level": v} + for ((p, m), v) in temp.items() + ] + +def gen_surveys(persons): + surveys = [] + counter = itertools.count() + next(counter) + for person in persons: + person_id = person["person_id"] + start = DATE_START + while start <= DATE_END: + survey_id = f"S{next(counter):04d}" + end = start + datetime.timedelta(days=random.randint(1, DATE_DURATION)) + surveys.append({ + "survey_id": survey_id, + "person_id": person_id, + "start": start.isoformat(), + "end": end.isoformat() if end <= DATE_END else None + }) + start = end + datetime.timedelta(days=random.randint(1, DATE_DURATION)) + return surveys + + +if __name__ == "__main__": + main() diff --git a/bin/extract.py b/bin/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..44b9134242ebc8da41c52b8d23cd2df32da213bf --- /dev/null +++ b/bin/extract.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""Extract lesson metadata and notebook lists into a JSON file.""" + +import argparse +import json +import re +from pathlib import Path + +import frontmatter + + +NOTEBOOK_PATTERN = re.compile(r"^\d{2}_.*\.py$") + + +def extract_lessons(root: Path) -> dict: + lessons = {} + for index_file in sorted(root.glob("*/index.md")): + lesson_dir = index_file.parent + post = frontmatter.load(index_file) + notebooks = sorted( + p.name + for p in lesson_dir.glob("*.py") + if NOTEBOOK_PATTERN.match(p.name) + ) + lessons[lesson_dir.name] = { + **post.metadata, + "notebooks": notebooks, + } + return lessons + + +def main(): + parser = argparse.ArgumentParser(description="Extract lesson metadata to JSON") + parser.add_argument("--root", required=True, help="Project root directory") + parser.add_argument("--data", required=True, help="Output JSON file") + args = parser.parse_args() + + root = Path(args.root) + data = Path(args.data) + data.parent.mkdir(parents=True, exist_ok=True) + + lessons = extract_lessons(root) + data.write_text(json.dumps(lessons, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/preview.py b/bin/preview.py similarity index 98% rename from scripts/preview.py rename to bin/preview.py index d5dce86167ced55f3744a4506b64e35ddb7224f1..626f688900b1e5e620918d9c12281c2eefdb1c9e 100644 --- a/scripts/preview.py +++ b/bin/preview.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import os import subprocess import argparse import webbrowser -import time import sys from pathlib import Path diff --git a/bin/run_notebooks.sh b/bin/run_notebooks.sh new file mode 100755 index 0000000000000000000000000000000000000000..a79da7c9575f0476edf25919085e96a8e2fc0535 --- /dev/null +++ b/bin/run_notebooks.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +for nb in $* +do + cd $(dirname $nb) + if ! output=$(uv run $(basename $nb) 2>&1); then + echo "=== $nb ===" + echo "$output" + echo + fi + cd $OLDPWD +done diff --git a/bin/utils.py b/bin/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e5a86e32a052c33ffa90bb4a2b47b20c59b1092b --- /dev/null +++ b/bin/utils.py @@ -0,0 +1,14 @@ +"""Utility functions for working with marimo notebooks.""" + +import re +from pathlib import Path + + +def get_notebook_title(path: Path) -> str | None: + """Return the first H1 Markdown heading in a marimo notebook, or None.""" + text = path.read_text(encoding="utf-8") + for match in re.finditer(r'mo\.md\(r?f?"""(.*?)"""', text, re.DOTALL): + for line in match.group(1).splitlines(): + if line.strip().startswith("# "): + return line.strip()[2:].strip() + return None diff --git a/daft/README.md b/daft/README.md deleted file mode 100644 index e51a66fd2f1991207d7bbe3a52703550a3526b3e..0000000000000000000000000000000000000000 --- a/daft/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Readme -marimo-version: 0.18.4 ---- - -# Learn Daft - -_π§ This collection is a work in progress. Please help us add notebooks!_ - -This collection of marimo notebooks is designed to teach you the basics of -Daft, a distributed dataframe engine that unifies data engineering, analytics & ML/AI workflows. - -**Help us build this course! βοΈ** - -We're seeking contributors to help us build these notebooks. Every contributor -will be acknowledged as an author in this README and in their contributed -notebooks. Head over to the [tracking -issue](https://github.com/marimo-team/learn/issues/43) to sign up for a planned -notebook or propose your own. - -**Running notebooks.** To run a notebook locally, use - -```bash -uvx marimo editExplore our collection of interactive notebooks for Python, data science, and machine learning.
- -
- Learn by doing with interactive notebooks that run directly in your browser.
-Real-world examples and applications to reinforce your understanding.
-From Python basics to advanced machine learning concepts.
-- {% if course.get('description_html') %} - {{ course.get('description_html')|safe }} - {% endif %} -
-Help us improve these learning materials by contributing to the GitHub repository. We welcome new content, bug fixes, and improvements!
- - - Contribute on GitHub - -Help us improve these learning materials by contributing to the GitHub repository. We welcome new content, bug fixes, and improvements!
+ + {% include "icons/github-20.svg" %} + Contribute on GitHub + +Explore our collection of interactive notebooks for Python, data science, and machine learning.
+ +
+ Learn by doing with interactive notebooks that run directly in your browser.
+Real-world examples and applications to reinforce your understanding.
+From Python basics to advanced machine learning concepts.
++ {% if course.get('description_html') %} + {{ course.get('description_html')|safe }} + {% endif %} +
+
+ To run a notebook locally, use:
+uvx marimo edit <URL>You can also open notebooks in our online playground by adding marimo.app/ to a notebook's URL.