Question: I’m doing a bunch of filters and other transform applications including a group_by on a polars data frame, the objective being to count the number of html tags in a single column per date per publisher. Here is the code: 120 def contains_html3(mindate, parquet_file = default_file, fieldname = "text"): 121 """ checks if html tags are in field """ 122 123 124 html_tags = [ 125 "<html>", "</html>", "<head>", "</head>", "<title>", "</title>", "<meta>", "</meta>", "<link>", "</link>", "<style>", 126 "<body>", "</body>", "<header>", "</header>", "<footer>", "</footer>", "<nav>", "</nav>", "<main>", "</main>", 127 "<section>", "</section>", "<article>", "</article>", "<aside>", "</aside>", "<h1>", "</h1>", "<h2>", "</h2>", 128 "<h3>", "</h3>", "<h4>", "</h4>", "<h5>", "</h5>", "<h6>", "</h6>", "<p>", "</p>", "<ul>", "</ul>", "<ol>", "</ol>", 129 "<li>", "</li>", "<div>", "</div>", "<span>", "</span>", "<a>", "</a>", "<img>", "</img>", "<table>", "</table>", 130 "<thead>", "</thead>", "<tbody>", "</tbody>", "<tr>", "</tr>", "<td>", "</td>", "<th>", "</th>", "<form>", "</form>", 131 "<input>", "</input>", "<textarea>", "</textarea>", "<button>", "</button>", "<select>", "</select>", "<option>", 132 "<script>", "</script>", "<noscript>", "</noscript>", "<iframe>", "</iframe>", "<canvas>", "</canvas>", "<source>"] 133 134 gg = (pl.scan_parquet(parquet_file) 135 .cast({"date": pl.Date}) 136 .select("publisher", "date", fieldname) 137 .drop_nulls() 138 .group_by("publisher", "date") 139 .agg(pl.col(fieldname).str.contains_any(html_tags).sum().alias(fieldname)) 140 .filter(pl.col(fieldname) > 0) 141 .sort(fieldname, descending = True)).collect() 142 143 return gg  Here is example output for fieldname = "text": Out[8]: shape: (22_925, 3) ┌───────────────────────────┬────────────┬──────┐ │ publisher ┆ date ┆ text │ │ --- ┆ --- ┆ --- │ │ str ┆ date ┆ u64 │ ╞═══════════════════════════╪════════════╪══════╡ │ Kronen Zeitung ┆ 2024-11-20 ┆ 183 │ │ Kronen Zeitung ┆ 2024-10-25 ┆ 180 │ │ Kronen Zeitung ┆ 2024-11-14 ┆ 174 │ │ Kronen Zeitung ┆ 2024-11-06 ┆ 172 │ │ Kronen Zeitung ┆ 2024-10-31 ┆ 171 │ │ … ┆ … ┆ … │ │ The Faroe Islands Podcast ┆ 2020-03-31 ┆ 1 │ │ Sunday Standard ┆ 2024-07-16 ┆ 1 │ │ Stabroek News ┆ 2024-08-17 ┆ 1 │ │ CivilNet ┆ 2024-09-01 ┆ 1 │ │ The Star ┆ 2024-06-23 ┆ 1 │ └───────────────────────────┴────────────┴──────┘  The issue is that instead of just passing a single fieldname = "text" argument, I would like to pass a list (for example ["text", "text1", "text2", ...]). The idea would be to run the bottom three lines in the chain for each element of the list. I could wrap the whole polars method chain in a for loop and then join the resulting data frames, but is there a better way? For example to insert a map, or foreach, or other such construct after the group_by clause, and then have polars add a new column for each field name without using a loop? What's the best way of handling this? EDIT WITH REPRODUCIBLE CODE This will produce a dataframe df and a sample output tc, with all four columns text1 through text4, summed and sorted, but not using polars for the last step. #colorscheme orbital dark import polars as pl import datetime as dt from math import sqrt import random random.seed(8472) from functools import reduce html_tags = [ "<html>", "</html>", "<head>", "</head>", "<title>", "</title>", "<meta>", "</meta>", "<link>", "</link>", "<style>", "</style>", "<body>", "</body>", "<header>", "</header>", "<footer>", "</footer>", "<nav>", "</nav>", "<main>", "</main>", "<section>", "</section>", "<article>", "</article>", "<aside>", "</aside>", "<h1>", "</h1>", "<h2>", "</h2>", "<h3>", "</h3>", "<h4>", "</h4>", "<h5>", "</h5>", "<h6>", "</h6>", "<p>", "</p>", "<ul>", "</ul>", "<ol>", "</ol>", "<li>", "</li>", "<div>", "</div>", "<span>", "</span>", "<a>", "</a>", "<img>", "</img>", "<table>", "</table>", "<thead>", "</thead>", "<tbody>", "</tbody>", "<tr>", "</tr>", "<td>", "</td>", "<th>", "</th>", "<form>", "</form>", "<input>", "</input>", "<textarea>", "</textarea>", "<button>", "</button>", "<select>", "</select>", "<option>", "</option>", "<script>", "</script>", "<noscript>", "</noscript>", "<iframe>", "</iframe>", "<canvas>", "</canvas>", "<source>", "</source>"] def makeword(alphaLength): """Make a dummy name if none provided.""" consonants = "bcdfghjklmnpqrstvwxyz" vowels = "aeiou" word = ''.join(random.choice(consonants if i % 2 == 0 else vowels) for i in range(alphaLength)) return word def makepara(nwords): """Make a paragraph of dummy text.""" words = [makeword(random.randint(3, 10)) for _ in range(nwords)] tags = random.choices(html_tags, k=3) parawords = random.choices(tags + words, k=nwords) para = " ".join(parawords) return para def generate_df_with_tags(rows = 100, numdates = 10, num_publishers = 6): publishers = [makeword(5) for _ in range(num_publishers)] datesrange = pl.date_range(start := dt.datetime(2024, 2, 1), end = start + dt.timedelta(days = numdates - 1), eager = True) dates = sorted(random.choices(datesrange, k = rows)) df = pl.DataFrame({ "publisher": random.choices(publishers, k = rows), "date": dates, "text1": [makepara(15) for _ in range(rows)], "text2": [makepara(15) for _ in range(rows)], "text3": [makepara(15) for _ in range(rows)], "text4": [makepara(15) for _ in range(rows)] }) return df def contains_html_so(parquet_file, fieldname = "text"): """ checks if html tags are in field """ gg = (pl.scan_parquet(parquet_file) .select("publisher", "date", fieldname) .drop_nulls() .group_by("publisher", "date") .agg(pl.col(fieldname).str.contains_any(html_tags).sum().alias(fieldname)) .filter(pl.col(fieldname) > 0) .sort(fieldname, descending = True)).collect() return gg if __name__ == "__main__": df = generate_df_with_tags(100) df.write_parquet("/tmp/test.parquet") tc = [contains_html_so("/tmp/test.parquet", fieldname = x) for x in ["text1", "text2", "text3", "text4"]] tcr = (reduce(lambda x, y: x.join(y, how = "full", on = ["publisher", "date"], coalesce = True), tc) .with_columns(( pl.col("text1").fill_null(0) + pl.col("text2").fill_null(0) + pl.col("text3").fill_null(0) + pl.col("text4").fill_null(0)).alias("sum")).sort("sum", descending = True)) print(tcr)  Desired output is below, but you'll see that in the bottom of the code I have run a functools.reduce on four dataframes, outside of the polars ecosystem, to join them, and it's basically this reduce that I want to put into the polars method chain somehow. [As an aside, my multiple (textX).fill_null(0) are also a bit clumsy but I'll leave that for a separate question] In [59]: %run so_question.py shape: (45, 7) ┌───────────┬────────────┬───────┬───────┬───────┬───────┬─────┐ │ publisher ┆ date ┆ text1 ┆ text2 ┆ text3 ┆ text4 ┆ sum │ │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ │ str ┆ date ┆ u64 ┆ u64 ┆ u64 ┆ u64 ┆ u64 │ ╞═══════════╪════════════╪═══════╪═══════╪═══════╪═══════╪═════╡ │ desob ┆ 2024-02-10 ┆ 5 ┆ 5 ┆ 5 ┆ 5 ┆ 20 │ │ qopir ┆ 2024-02-03 ┆ 5 ┆ 5 ┆ 5 ┆ 4 ┆ 19 │ │ jerag ┆ 2024-02-04 ┆ 5 ┆ 5 ┆ 5 ┆ 4 ┆ 19 │ │ jerag ┆ 2024-02-07 ┆ 5 ┆ 4 ┆ 5 ┆ 5 ┆ 19 │ │ wopav ┆ 2024-02-07 ┆ 4 ┆ 5 ┆ 3 ┆ 5 ┆ 17 │ │ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │ │ jerag ┆ 2024-02-06 ┆ 1 ┆ null ┆ 1 ┆ 1 ┆ 3 │ │ desob ┆ 2024-02-05 ┆ 1 ┆ 1 ┆ null ┆ 1 ┆ 3 │ │ cufeg ┆ 2024-02-04 ┆ 1 ┆ 1 ┆ 1 ┆ null ┆ 3 │ │ cufeg ┆ 2024-02-05 ┆ 1 ┆ null ┆ 1 ┆ 1 ┆ 3 │ │ wopav ┆ 2024-02-06 ┆ null ┆ 1 ┆ 1 ┆ 1 ┆ 3 │ └───────────┴────────────┴───────┴───────┴───────┴───────┴─────┘  So basically, tag counts by columns ["text1", "text2", "text3", "text4"], then summed ignoring nulls, and sorted descending on the sum. Join should be on publisher and date, outer (= "full"), and coalescing.

Answer: Is it not the same as aggregating the list of col() names at the same time? fieldnames = ["text1", "text2", "text3", "text4"] (df.group_by("publisher", "date") .agg(pl.col(fieldnames).str.contains_any(html_tags).sum()) .with_columns(sum = pl.sum_horizontal(fieldnames)) )  shape: (45, 7) ┌───────────┬────────────┬───────┬───────┬───────┬───────┬─────┐ │ publisher ┆ date ┆ text1 ┆ text2 ┆ text3 ┆ text4 ┆ sum │ │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ │ str ┆ date ┆ u32 ┆ u32 ┆ u32 ┆ u32 ┆ u32 │ ╞═══════════╪════════════╪═══════╪═══════╪═══════╪═══════╪═════╡ │ desob ┆ 2024-02-01 ┆ 2 ┆ 2 ┆ 2 ┆ 2 ┆ 8 │ │ xikoy ┆ 2024-02-06 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 4 │ │ wopav ┆ 2024-02-03 ┆ 2 ┆ 2 ┆ 2 ┆ 2 ┆ 8 │ │ jerag ┆ 2024-02-05 ┆ 3 ┆ 2 ┆ 3 ┆ 3 ┆ 11 │ │ qopir ┆ 2024-02-03 ┆ 5 ┆ 5 ┆ 5 ┆ 4 ┆ 19 │ │ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │ │ xikoy ┆ 2024-02-10 ┆ 2 ┆ 2 ┆ 2 ┆ 2 ┆ 8 │ │ xikoy ┆ 2024-02-02 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 4 │ │ cufeg ┆ 2024-02-10 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 4 │ │ jerag ┆ 2024-02-06 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 3 │ │ desob ┆ 2024-02-03 ┆ 2 ┆ 2 ┆ 2 ┆ 2 ┆ 8 │ └───────────┴────────────┴───────┴───────┴───────┴───────┴─────┘  pl.sum_horizontal() replaces the multiple fill_null / addition combination.