Project: ClickHouse (App Definition and Development / Database)
File: brown-benchmark.md

slug: /en/getting-started/example-datasets/brown-benchmark sidebar_label: Brown University Benchmark description: A new analytical benchmark for machine-generated log data title: "Brown University Benchmark" `MgBench` is a new analytical benchmark for machine-generated log data, . Download the data: ``` wget https://datasets.clickhouse.com/mgbench{1..3}.csv.xz ``` Unpack the data: ``` xz -v -d mgbench{1..3}.csv.xz ``` Create the database and tables: ```sql CREATE DATABASE mgbench; ``` ```sql USE mgbench; ``` ```sql CREATE TABLE mgbench.logs1 ( log_time DateTime, machine_name LowCardinality(String), machine_group LowCardinality(String), cpu_idle Nullable(Float32), cpu_nice Nullable(Float32), cpu_system Nullable(Float32), cpu_user Nullable(Float32), cpu_wio Nullable(Float32), disk_free Nullable(Float32), disk_total Nullable(Float32), partmaxused Nullable(Float32), load_fifteen Nullable(Float32), load_five Nullable(Float32), load_one Nullable(Float32), mem_buffers Nullable(Float32), mem_cached Nullable(Float32), mem_free Nullable(Float32), mem_shared Nullable(Float32), swap_free Nullable(Float32), bytes_in Nullable(Float32), bytes_out Nullable(Float32) ) ENGINE = MergeTree() ORDER BY (machinegroup, machinename, log_time); ``` ```sql CREATE TABLE mgbench.logs2 ( log_time DateTime, client_ip IPv4, request String, status_code UInt16, object_size UInt64 ) ENGINE = MergeTree() ORDER BY log_time; ``` ```sql CREATE TABLE mgbench.logs3 ( log_time DateTime64, device_id FixedString(15), device_name LowCardinality(String), device_type LowCardinality(String), device_floor UInt8, event_type LowCardinality(String), event_unit FixedString(1), event_value Nullable(Float32) ) ENGINE = MergeTree() ORDER BY (eventtype, logtime); ``` Insert data: ``` clickhouse-client --query "INSERT INTO mgbench.logs1 FORMAT CSVWithNames" < mgbench1.csv clickhouse-client --query "INSERT INTO mgbench.logs2 FORMAT CSVWithNames" < mgbench2.csv clickhouse-client --query "INSERT INTO mgbench.logs3 FORMAT CSVWithNames" < mgbench3.csv ``` ```sql USE mgbench; ``` ```sql -- Q1.1: What is the CPU/network utilization for each web server since midnight? SELECT machine_name, MIN(cpu) AS cpu_min, MAX(cpu) AS cpu_max, AVG(cpu) AS cpu_avg, MIN(netin) AS netin_min, MAX(netin) AS netin_max, AVG(netin) AS netin_avg, MIN(netout) AS netout_min, MAX(netout) AS netout_max, AVG(netout) AS netout_avg FROM ( SELECT machine_name, COALESCE(cpu_user, 0.0) AS cpu, COALESCE(bytesin, 0.0) AS netin, COALESCE(bytesout, 0.0) AS netout FROM logs1 WHERE machine_name IN ('anansi','aragog','urd') AND log_time >= TIMESTAMP '2017-01-11 00:00:00' ) AS r GROUP BY machine_name; ``` ```sql -- Q1.2: Which computer lab machines have been offline in the past day? SELECT machine_name, log_time FROM logs1 WHERE (machine_name LIKE 'cslab%' OR machine_name LIKE 'mslab%') AND load_one IS NULL AND log_time >= TIMESTAMP '2017-01-10 00:00:00' ORDER BY machine_name, log_time; ``` ```sql --

What are the hourly average metrics during the past 10 days for a specific workstation? SELECT dt, hr, AVG(loadfifteen) AS loadfifteen_avg, AVG(loadfive) AS loadfive_avg, AVG(loadone) AS loadone_avg, AVG(memfree) AS memfree_avg, AVG(swapfree) AS swapfree_avg FROM ( SELECT CAST(log_time AS DATE) AS dt, EXTRACT(HOUR FROM log_time) AS hr, load_fifteen, load_five, load_one, mem_free, swap_free FROM logs1 WHERE machine_name = 'babbage' AND load_fifteen IS NOT NULL AND load_five IS NOT NULL AND load_one IS NOT NULL AND mem_free IS NOT NULL AND swap_free IS NOT NULL AND log_time >= TIMESTAMP '2017-01-01 00:00:00' ) AS r GROUP BY dt, hr ORDER BY dt, hr; ``` ```sql -- Q1.4: Over 1 month, how often was each server blocked on disk I/O? SELECT machine_name, COUNT(*) AS spikes FROM logs1 WHERE machine_group = 'Servers' AND cpu_wio > 0.99 AND log_time >= TIMESTAMP '2016-12-01 00:00:00' AND log_time < TIMESTAMP '2017-01-01 00:00:00' GROUP BY machine_name ORDER BY spikes DESC LIMIT 10; ``` ```sql -- Q1.5: Which externally reachable VMs have run low on memory? SELECT machine_name, dt, MIN(memfree) AS memfree_min FROM ( SELECT machine_name, CAST(log_time AS DATE) AS dt, mem_free FROM logs1 WHERE machine_group = 'DMZ' AND mem_free IS NOT NULL ) AS r GROUP BY machine_name, dt HAVING MIN(mem_free) < 10000 ORDER BY machine_name, dt; ``` ```sql -- Q1.6: What is the total hourly network traffic across all file servers? SELECT dt, hr, SUM(netin) AS netin_sum, SUM(netout) AS netout_sum, SUM(netin) + SUM(netout) AS both_sum FROM ( SELECT CAST(log_time AS DATE) AS dt, EXTRACT(HOUR FROM log_time) AS hr, COALESCE(bytesin, 0.0) / 1000000000.0 AS netin, COALESCE(bytesout, 0.0) / 1000000000.0 AS netout FROM logs1 WHERE machine_name IN ('allsorts','andes','bigred','blackjack','bonbon', 'cadbury','chiclets','cotton','crows','dove','fireball','hearts','huey', 'lindt','milkduds','milkyway','mnm','necco','nerds','orbit','peeps', 'poprocks','razzles','runts','smarties','smuggler','spree','stride', 'tootsie','trident','wrigley','york') ) AS r GROUP BY dt, hr ORDER BY both_sum DESC LIMIT 10; ``` ```sql -- Q2.1: Which requests have caused server errors within the past 2 weeks? SELECT * FROM logs2 WHERE status_code >= 500 AND log_time >= TIMESTAMP '2012-12-18 00:00:00' ORDER BY log_time; ``` ```sql -- Q2.2: During a specific 2-week period, was the user password file leaked? SELECT * FROM logs2 WHERE status_code >= 200 AND status_code < 300 AND request LIKE '%/etc/passwd%' AND log_time >= TIMESTAMP '2012-05-06 00:00:00' AND log_time < TIMESTAMP '2012-05-20 00:00:00'; ``` ```sql -- Q2.3: What was the average path depth for top-level requests in the past month? SELECT top_level, AVG(LENGTH(request) - LENGTH(REPLACE(request, '/', ''))) AS depth_avg FROM ( SELECT SUBSTRING(request FROM 1 FOR len) AS top_level, request FROM ( SELECT POSITION(SUBSTRING(request FROM 2), '/') AS len, request FROM logs2 WHERE status_code >= 200 AND status_code < 300 AND log_time >= TIMESTAMP '2012-12-01 00:00:00' ) AS r WHERE len > 0 ) AS s WHERE top_level IN ('/about','/courses','/degrees','/events', '/grad','/industry','/news','/people', '/publications','/research','/teaching','/ugrad') GROUP BY top_level ORDER BY top_level; ``` ```sql -- Q2.4: During the last 3 months, which clients have made an excessive number of requests? SELECT client_ip, COUNT(*) AS num_requests FROM logs2 WHERE log_time >= TIMESTAMP '2012-10-01 00:00:00' GROUP BY client_ip HAVING COUNT(*) >= 100000 ORDER BY num_requests DESC; ``` ```sql -- Q2.5: What are the daily unique visitors? SELECT dt, COUNT(DISTINCT client_ip) FROM ( SELECT CAST(log_time AS DATE) AS dt, client_ip FROM logs2 ) AS r GROUP BY dt ORDER BY dt; ``` ```sql --

What are the average and maximum data transfer rates (Gbps)? SELECT AVG(transfer) / 125000000.0 AS transfer_avg, MAX(transfer) / 125000000.0 AS transfer_max FROM ( SELECT log_time, SUM(object_size) AS transfer FROM logs2 GROUP BY log_time ) AS r; ``` ```sql -- Q3.1: Did the indoor temperature reach freezing over the weekend? SELECT * FROM logs3 WHERE event_type = 'temperature' AND event_value <= 32.0 AND log_time >= '2019-11-29 17:00:00.000'; ``` ```sql -- Q3.4: Over the past 6 months, how frequently were each door opened? SELECT device_name, device_floor, COUNT(*) AS ct FROM logs3 WHERE eventtype = 'dooropen' AND log_time >= '2019-06-01 00:00:00.000' GROUP BY device_name, device_floor ORDER BY ct DESC; ``` Query 3.5 below uses a UNION. Set the mode for combining SELECT query results. The setting is only used when shared with UNION without explicitly specifying the UNION ALL or UNION DISTINCT. ```sql SET uniondefaultmode = 'DISTINCT' ``` ```sql -- Q3.5: Where in the building do large temperature variations occur in winter and summer? WITH temperature AS ( SELECT dt, device_name, device_type, device_floor FROM ( SELECT dt, hr, device_name, device_type, device_floor, AVG(eventvalue) AS temperaturehourly_avg FROM ( SELECT CAST(log_time AS DATE) AS dt, EXTRACT(HOUR FROM log_time) AS hr, device_name, device_type, device_floor, event_value FROM logs3 WHERE event_type = 'temperature' ) AS r GROUP BY dt, hr, device_name, device_type, device_floor ) AS s GROUP BY dt, device_name, device_type, device_floor HAVING MAX(temperaturehourlyavg) - MIN(temperaturehourlyavg) >= 25.0 ) SELECT DISTINCT device_name, device_type, device_floor, 'WINTER' FROM temperature WHERE dt >= DATE '2018-12-01' AND dt < DATE '2019-03-01' UNION SELECT DISTINCT device_name, device_type, device_floor, 'SUMMER' FROM temperature WHERE dt >= DATE '2019-06-01' AND dt < DATE '2019-09-01'; ``` ```sql -- Q3.6: For each device category, what are the monthly power consumption metrics? SELECT yr, mo, SUM(coffeehourlyavg) AS coffeemonthlysum, AVG(coffeehourlyavg) AS coffeemonthlyavg, SUM(printerhourlyavg) AS printermonthlysum, AVG(printerhourlyavg) AS printermonthlyavg, SUM(projectorhourlyavg) AS projectormonthlysum, AVG(projectorhourlyavg) AS projectormonthlyavg, SUM(vendinghourlyavg) AS vendingmonthlysum, AVG(vendinghourlyavg) AS vendingmonthlyavg FROM ( SELECT dt, yr, mo, hr, AVG(coffee) AS coffeehourlyavg, AVG(printer) AS printerhourlyavg, AVG(projector) AS projectorhourlyavg, AVG(vending) AS vendinghourlyavg FROM ( SELECT CAST(log_time AS DATE) AS dt, EXTRACT(YEAR FROM log_time) AS yr, EXTRACT(MONTH FROM log_time) AS mo, EXTRACT(HOUR FROM log_time) AS hr, CASE WHEN devicename LIKE 'coffee%' THEN eventvalue END AS coffee, CASE WHEN devicename LIKE 'printer%' THEN eventvalue END AS printer, CASE WHEN devicename LIKE 'projector%' THEN eventvalue END AS projector, CASE WHEN devicename LIKE 'vending%' THEN eventvalue END AS vending FROM logs3 WHERE device_type = 'meter' ) AS r GROUP BY dt, yr, mo, hr ) AS s GROUP BY yr, mo ORDER BY yr, mo; ``` The data is also available for interactive queries in the , .