facebook/presto — pull request #944 by dain (2014-01-04 20:59:09+00:00)
Title: TPCH plugin

Add a plugin based on the airlift TPCH data generator. This also converts test and benchmarks to use generated data instead of static files.

For benchmarks this code replaces TpchBlocksProvider with CREATE TABLE commands using LocalQueryRunner. The means that benchmarks now use the native plugin which adds about 1-3ms to the benchmarks times. For super fast benchmarks (e.g., count_agg and raw_stream), this appears as a huge performance drop.

Before:
```
count_agg :: 0.697 cpu ms :: in 1.5M, 12.9MB, 2.15B/s, 18GB/s :: out 1, 9B, 1.43K/s, 12.6KB/s
double_sum_agg :: 10.741 cpu ms :: in 1.5M, 12.9MB, 140M/s, 1.17GB/s :: out 1, 9B, 93/s, 837B/s
hash_agg :: 137.724 cpu ms :: in 1.5M, 21.5MB, 10.9M/s, 156MB/s :: out 3, 45B, 21/s, 326B/s
predicate_filter :: 55.698 cpu ms :: in 1.5M, 12.9MB, 26.9M/s, 231MB/s :: out 1.29M, 11.1MB, 23.1M/s, 199MB/s
raw_stream :: 0.810 cpu ms :: in 1.5M, 12.9MB, 1.85B/s, 15.5GB/s :: out 1.5M, 12.9MB, 1.85B/s, 15.5GB/s
top100 :: 41.238 cpu ms :: in 1.5M, 12.9MB, 36.4M/s, 312MB/s :: out 100, 900B, 2.42K/s, 21.3KB/s
in_memory_orderby_1.5M :: 1521.210 cpu ms :: in 1.5M, 41.5MB, 986K/s, 27.3MB/s :: out 1.5M, 28.6MB, 986K/s, 18.8MB/s
hash_build :: 529.348 cpu ms :: in 1.5M, 25.7MB, 2.83M/s, 48.6MB/s :: out 1.5M, 25.7MB, 2.83M/s, 48.6MB/s
hash_join :: 1643.575 cpu ms :: in 6M, 103MB, 3.65M/s, 62.7MB/s :: out 6M, 206MB, 3.65M/s, 125MB/s
hash_build_and_join :: 2215.610 cpu ms :: in 7.5M, 129MB, 3.39M/s, 58.1MB/s :: out 6M, 206MB, 2.71M/s, 93MB/s
hand_tpch_query_1 :: 2450.150 cpu ms :: in 6M, 361MB, 2.45M/s, 147MB/s :: out 4, 300B, 1/s, 122B/s
hand_tpch_query_6 :: 289.359 cpu ms :: in 6M, 240MB, 20.7M/s, 831MB/s :: out 1, 9B, 3/s, 31B/s
sql_groupby_agg_with_arithmetic :: 891.984 cpu ms :: in 6M, 137MB, 6.73M/s, 154MB/s :: out 2, 30B, 2/s, 33B/s
sql_count_agg :: 1.578 cpu ms :: in 1.5M, 12.9MB, 950M/s, 7.97GB/s :: out 1, 9B, 633/s, 5.57KB/s
sql_double_sum_agg :: 12.196 cpu ms :: in 1.5M, 12.9MB, 123M/s, 1.03GB/s :: out 1, 9B, 81/s, 737B/s
sql_count_with_filter :: 61.586 cpu ms :: in 1.5M, 8.58MB, 24.4M/s, 139MB/s :: out 1, 9B, 16/s, 146B/s
sql_groupby_agg :: 139.857 cpu ms :: in 1.5M, 21.5MB, 10.7M/s, 153MB/s :: out 3, 45B, 21/s, 321B/s
sql_predicate_filter :: 37.860 cpu ms :: in 1.5M, 12.9MB, 39.6M/s, 340MB/s :: out 1.29M, 11.1MB, 34M/s, 292MB/s
sql_raw_stream :: 1.508 cpu ms :: in 1.5M, 12.9MB, 995M/s, 8.34GB/s :: out 1.5M, 12.9MB, 995M/s, 8.34GB/s
sql_top_100 :: 42.490 cpu ms :: in 1.5M, 12.9MB, 35.3M/s, 303MB/s :: out 100, 900B, 2.35K/s, 20.7KB/s
sql_hash_join :: 3281.675 cpu ms :: in 7.5M, 129MB, 2.29M/s, 39.2MB/s :: out 6M, 206MB, 1.83M/s, 62.8MB/s
sql_join_with_predicate :: 787.630 cpu ms :: in 7.5M, 116MB, 9.52M/s, 147MB/s :: out 1, 9B, 1/s, 11B/s
sql_varbinary_max :: 182.818 cpu ms :: in 6M, 97.3MB, 32.8M/s, 532MB/s :: out 1, 21B, 5/s, 114B/s
sql_distinct_multi :: 199.495 cpu ms :: in 1.5M, 32MB, 7.52M/s, 161MB/s :: out 5, 112B, 25/s, 561B/s
sql_distinct_single :: 93.074 cpu ms :: in 1.5M, 12.9MB, 16.1M/s, 138MB/s :: out 1, 9B, 10/s, 96B/s
sql_tpch_query_1 :: 2503.541 cpu ms :: in 6M, 361MB, 2.4M/s, 144MB/s :: out 4, 336B, 1/s, 134B/s
sql_tpch_query_6 :: 268.098 cpu ms :: in 6M, 240MB, 22.4M/s, 897MB/s :: out 1, 9B, 3/s, 33B/s
sql_like :: 2812.936 cpu ms :: in 6M, 232MB, 2.13M/s, 82.4MB/s :: out 1.15M, 9.84MB, 408K/s, 3.5MB/s
sql_in :: 67.976 cpu ms :: in 6M, 51.5MB, 88.3M/s, 758MB/s :: out 25, 225B, 367/s, 3.23KB/s
sql_semijoin_in :: 1660.280 cpu ms :: in 7.5M, 64.4MB, 4.52M/s, 38.8MB/s :: out 3M, 25.8MB, 1.81M/s, 15.5MB/s
sql_regexp_like :: 1853.437 cpu ms :: in 1.5M, 76.6MB, 809K/s, 41.3MB/s :: out 1, 9B, 0/s, 4B/s
sql_approx_percentile_long :: 559.526 cpu ms :: in 1.5M, 12.9MB, 2.68M/s, 23MB/s :: out 1, 9B, 1/s, 16B/s
stat_long_variance :: 21.974 cpu ms :: in 1.5M, 12.9MB, 68.3M/s, 586MB/s :: out 1, 9B, 45/s, 409B/s
stat_long_variance_pop :: 21.928 cpu ms :: in 1.5M, 12.9MB, 68.4M/s, 587MB/s :: out 1, 9B, 45/s, 410B/s
stat_double_variance :: 19.587 cpu ms :: in 1.5M, 12.9MB, 76.6M/s, 657MB/s :: out 1, 9B, 51/s, 459B/s
stat_double_variance_pop :: 19.382 cpu ms :: in 1.5M, 12.9MB, 77.4M/s, 664MB/s :: out 1, 9B, 51/s, 464B/s
stat_long_stddev :: 22.252 cpu ms :: in 1.5M, 12.9MB, 67.4M/s, 579MB/s :: out 1, 9B, 44/s, 404B/s
stat_long_stddev_pop :: 22.540 cpu ms :: in 1.5M, 12.9MB, 66.5M/s, 571MB/s :: out 1, 9B, 44/s, 399B/s
stat_double_stddev :: 19.305 cpu ms :: in 1.5M, 12.9MB, 77.7M/s, 667MB/s :: out 1, 9B, 51/s, 466B/s
stat_double_stddev_pop :: 19.459 cpu ms :: in 1.5M, 12.9MB, 77.1M/s, 662MB/s :: out 1, 9B, 51/s, 462B/s
sql_approx_count_distinct_long :: 128.087 cpu ms :: in 1.5M, 12.9MB, 11.7M/s, 101MB/s :: out 1, 9B, 7/s, 70B/s
sql_approx_count_distinct_double :: 134.577 cpu ms :: in 1.5M, 12.9MB, 11.1M/s, 95.7MB/s :: out 1, 9B, 7/s, 66B/s
sql_approx_count_distinct_varbinary :: 227.544 cpu ms :: in 1.5M, 21.5MB, 6.59M/s, 94.3MB/s :: out 1, 9B, 4/s, 39B/s
```

After:
```
count_agg :: 1.781 cpu ms :: in 1.5M, 12.9MB, 842M/s, 7.06GB/s :: out 1, 9B, 561/s, 4.93KB/s
double_sum_agg :: 12.181 cpu ms :: in 1.5M, 12.9MB, 123M/s, 1.03GB/s :: out 1, 9B, 82/s, 738B/s
hash_agg :: 141.757 cpu ms :: in 1.5M, 21.5MB, 10.6M/s, 151MB/s :: out 3, 45B, 21/s, 317B/s
predicate_filter :: 58.120 cpu ms :: in 1.5M, 12.9MB, 25.8M/s, 222MB/s :: out 1.29M, 11.1MB, 22.2M/s, 190MB/s
raw_stream :: 1.696 cpu ms :: in 1.5M, 12.9MB, 885M/s, 7.41GB/s :: out 1.5M, 12.9MB, 885M/s, 7.41GB/s
top100 :: 45.253 cpu ms :: in 1.5M, 12.9MB, 33.1M/s, 285MB/s :: out 100, 900B, 2.21K/s, 19.4KB/s
in_memory_orderby_1.5M :: 1531.159 cpu ms :: in 1.5M, 41.5MB, 980K/s, 27.1MB/s :: out 1.5M, 28.6MB, 980K/s, 18.7MB/s
hash_build :: 541.420 cpu ms :: in 1.5M, 25.7MB, 2.77M/s, 47.6MB/s :: out 1.5M, 25.7MB, 2.77M/s, 47.6MB/s
hash_join :: 1673.213 cpu ms :: in 6M, 103MB, 3.59M/s, 61.6MB/s :: out 6M, 206MB, 3.59M/s, 123MB/s
hash_build_and_join :: 2204.910 cpu ms :: in 7.5M, 129MB, 3.4M/s, 58.4MB/s :: out 6M, 206MB, 2.72M/s, 93.4MB/s
hand_tpch_query_1 :: 2464.116 cpu ms :: in 6M, 361MB, 2.44M/s, 146MB/s :: out 4, 300B, 1/s, 121B/s
hand_tpch_query_6 :: 293.306 cpu ms :: in 6M, 240MB, 20.5M/s, 820MB/s :: out 1, 9B, 3/s, 30B/s
sql_groupby_agg_with_arithmetic :: 904.943 cpu ms :: in 6M, 137MB, 6.63M/s, 152MB/s :: out 2, 30B, 2/s, 33B/s
sql_count_agg :: 3.073 cpu ms :: in 1.5M, 12.9MB, 488M/s, 4.09GB/s :: out 1, 9B, 325/s, 2.86KB/s
sql_double_sum_agg :: 14.070 cpu ms :: in 1.5M, 12.9MB, 107M/s, 915MB/s :: out 1, 9B, 71/s, 639B/s
sql_count_with_filter :: 64.687 cpu ms :: in 1.5M, 8.58MB, 23.2M/s, 133MB/s :: out 1, 9B, 15/s, 139B/s
sql_groupby_agg :: 142.734 cpu ms :: in 1.5M, 21.5MB, 10.5M/s, 150MB/s :: out 3, 45B, 21/s, 315B/s
sql_predicate_filter :: 39.810 cpu ms :: in 1.5M, 12.9MB, 37.7M/s, 323MB/s :: out 1.29M, 11.1MB, 32.4M/s, 278MB/s
sql_raw_stream :: 3.085 cpu ms :: in 1.5M, 12.9MB, 486M/s, 4.08GB/s :: out 1.5M, 12.9MB, 486M/s, 4.08GB/s
sql_top_100 :: 44.824 cpu ms :: in 1.5M, 12.9MB, 33.5M/s, 287MB/s :: out 100, 900B, 2.23K/s, 19.6KB/s
sql_hash_join :: 3308.330 cpu ms :: in 7.5M, 129MB, 2.27M/s, 38.9MB/s :: out 6M, 206MB, 1.81M/s, 62.3MB/s
sql_join_with_predicate :: 794.320 cpu ms :: in 7.5M, 116MB, 9.44M/s, 146MB/s :: out 1, 9B, 1/s, 11B/s
sql_varbinary_max :: 184.251 cpu ms :: in 6M, 97.3MB, 32.6M/s, 528MB/s :: out 1, 21B, 5/s, 113B/s
sql_distinct_multi :: 213.832 cpu ms :: in 1.5M, 32MB, 7.01M/s, 150MB/s :: out 5, 112B, 23/s, 523B/s
sql_distinct_single :: 94.300 cpu ms :: in 1.5M, 12.9MB, 15.9M/s, 137MB/s :: out 1, 9B, 10/s, 95B/s
sql_tpch_query_1 :: 2532.160 cpu ms :: in 6M, 361MB, 2.37M/s, 142MB/s :: out 4, 336B, 1/s, 132B/s
sql_tpch_query_6 :: 248.466 cpu ms :: in 6M, 240MB, 24.2M/s, 967MB/s :: out 1, 9B, 4/s, 36B/s
sql_like :: 2732.147 cpu ms :: in 6M, 232MB, 2.2M/s, 84.8MB/s :: out 1.15M, 9.84MB, 420K/s, 3.6MB/s
sql_in :: 66.849 cpu ms :: in 6M, 51.5MB, 89.8M/s, 771MB/s :: out 25, 225B, 373/s, 3.29KB/s
sql_semijoin_in :: 1675.488 cpu ms :: in 7.5M, 64.4MB, 4.48M/s, 38.4MB/s :: out 3M, 25.8MB, 1.79M/s, 15.4MB/s
sql_regexp_like :: 1891.597 cpu ms :: in 1.5M, 76.6MB, 793K/s, 40.5MB/s :: out 1, 9B, 0/s, 4B/s
sql_approx_percentile_long :: 560.500 cpu ms :: in 1.5M, 12.9MB, 2.68M/s, 23MB/s :: out 1, 9B, 1/s, 16B/s
stat_long_variance :: 24.178 cpu ms :: in 1.5M, 12.9MB, 62M/s, 532MB/s :: out 1, 9B, 41/s, 372B/s
stat_long_variance_pop :: 23.890 cpu ms :: in 1.5M, 12.9MB, 62.8M/s, 539MB/s :: out 1, 9B, 41/s, 376B/s
stat_double_variance :: 21.324 cpu ms :: in 1.5M, 12.9MB, 70.3M/s, 604MB/s :: out 1, 9B, 46/s, 422B/s
stat_double_variance_pop :: 21.284 cpu ms :: in 1.5M, 12.9MB, 70.5M/s, 605MB/s :: out 1, 9B, 46/s, 422B/s
stat_long_stddev :: 23.768 cpu ms :: in 1.5M, 12.9MB, 63.1M/s, 542MB/s :: out 1, 9B, 42/s, 378B/s
stat_long_stddev_pop :: 23.717 cpu ms :: in 1.5M, 12.9MB, 63.2M/s, 543MB/s :: out 1, 9B, 42/s, 379B/s
stat_double_stddev :: 21.165 cpu ms :: in 1.5M, 12.9MB, 70.9M/s, 608MB/s :: out 1, 9B, 47/s, 425B/s
stat_double_stddev_pop :: 21.314 cpu ms :: in 1.5M, 12.9MB, 70.4M/s, 604MB/s :: out 1, 9B, 46/s, 422B/s
sql_approx_count_distinct_long :: 132.816 cpu ms :: in 1.5M, 12.9MB, 11.3M/s, 96.9MB/s :: out 1, 9B, 7/s, 67B/s
sql_approx_count_distinct_double :: 136.426 cpu ms :: in 1.5M, 12.9MB, 11M/s, 94.4MB/s :: out 1, 9B, 7/s, 65B/s
sql_approx_count_distinct_varbinary :: 218.193 cpu ms :: in 1.5M, 21.5MB, 6.87M/s, 98.3MB/s :: out 1, 9B, 4/s, 41B/s
```