Bug: Same SQL on CSV and on Parquet gives different result
Status: Resolved | Priority: Major | Resolution: Not A Problem
Created: 23/Feb/20 18:18
Affects Version/s: 2.4.4

SO I played around with a data set from here: [https://www.kaggle.com/hmavrodiev/sofia-air-quality-dataset]

I ran the same query for the base CSVs and against a parquet version of them:

{color:#008000}SELECT * FROM airQualityP WHERE P1 > 20{color}

Here is the csv code:

{color:#000080}import {color}{color:#660e7a}session{color}.{color:#660e7a}sqlContext{color}.implicits._

{color:#000080}val {color}df = {color:#660e7a}session{color}.read.option({color:#008000}"header"{color}, {color:#008000}"true"{color}).csv({color:#660e7a}originalDataset{color})

df.createTempView({color:#008000}"airQuality"{color})

{color:#000080}val {color}result = {color:#660e7a}session{color}.sql({color:#008000}"SELECT * FROM airQuality WHERE P1 > 20"{color})
 .map(ParticleAirQuality.{color:#660e7a}mappingFunction{color})

println(result.count())

 

Here is the parquet code:

 

{color:#000080}import {color}{color:#660e7a}session{color}.{color:#660e7a}sqlContext{color}.implicits._

{color:#000080}val {color}df = {color:#660e7a}session{color}.read.option({color:#008000}"header"{color}, {color:#008000}"true"{color}).parquet({color:#660e7a}bigParquetDataset{color})

df.createTempView({color:#008000}"airQualityP"{color})

{color:#000080}val {color}result = {color:#660e7a}session{color} .sql({color:#008000}"SELECT * FROM airQualityP WHERE P1 > 20"{color})
 .map(ParticleAirQuality.{color:#660e7a}namedMappingFunction{color})

println(result.count())

 

And this is how I transform the csv into parquets:

{color:#000080}import {color}{color:#660e7a}session{color}.{color:#660e7a}sqlContext{color}.implicits._

{color:#000080}val {color}df = {color:#660e7a}session{color}.read.option({color:#008000}"header"{color}, {color:#008000}"true"{color})
 .csv({color:#660e7a}originalDataset{color})
 .map(ParticleAirQuality.{color:#660e7a}mappingFunction{color})

df.write.parquet({color:#660e7a}bigParquetDataset{color})

 

These are the two mapping functions:

{color:#000080}val {color}{color:#660e7a}mappingFunction {color}= {
 r: Row => ParticleAirQuality(
 r.getString({color:#0000ff}1{color}),
 r.getString({color:#0000ff}2{color}),
 r.getString({color:#0000ff}3{color}),
 r.getString({color:#0000ff}4{color}),
 r.getString({color:#0000ff}5{color}),
 {
 {color:#000080}val {color}p1 = r.getString({color:#0000ff}6{color})
 {color:#000080}if{color}(p1 == {color:#000080}null{color}) Double.{color:#660e7a}NaN{color} {color:#000080}else {color}p1.toDouble
 },
 {
 {color:#000080}val {color}p2 = r.getString({color:#0000ff}7{color})
 {color:#000080}if{color}(p2 == {color:#000080}null{color}) Double.{color:#660e7a}NaN{color} {color:#000080}else {color}p2.toDouble
 }
 ) }

{color:#000080}val {color}{color:#660e7a}namedMappingFunction {color}= {
 r: Row => ParticleAirQuality(
 r.getAs[{color:#20999d}String{color}]({color:#008000}"sensor_id"{color}),
 r.getAs[{color:#20999d}String{color}]({color:#008000}"location"{color}),
 r.getAs[{color:#20999d}String{color}]({color:#008000}"lat"{color}),
 r.getAs[{color:#20999d}String{color}]({color:#008000}"lon"{color}),
 r.getAs[{color:#20999d}String{color}]({color:#008000}"timestamp"{color}),
 r.getAs[Double]({color:#008000}"P1"{color}),
 r.getAs[Double]({color:#008000}"P2"{color})
 )
 }

 

If it matters this is the paths (Note that I actually use double \ instead of / since it is windows - but that doesn't really matter):

{color:#000080}val {color}{color:#660e7a}originalDataset {color}= {color:#008000}"D:/source/datasets/sofia-air-quality-dataset/*{color}{color:#008000}*sds**.csv"{color}

{color:#000080}val {color}{color:#660e7a}bigParquetDataset {color}= {color:#008000}"D:/source/datasets/air-tests/all-parquet"{color}

 

The count from the csvs I get is: 33934609

While the count from the parquets is: 35739394