Question: Date Validation in Pyspark

I have a data file having multiple date fields coming in string data type. I am trying to validate the date field and discard the records having wrong date format. Data looks like below.
schema = StructType([StructField("id",StringType(),True), \
StructField("dt1",StringType(),True), \
StructField("dt2",StringType(),True)])
df = spark.createDataFrame([(1, "01/22/2010","03/25/2012"), (2, "01/12/2014",None),(3,"04/09/2011","12/23"),(5,None,"01/22/2010"),(6,"2005/12/04","2000/12/04"),(7,"01/01/2020","30/12/2019"),(8,"12/1999/21","05/01/2021"),(9,"12/2013/21",None),(9,None,None)], schema=schema)

Only the two date formats "M/d/y", "M/yy" are allowed to be passed in the validation. Records that do not pass the validation will be loaded in the error table.

This is a sample data. In actual file there are n numbers of date fields present. I am trying to get a function which can do this validation on all the date fields.


Answer:
I think you can try to parse the date and then filter rows with None values, you can have a list of date columns to be more generic
from pyspark.sql.functions import coalesce, to_date, col
from pyspark.sql.types import StructField, StructType, StringType

schema = StructType([StructField("id",StringType(),True), \
StructField("dt1",StringType(),True), \
StructField("dt2",StringType(),True)])
df = spark.createDataFrame([(1, "01/22/2010","03/25/2012"), (2, "01/12/2014",None),(3,"04/09/2011","12/23"),(5,None,"01/22/2010"),(6,"2005/12/04","2000/12/04"),(7,"01/01/2020","30/12/2019"),(8,"12/1999/21","05/01/2021"),(9,"12/2013/21",None),(9,None,None)], schema=schema)


def custom_to_date(col):
    formats = ("M/d/y", "M/yy")
    return coalesce(*[to_date(col, f) for f in formats])

# Set CORRECTED mode to deal with invalid dates as None
spark.sql("set spark.sql.legacy.timeParserPolicy=CORRECTED")

df.cache()

date_columns = ["dt1", "dt2"]

valid_df = df
for c in date_columns:
    valid_df = valid_df.filter(custom_to_date(col(c)).isNotNull())

error_df = df.subtract(valid_df)

error_df.show()
+---+----------+----------+
| id|       dt1|       dt2|
+---+----------+----------+
|  2|01/12/2014|      null|
|  5|      null|01/22/2010|
|  6|2005/12/04|2000/12/04|
|  7|01/01/2020|30/12/2019|
|  8|12/1999/21|05/01/2021|
|  9|12/2013/21|      null|
|  9|      null|      null|
+---+----------+----------+

valid_df.show()
+---+----------+----------+                                                     
| id|       dt1|       dt2|
+---+----------+----------+
|  1|01/22/2010|03/25/2012|
|  3|04/09/2011|     12/23|
+---+----------+----------+

Update: If we need to replace the String columns as well with the parsed dates then we can do something like this:
from pyspark.sql.functions import coalesce, to_date, col
from pyspark.sql.types import StructField, StructType, StringType

schema = StructType([StructField("id",StringType(),True), \
StructField("dt1",StringType(),True), \
StructField("dt2",StringType(),True)])
df = spark.createDataFrame([(1, "01/22/2010","03/25/2012"), (2, "01/12/2014",None),(3,"04/09/2011","12/23"),(5,None,"01/22/2010"),(6,"2005/12/04","2000/12/04"),(7,"01/01/2020","30/12/2019"),(8,"12/1999/21","05/01/2021"),(9,"12/2013/21",None),(9,None,None)], schema=schema)


def custom_to_date(col):
    formats = ("M/d/y", "M/yy")
    return coalesce(*[to_date(col, f) for f in formats])

# Set CORRECTED mode to deal with invalid dates as None
spark.sql("set spark.sql.legacy.timeParserPolicy=CORRECTED")

df.cache()

date_columns = ["dt1", "dt2"]


for c in date_columns:
    df = df.withColumn(c, custom_to_date(col(c)))

valid_df = df
for c in date_columns:
    valid_df = valid_df.filter(col(c).isNotNull())

error_df = df.subtract(valid_df)

>>> valid_df.show()
+---+----------+----------+
| id|       dt1|       dt2|
+---+----------+----------+
|  1|2010-01-22|2012-03-25|
|  3|2011-04-09|2023-12-01|
+---+----------+----------+

>>> error_df.show()
+---+----------+----------+
| id|       dt1|       dt2|
+---+----------+----------+
|  2|2014-01-12|      null|
|  5|      null|2010-01-22|
|  6|      null|      null|
|  7|2020-01-01|      null|
|  8|      null|2021-05-01|
|  9|      null|      null|
+---+----------+----------+