How to generate stale flag using analytic functions Pyspark? - function

Question

What is the efficient way to create below output table with minimal join and by using the analytic functions?
expiry_date - Date until the answer is valid.
stale_answer_flag - Flag on the child answer showing that it predates the parent answer
Input tables:
Question:
question_id
parent_question_id
question
1
Are you living in the US?
1A
1
What is your state
Answer:
question_id
date
answer
1
01-sept-2022
yes
1A
01-sept-2022
NY
1
05-sept-2022
yes
Expected Output table:
question_id
parent_question_id
question
answer
date
expiry_date
stale_ans_flag
1
Are you living in the US?
Yes
01-sept-2022
05-sept-2022
Y
1A
1
What is your state
NY
01-sept-2022
'NULL'
N
1
Are you living in the US?
Yes
05-sept-2022
'NULL'
N

Answer 1

A standard SCD2 can be done the following
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
from pyspark.sql import SQLContext
from pyspark.sql.types import *
from pyspark.sql.window import Window
import pyspark.sql.functions as f
# create some data, I added a user column to also add new entries
data_1 = [
("alex","1","2022-09-01","yes"),
("alex","1A","2022-09-01","NY"),
]
schema = ["user","id","date","answer"]
df_1 = spark.createDataFrame(data=data_1, schema = schema)
df_1.show(truncate=False)
+----+---+----------+------+
|user|id |date |answer|
+----+---+----------+------+
|alex|1 |2022-09-01|yes |
|alex|1A |2022-09-01|NY |
+----+---+----------+------+
data_2 = [
("alex","1","2022-09-05","no"),
("john","1","2022-09-05","yes"),
("john","1A","2022-09-05","VA")
]
schema = ["user","id","date","answer"]
df_2 = spark.createDataFrame(data=data_2, schema = schema)
df_2.show(truncate=False)
+----+---+----------+------+
|user|id |date |answer|
+----+---+----------+------+
|alex|1 |2022-09-05|no |
|john|1 |2022-09-05|yes |
|john|1A |2022-09-05|VA |
+----+---+----------+------+
# create the initial dataset of the "old" data with the two additional columns
df_old=(df_1
.withColumn("expiry_date",f.lit(None).cast(DateType()))
.withColumn("stale_ans_flag",f.lit(False))
)
df_old.show()
+----+---+----------+------+-----------+--------------+
|user| id| date|answer|expiry_date|stale_ans_flag|
+----+---+----------+------+-----------+--------------+
|alex| 1|2022-09-01| yes| null| false|
|alex| 1A|2022-09-01| NY| null| false|
+----+---+----------+------+-----------+--------------+
# create a new column on the new dataset that will update the old one
df_new=(df_2
.withColumn("expiry_date",f.lit(None).cast(DateType()))
.withColumn("stale_ans_flag",f.lit(False))
)
df_new.show()
+----+---+----------+------+-----------+--------------+
|user| id| date|answer|expiry_date|stale_ans_flag|
+----+---+----------+------+-----------+--------------+
|alex| 1|2022-09-05| no| null| false|
|john| 1|2022-09-05| yes| null| false|
|john| 1A|2022-09-05| VA| null| false|
+----+---+----------+------+-----------+--------------+
# now join both together on the relevant keys and add an helper column
df_merge=(df_old.alias("old")
# a full outer join makes it easy to compare new and old and create manual update strategies
.join(df_new.alias("new"),
(df_old.user == df_new.user) &
(df_old.id == df_new.id),
how='fullouter'
)
# helper columns, you can do it without but due to lazy evaluatiuon it does not matter but helps reading the code and debugging
.withColumn("_action",f.when(
# identify new records that have to be inserted
(f.col("old.user").isNull())
& (f.col("old.id").isNull())
& (f.col("new.user").isNotNull())
& (f.col("new.id").isNotNull())
, "new"
).when(
# identify old records that are not changed
(f.col("old.user").isNotNull())
& (f.col("old.id").isNotNull())
& (f.col("new.user").isNull())
& (f.col("new.id").isNull())
, "old"
).when(
# identify update records
(f.col("old.user").isNotNull())
& (f.col("old.id").isNotNull())
& (f.col("new.user").isNotNull())
& (f.col("new.id").isNotNull())
, "update"
)
)
)
df_merge.show()
+----+----+----------+------+-----------+--------------+----+----+----------+------+-----------+--------------+-------+
|user| id| date|answer|expiry_date|stale_ans_flag|user| id| date|answer|expiry_date|stale_ans_flag|_action|
+----+----+----------+------+-----------+--------------+----+----+----------+------+-----------+--------------+-------+
|null|null| null| null| null| null|john| 1A|2022-09-05| VA| null| false| new|
|alex| 1|2022-09-01| yes| null| false|alex| 1|2022-09-05| no| null| false| update|
|alex| 1A|2022-09-01| NY| null| false|null|null| null| null| null| null| old|
|null|null| null| null| null| null|john| 1|2022-09-05| yes| null| false| new|
+----+----+----------+------+-----------+--------------+----+----+----------+------+-----------+--------------+-------+
# finally put all together with a union
df_merge_union=( df_merge
#first take the old ones of the dataset that did not change and are kept as they are
.filter(f.col("_action")=="old")
.select("old.user",
"old.id",
"old.date",
"old.answer",
"old.expiry_date",
"old.stale_ans_flag"
)
# than add the brand new ones that did not exist before
.union(df_merge
.filter(f.col("_action")=="new")
.select("new.user",
"new.id",
"new.date",
"new.answer",
"new.expiry_date",
"new.stale_ans_flag"
)
)
# add the old row with the old values that has been updated and change the flag
.union(df_merge
.filter(f.col("_action")=="update")
.select("old.user",
"old.id",
"old.date",
"old.answer",
"new.date", # or f.current_date(),
f.lit(True)
)
)
# add the old row with the new values that has been updated
.union(df_merge
.filter(f.col("_action")=="update")
.select("new.user",
"new.id",
"new.date",
"new.answer",
f.lit(None),
f.lit(False)
)
)
)
df_merge_union.sort("user","id").show()
+----+---+----------+------+-----------+--------------+
|user| id| date|answer|expiry_date|stale_ans_flag|
+----+---+----------+------+-----------+--------------+
|alex| 1|2022-09-01| yes| 2022-09-05| true|
|alex| 1|2022-09-05| no| null| false|
|alex| 1A|2022-09-01| NY| null| false|
|john| 1|2022-09-05| yes| null| false|
|john| 1A|2022-09-05| VA| null| false|
+----+---+----------+------+-----------+--------------+