Maptype column in pyspark
Maptype column in pyspark #pysparktutorial #dataengineering #data
from pyspark.sql.functions import *
from pyspark.sql.types import *
schema = [
"emp_id",
"name",
"marks"]
Data
data = [
(1, "Amit", {"Math": 80, "Science": 90}),
(2, "Sumit", {"Math": 70, "Science": 85}),
(3, "Neeta", {"Math": 95, "Science": 88})
]
df = spark.createDataFrame(data, schema)
df.show(truncate=False)
df.printSchema()
df.select("emp_id","name",col("marks")["Math"].alias("Math")).show()
df.select("emp_id","name",col("marks")["Science"].alias("Science")).show()
df.select("emp_id",
"name",
map_keys("marks"),
map_values("marks")
).show()
df.select("emp_id",
"name",
explode("marks")
).show()