可以对每个级别列执行自联接：

from pyspark.sql import functions as F
df1 = df.alias("df") \
    .join(df.alias("lvl1"), F.col("lvl1.id") == F.col("df.`level 1`"), "left") \
    .join(df.alias("lvl2"), F.col("lvl2.id") == F.col("df.`level 2`"), "left") \
    .join(df.alias("lvl3"), F.col("lvl3.id") == F.col("df.`level 3`"), "left") \
    .selectExpr("df.id", "df.name", "lvl1.name as  `level 1`", "lvl2.name as `level 2`", "lvl3.name as `level 3`")
df1.show()
# +---+---------+---------+---------+---------+
# | id|     name|  level 1|  level 2|  level 3|
# +---+---------+---------+---------+---------+
# |222|sup org 2|sup org 1|sup org 2|sup org 3|
# |333|sup org 3|sup org 1|sup org 2|sup org 3|
# |444|sup org 4|sup org 2|sup org 4|sup org 3|
# | 11|sup org 1|sup org 2|sup org 3|sup org 4|
# +---+---------+---------+---------+---------+

可以使用相关子查询从id获取相应的名称：

df.createOrReplaceTempView('df')
result = spark.sql("""
    select
        id,
        name,
        (select first(df2.name) from df as df2 where df1.`level 1` = df2.id) as `level 1`,
        (select first(df2.name) from df as df2 where df1.`level 2` = df2.id) as `level 2`,
        (select first(df2.name) from df as df2 where df1.`level 3` = df2.id) as `level 3`
    from df as df1
""")
result.show()
+---+---------+---------+---------+---------+
| id|     name|  level 1|  level 2|  level 3|
+---+---------+---------+---------+---------+
| 11|sup org 1|sup org 2|sup org 3|sup org 4|
|222|sup org 2|sup org 1|sup org 2|sup org 3|
|333|sup org 3|sup org 1|sup org 2|sup org 3|
|444|sup org 4|sup org 2|sup org 4|sup org 3|
+---+---------+---------+---------+---------+