在spark中，您的query实际上是一个filter：

from pyspark.sql.functions import col

df1 = (
    spark
    .createDataFrame([
        ('girl', 12, 'Primary'),
        ('boy', 14, 'Secondary'),
        ('baby', 1, 'Nursery')], 
        ['col_1', 'col_2', 'col_to_check']
    )
)

df2 = (
    spark
    .createDataFrame([
        ('girl', 12, 'Primary'),
        ('boy', 14, 'Secondary'),
        ('baby', 1, 'Secondary'),
        ('toddler', 3, 'Kindergarten')],
        ['col_1', 'col_2', 'col_to_check'])
    .select(
        'col_1',
        'col_2',
        col('col_to_check').alias('to_check') # alias this column so it isn't ambiguous
    )
)

df3 = (
    df1
    .join(df2, on=['col_1', 'col_2'])
    .filter('col_to_check != to_check')
)

df3.show()

+-----+-----+------------+---------+
|col_1|col_2|col_to_check| to_check|
+-----+-----+------------+---------+
| baby|    1|     Nursery|Secondary|
+-----+-----+------------+---------+

编辑

要处理NULL值，需要使用coalesce，并带有某种类型的默认返回值：

df3 = (
    df1
    .join(df2, on=['col_1', 'col_2'])
    .filter('coalesce(col_to_check, "default") != coalesce(to_check, "default")')
)

df3.show()
+------+-----+------------+---------+
| col_1|col_2|col_to_check| to_check|
+------+-----+------------+---------+
|  baby|    1|     Nursery|Secondary|
|girl_1|   10|   Secondary|     null|
|girl_2|   10|   Secondary|     null|
+------+-----+------------+---------+