如果你知道 elements 你要数数，然后你可以用这个 spark2.4+. 而且会很快 higher order function filter 以及 structs )

df.show()
# +------------+
# |    atr_list|
# +------------+
# |[a, b, b, c]|
# |   [b, c, d]|
# +------------+
elements=['a','b','c','d']
from pyspark.sql import functions as F
collected=df.withColumn("struct", F.struct(*[(F.struct(F.expr("size(filter(atr_list,x->x={}))"\
                                                    .format("'"+y+"'"))).alias(y)) for y in elements]))\
            .select(*[F.sum(F.col("struct.{}.col1".format(x))).alias(x) for x in elements])\
            .collect()[0]
{elements[i]: [x for x in collected][i] for i in range(len(elements))}
``` `Out: {'a': 1, 'b': 3, 'c': 2, 'd': 1}` 第二种方法，使用 `transform, aggregate, explode and groupby` （不需要指定元素）：

from pyspark.sql import functions as F

a=df.withColumn("atr", F.expr("""transform(array_distinct(atr_list),x->aggregate(atr_list,0,(acc,y)->
IF(y=x, acc+1,acc)))"""))
.withColumn("zip", F.explode(F.arrays_zip(F.array_distinct("atr_list"),("atr"))))
.select("zip.*").withColumnRenamed("0","elements")
.groupBy("elements").agg(F.sum("atr").alias("sum"))
.collect()

{a[i][0]: a[i][1] for i in range(len(a))}

转换为rdd的一种方法是将所有数组合并为一个数组，然后使用 Counter 上面有东西。

from collections import Counter
all_lists = df.select('listCol').rdd
print(Counter(all_lists.map(lambda x: [i for i in x[0]]).reduce(lambda x,y: x+y)))

另一个选择是 explode 以及 groupBy 并将结果合并到 dictionary .

from pyspark.sql.functions import explode
explode_df = df.withColumn('exploded_list',explode(df.listCol))
counts = explode_df.groupBy('exploded_list').count()
counts_tuple = counts.rdd.reduce(lambda a,b : a+b)
print({counts_tuple[i]:counts_tuple[i+1] for i in range(0,len(counts_tuple)-1,2)})

你可以试着用 distinct 以及 flatMap 方法，为此只需将列转换为和rdd并执行这些操作。

counter = (df
           .select("attr_list")
           .rdd
           # join all strings in the list and then split to get each word
           .map(lambda x: " ".join(x).split(" ")) 
           .flatMap(lambda x: x)
           # make a tuple for each word so later it can be grouped by to get its frequency count
           .map(lambda x: (x, 1))
           .reduceByKey(lambda a,b: a+b)
           .collectAsMap())