人口普查（id:string，emptype:string,salary:int)非金属 MOXy (emptype:string,slab:int)在联接这两个rdd之后，如何过滤薪水大于50000的值以及联接表中的其他字段(Spark（scala）
uqdfh47h1#
使用 filter 或者 where 从Dataframe中筛选数据。检查以下代码。
filter
where
scala> case class Census(id:String, emptype:String, salary:Int)defined class Censusscala> case class Incometax(emptype:String,slab:Int)defined class Incometaxscala> val censusDF = Seq(Census("1","a",10000),Census("2","b",20000),Census("3","c",60000)).toDFcensusDF: org.apache.spark.sql.DataFrame = [id: string, emptype: string ... 1 more field]scala> val incometaxDF = Seq(Incometax("a",10),Incometax("b",15),Incometax("c",20)).toDFincometaxDF: org.apache.spark.sql.DataFrame = [emptype: string, slab: int]scala> censusDF.join(incometaxDF,Seq("emptype"),"left").filter(censusDF("salary") > 50000).show(false)+-------+---+------+----+|emptype|id |salary|slab|+-------+---+------+----+|c |3 |60000 |20 |+-------+---+------+----+scala> censusDF.join(incometaxDF,Seq("emptype"),"left").where(censusDF("salary") > 50000).show(false)+-------+---+------+----+|emptype|id |salary|slab|+-------+---+------+----+|c |3 |60000 |20 |+-------+---+------+----+
scala> case class Census(id:String, emptype:String, salary:Int)
defined class Census
scala> case class Incometax(emptype:String,slab:Int)
defined class Incometax
scala> val censusDF = Seq(Census("1","a",10000),Census("2","b",20000),Census("3","c",60000)).toDF
censusDF: org.apache.spark.sql.DataFrame = [id: string, emptype: string ... 1 more field]
scala> val incometaxDF = Seq(Incometax("a",10),Incometax("b",15),Incometax("c",20)).toDF
incometaxDF: org.apache.spark.sql.DataFrame = [emptype: string, slab: int]
scala> censusDF.join(incometaxDF,Seq("emptype"),"left").filter(censusDF("salary") > 50000).show(false)
+-------+---+------+----+
|emptype|id |salary|slab|
|c |3 |60000 |20 |
scala> censusDF.join(incometaxDF,Seq("emptype"),"left").where(censusDF("salary") > 50000).show(false)
1条答案
按热度按时间uqdfh47h1#
使用
filter或者where从Dataframe中筛选数据。检查以下代码。