我是spark scala的新手,在尝试运行这个简单的代码(试图从kafka主题中读取内容)时,我在创建direct stream时遇到了一个错误,这表明我为重载函数createdirectstream提供了错误数量的类型参数。下面是我得到错误的那一行
val messages = KafkaUtils.createDirectStream [String, String, StringDecoder, StringDecoder]
(streamingContext, kafkaParams, topicsSet)
下面是完整的代码。
package com.test.spark
import java.util.Properties
import org.apache.spark
import kafka.serializer.StringDecoder
import org.apache.spark.streaming.kafka010._
import org.apache.spark.sql.SQLContext
import org.apache.spark.streaming._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
object KafkaAirDRsProcess {
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("AirDR Kafka to Spark")
val sc = new SparkContext(sparkConf)
val streamingContext = new StreamingContext(sc, Seconds(10))
// Create direct kafka stream with brokers and topics
val brokers = "10.21.165.145:6667 "
val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers)
val topics="AIRMAIN , dummy"
val topicsSet = topics.split(",").toSet
//val topicsSet=topics.map(_.toString).toSet
val messages = KafkaUtils.createDirectStream [String, String, StringDecoder, StringDecoder]
(streamingContext, kafkaParams, topicsSet)
val LinesDStream = messages.map(_._2)
val AirDRStream= LinesDStream.map(AirDRFilter.parseAirDR)
AirDRStream.foreachRDD(foreachFunc = rdd => {
System.out.println("--- New RDD with " + rdd.count() + " records");
if (rdd.count() > 0) {
rdd.toDF().registerTempTable("AirDRTemp")
val FilteredCDR = sqlContext.sql("select * from AirDRTemp" )
println("======================print result =================")
FilteredCDR.show()
}
});
//streamingContext.checkpoint("/tmp/mytest/ckpt/")
streamingContext.start()
streamingContext.awaitTermination()
}
}
下面是intellij错误的快照
1条答案
按热度按时间gojuced71#
因为您使用的是kafka-0-10,所以可以如下所示创建inputdstream
希望这有帮助!