我正在使用spark连接到presto。我们的查询在之后超时 60m
,以增加我设置的查询执行时间 query.max-execution-time
中的参数 getDBProperties()
就像下面一样
private def constructPrestoDataFrame(sparkSession : SparkSession, jobConfig : Config, query : String) : DataFrame = {
sparkSession
.read
.jdbc(getPrestoConnectionUrl(jobConfig), query, getDBProperties(jobConfig))
}
private def getDBProperties(jobConfig : Config) : Properties = {
val dbProperties = new Properties
dbProperties.put("user", jobConfig.getString("presto.user"))
dbProperties.put("password", jobConfig.getString("presto.password"))
dbProperties.put("Driver", jobConfig.getString("presto.driver.name"))
dbProperties.put("query.max-execution-time", "2d")
dbProperties
}
private def getPrestoConnectionUrl(jobConfig : Config) : String = {
s"jdbc:presto://${jobConfig.getString("presto.host")}:8443/${jobConfig.getString("presto.catalogue.name")}?SSL=true&SSLTrustStorePath=${jobConfig.getString("sslTrustStorePath")}"+
"&SSLTrustStorePassword="+URLEncoder.encode(jobConfig.getString("sslTrustStorePassword"))
}
当我执行任务时,我会说 exception caught: Cause = null Message = Unrecognized connection property 'query.max-execution-time'
我们使用 apache-spark-2.3.x
, presto-jdbc-driver-300
.
暂无答案!
目前还没有任何答案,快来回答吧!