我已经创建了select和join语句，可以在hivecli和/或beelinecli和/或spark（2.3.1）中运行 enableHiveSupport=TRUE . (注意：我正在使用sparkr作为我的api）
使用beeline进行连接和写入需要30分钟，但是使用spark进行连接和写入需要30分钟 enableHiveSupport=TRUE 需要3.5小时。这要么意味着Spark和它的连接器是垃圾，要么我没有使用Spark的方式，我应该。。。我读到的所有关于spark's best thing since sliced bread'的评论都意味着我可能没有正确使用它。
我想从Hive的table上看书，但我不想Hive做任何事。我想在每月的数据上运行连接，在每个记录的每月增量上运行回归，然后输出我的最终斜率/beta到parquet中的一个输出表，如果需要的话，它可以从hive读取。。。最好采用与我从配置单元中将表作为输入数据进行分区的相同方式进行分区。
这是一些代码，按要求。。。但我不认为你会学到什么。大数据查询不会得到可重复的结果。

Sys.setenv(SPARK_HOME="/usr/hdp/current/spark2-client")
sessionInfo()
library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))
sparkR.stop()
Sys.setenv(SPARKR_SUBMIT_ARGS="--master yarn sparkr-shell") #--master yarn-client sparkr-shell
Sys.setenv(LOCAL_DIRS="/tmp")
config = list()
config$spark.cores.max <- 144L
config$spark.executor.cores <- 2L
config$spark.executor.memory <- '8g'
config$spark.driver.cores <- 6L
config$spark.driver.maxResultSize <-"0"
config$spark.driver.memory <- "32g"
config$spark.shuffle.service.enabled<-TRUE
config$spark.dynamicAllocation.enabled <-FALSE
config$spark.scheduler.mode <- 'FIFO'
config$spark.ui.port<-4044L
sparkR.session(master = "yarn",
           sparkHome = Sys.getenv("SPARK_HOME"),
           sparkConfig = config,
           enableHiveSupport = TRUE)
print("Connected!")

############ SET HIVE CONFIG

collect(sql("SET hive.exec.dynamic.partition") )
sql("SET hive.exec.dynamic.partition=true")
collect(sql("SET hive.exec.dynamic.partition.mode"))
sql("SET hive.exec.dynamic.partition.mode=nonstrict")

## 

start_time <- Sys.time()

############### READ IN DATA {FROM HIVE}

sql('use historicdata')
data_tables<-collect(sql('show tables'))
exporttabs <- grep(pattern = 'export_historic_archive_records',x = data_tables$tableName,value = TRUE)
jointabs<-sort(exporttabs)[length(exporttabs)-(nMonths-1):0]
currenttab<-jointabs[6]

############### CREATE TABLE AND INSERT SCRIPTS

sql(paste0('use ',hivedb))
sql(paste0('DROP TABLE IF EXISTS histdata_regression',tab_suffix))
sSelect<-paste0("Insert Into TABLE histdata_regression",tab_suffix," partition (scf) SELECT a.idkey01, a.ssn7")
sCreateQuery<-paste0("CREATE TABLE histdata_regression",tab_suffix," (idkey01 string, ssn7 string")
sFrom<-paste0("FROM historicdata.",jointabs[nMonths]," a")
sAlias<-letters[nMonths:1]
DT <- gsub(pattern = "export_historic_archive_records_",replacement = "",jointabs)
DT<-paste0(DT)
for (i in nMonths:1) {
  sSelect<-paste0(sSelect,", ",sAlias[i],".",hdAttr," as ",hdAttr,"_",i,", ",sAlias[i],".recordid as recordid_",DT[i])
  sCreateQuery<-paste0(sCreateQuery,", ",hdAttr,"_",i," int, recordid_",DT[i]," int")
  if (i==1) sCreateQuery<-paste0(sCreateQuery,') PARTITIONED BY (scf string) STORED AS ORC')
  if (i==1) sSelect<-paste0(sSelect,", a.scf")
  if (i!=nMonths) sFrom<-paste0(sFrom," inner join historicdata.",jointabs[i]," ",sAlias[i]," on ",
                              paste(paste0(paste0("a.",c("scf","idkey01","ssn7")),"=",
                                     paste0(sAlias[i],".",c("scf","idkey01","ssn7"))),collapse=" AND "))
}

system(paste0('beeline -u "jdbc:hive2://myserver1.com,myserver2.com,myserver3.com,myserver4.com,myserver5.com/work;\
            serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2" -e "',sCreateQuery,'"'))

system(paste0("beeline -u \"jdbc:hive2://myserver1.com,myserver2.com,myserver3.com,myserver4.com,myserver5.com/work;\
            serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2\" -e \"",sSelect," ",sFrom,"\""))