oozie spark操作工作流无法启动

lsmepo6l  于 2021-05-24  发布在  Spark
关注(0)|答案(1)|浏览(722)

我有一个简单的Spark工作，不能通过oozie运行。相同的spark作业通过spark submit运行。我提交作业工作流并出现以下错误：

  1. 2020-10-06 11:30:05,677 INFO [main] org.apache.hadoop.service.AbstractService: Service org.apache.hadoop.mapreduce.v2.app.MRAppMaster failed in state INITED
  2. org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Error while initializing
  3.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:368)
  4.         at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164)
  5.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1760)
  6.         at java.security.AccessController.doPrivileged(Native Method)
  7.         at javax.security.auth.Subject.doAs(Subject.java:422)
  8.         at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
  9.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1757)
  10.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1691)
  11. Caused by: java.io.IOException: Couldn't create proxy provider class org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
  12.         at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:436)
  13.         at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:143)
  14.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:510)
  15.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)
  16.         at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)
  17.         at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3303)
  18.         at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124)
  19.         at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3352)
  20.         at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3320)
  21.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:479)
  22.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:227)
  23.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.getFileSystem(MRAppMaster.java:605)
  24.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:315)
  25.         ... 7 more
  26. Caused by: java.lang.reflect.InvocationTargetException
  27.         at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  28.         at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  29.         at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  30.         at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  31.         at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:425)
  32.         ... 19 more
  33. Caused by: java.lang.IllegalAccessError: tried to access class org.apache.hadoop.security.token.Token$PrivateToken from class org.apache.hadoop.hdfs.HAUtil
  34.         at org.apache.hadoop.hdfs.HAUtil.cloneDelegationTokenForLogicalUri(HAUtil.java:271)
  35.         at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:105)
  36.         ... 24 more
  37. 2020-10-06 11:30:05,689 ERROR [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Error starting MRAppMaster
  38. org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Error while initializing
  39.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:368)
  40.         at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164)
  41.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1760)
  42.         at java.security.AccessController.doPrivileged(Native Method)
  43.         at javax.security.auth.Subject.doAs(Subject.java:422)
  44.         at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
  45.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1757)
  46.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1691)
  47. Caused by: java.io.IOException: Couldn't create proxy provider class org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
  48.         at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:436)
  49.         at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:143)
  50.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:510)
  51.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)
  52.         at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)
  53.         at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3303)
  54.         at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124)
  55.         at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3352)
  56.         at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3320)
  57.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:479)
  58.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:227)
  59.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.getFileSystem(MRAppMaster.java:605)
  60.         at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceInit(MRAppMaster.java:315)
  61.         ... 7 more
  62. Caused by: java.lang.reflect.InvocationTargetException
  63.         at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  64.         at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  65.         at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  66.         at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  67.         at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:425)
  68.         ... 19 more
  69. Caused by: java.lang.IllegalAccessError: tried to access class org.apache.hadoop.security.token.Token$PrivateToken from class org.apache.hadoop.hdfs.HAUtil
  70.         at org.apache.hadoop.hdfs.HAUtil.cloneDelegationTokenForLogicalUri(HAUtil.java:271)
  71.         at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:105)
  72.         ... 24 more
  73. 2020-10-06 11:30:05,694 INFO [main] org.apache.hadoop.util.ExitUtil: Exiting with status 1: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: Error while initializing

这是job.properties：

  1. nameNodeHost=***
  2. resourceNodeHost=***
  3. nameNode=hdfs://${nameNodeHost}:8020
  4. jobTracker=http://${resourceNodeHost}:8050
  5. queueName=DataWrangling
  7. appHomeDir=/projects/msti
  8. oozie.wf.application.path=${nameNode}${appHomeDir}/Oozie/move-data/workflow-daily-data.xml
  9. resourceManager=hdfs://${resourceNodeHost}:8050
  10. oozie.use.system.libpath = true

这是工作流xml:

  1. <?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2. <workflow-app xmlns="uri:oozie:workflow:0.5" name="MSTI-Daily-Data">
  3.     <global>
  4.         <job-tracker>http://***:8050</job-tracker>
  5.         <name-node>hdfs://***:8020</name-node>
  6.     </global>
  7.     <credentials>
  8.         <credential name="hive_auth" type="hcat">
  9.             <property>
  10.                 <name>hcat.metastore.principal</name>
  11.                 <value>hive/_HOST@***/value>
  12.             </property>
  13.             <property>
  14.                 <name>hcat.metastore.uri</name>
  15.                 <value>thrift://***:9083</value>
  16.             </property>
  17.         </credential>
  18.         <credential name="hive_jdbc" type="hive2">
  19.             <property>
  20.                 <name>hive2.jdbc.url</name>
  21.                 <value>jdbc:hive2://***:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2</value>
  22.             </property>
  23.             <property>
  24.                 <name>hive2.server.principal</name>
  25.                 <value>hive/_HOST@***</value>
  26.             </property>
  27.         </credential>
  28.     </credentials>
  29.     <start to="import-sqooped-data"/>
  30.     <action name="import-sqooped-data">
  31.         <spark xmlns="uri:oozie:spark-action:0.2">
  32.             <job-tracker>${resourceManager}</job-tracker>
  33.             <name-node>${nameNode}</name-node>
  34.             <configuration>
  35.                 <property>
  36.                     <name>mapred.job.queue.name</name>
  37.                     <value>DataWrangling</value>
  38.                 </property>
  39.             </configuration>
  40.             <master>yarn-cluster</master>
  41.             <name>Daily-Ingest</name>
  42.             <class>ClassName</class>
  43.             <jar>/path-to-scala-app.jar</jar>
  44.             <spark-opts>--driver-memory 16g --master yarn --queue QueueName --executor-memory 12G --num-executors 12 --executor-cores 2</spark-opts>
  45.             <file>/path-to-scala-app.jar</file>
  46.             <file>/path-to/hive-warehouse-connector-assembly-1.0.0.3.1.0.0-78.jar</file>
  47.         </spark>
  48.         <ok to="success-e-mail"/>
  49.         <error to="failure-e-mail"/>
  50.     </action>
  51.     <action name="success-e-mail">
  52.         <email xmlns="uri:oozie:email-action:0.2">
  53.             <to>***</to>
  54.             <subject>Daily Data Job Succeeded</subject>
  55.             <body>The Oozie job completed successfully. See logs for details.</body>
  56.         </email>
  57.         <ok to="end"/>
  58.         <error to="kill"/>
  59.     </action>
  60.     <action name="failure-e-mail">
  61.         <email xmlns="uri:oozie:email-action:0.2">
  62.             <to>***</to>
  63.             <subject>Daily Data Job Failed</subject>
  64.             <body>The Oozie job failed. See logs for details.</body>
  65.         </email>
  66.         <ok to="kill"/>
  67.         <error to="kill"/>
  68.     </action>
  69.     <kill name="kill">
  70.         <message>${wf:errorMessage(wf:lastErrorNode())}</message>
  71.     </kill>
  72.     <end name="end"/>
  73. </workflow-app>

同一个集群运行其他操作和工作流没有问题。一旦spark操作成为工作流的一部分，这个错误就会导致launcher应用程序几乎立即失败。任何帮助都将不胜感激。

apache-sparkhortonworks-data-platformoozie-workflow

1条答案

按热度按时间
zfciruhq

zfciruhq1#

根据另一篇文章的建议，我将hadoop-client-3.1.1.3.1.0.0-78.jar添加到spark-share-lib文件夹中。
这不起作用，产生了这个错误。删除那个文件后，我遇到了另一个错误。
所有其他类型的oozie操作都可以工作，只是用Hive仓库连接器引发的问题。这是新的错误。当包含hive-warehouse-connector-assembly-1.0.0.3.1.0.0-78.jar时会发生这种情况：
在oozie sharelib文件夹中
在spark动作中使用标记
如果我不包括hive-warehouse-connector-assembly-1.0.0.3.1.0.0-78.jar，我显然会得到class not found exception，但是scala应用程序会被执行。

  1. ERROR [main] org.apache.hadoop.mapred.YarnChild: Error running child : java.lang.NoSuchMethodError: org.apache.hadoop.io.retry.RetryUtil
  2. s.getDefaultRetryPolicy(Lorg/apache/hadoop/conf/Configuration;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Class;)Lorg/apache/hadoop/io/ret
  3. ry/RetryPolicy;
  4.         at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:318)
  5.         at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:235)
  6.         at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:139)
  7.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:510)
  8.         at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)
  9.         at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)
  10.         at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3303)
  11.         at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124)
  12.         at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3352)
  13.         at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3320)
  14.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:479)
  15.         at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:227)
  16.         at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:173)
  17.         at java.security.AccessController.doPrivileged(Native Method)
  18.         at javax.security.auth.Subject.doAs(Subject.java:422)
  19.         at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
  20.         at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168)
展开查看全部
赞(0）分享  回复(0）举报  2021-05-25
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com