DataX ha模式下,hdfs 到hdfs报错

y3bcpkx1  于 2022-10-25  发布在  HDFS
关注(0)|答案(2)|浏览(377)
java.io.IOException: Couldn't create proxy provider class org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
	at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:516)
	at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:170)
	at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:690)
	at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:631)
	at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:160)
	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2795)
	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99)
	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2829)
	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2811)
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390)
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:179)
	at com.alibaba.datax.plugin.reader.hdfsreader.DFSUtil.getInputStream(DFSUtil.java:267)
	at com.alibaba.datax.plugin.reader.hdfsreader.HdfsReader$Task.startRead(HdfsReader.java:354)
	at com.alibaba.datax.core.taskgroup.runner.ReaderRunner.run(ReaderRunner.java:62)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:499)
	... 14 more
Caused by: java.lang.IllegalArgumentException: Interface class %s is not a valid NameNode protocol!
	at com.google.common.base.Preconditions.checkArgument(Preconditions.java:88)
	at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:91)
	at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:84)
	... 19 more

	at com.alibaba.datax.common.exception.DataXException.asDataXException(DataXException.java:33) ~[datax-common-0.0.1-SNAPSHOT.jar:na]
	at com.alibaba.datax.plugin.reader.hdfsreader.DFSUtil.getInputStream(DFSUtil.java:274) ~[hdfsreader-0.0.1-SNAPSHOT.jar:na]
	at com.alibaba.datax.plugin.reader.hdfsreader.HdfsReader$Task.startRead(HdfsReader.java:354) ~[hdfsreader-0.0.1-SNAPSHOT.jar:na]
	at com.alibaba.datax.core.taskgroup.runner.ReaderRunner.run(ReaderRunner.java:62) ~[datax-core-0.0.1-SNAPSHOT.jar:na]
	at java.lang.Thread.run(Thread.java:748) [na:1.8.0_144]
Caused by: java.io.IOException: Couldn't create proxy provider class org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
	at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:516) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:170) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:690) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:631) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:160) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2795) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2829) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2811) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:179) ~[hadoop-common-2.7.3.2.6.0.3-8.jar:na]
	at com.alibaba.datax.plugin.reader.hdfsreader.DFSUtil.getInputStream(DFSUtil.java:267) ~[hdfsreader-0.0.1-SNAPSHOT.jar:na]
	... 3 common frames omitted
Caused by: java.lang.reflect.InvocationTargetException: null
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.8.0_144]
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[na:1.8.0_144]
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.8.0_144]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[na:1.8.0_144]
	at org.apache.hadoop.hdfs.NameNodeProxies.createFailoverProxyProvider(NameNodeProxies.java:499) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	... 14 common frames omitted
Caused by: java.lang.IllegalArgumentException: Interface class %s is not a valid NameNode protocol!
	at com.google.common.base.Preconditions.checkArgument(Preconditions.java:88) ~[guava-11.0.2.jar:na]
	at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:91) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.<init>(ConfiguredFailoverProxyProvider.java:84) ~[hadoop-hdfs-2.7.3.2.6.0.3-8.jar:na]
	... 19 common frames omitted
svgewumm

svgewumm1#

如果目的端或者源端只有一个为hdfs,则没有这个问题,有没有人有解决方案

v7pvogib

v7pvogib2#

重写下 把hadoop配置文件加入就好了

相关问题