我尝试从Python连接到本地运行的Hadoop集群,但一直收到以下错误:requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version
下面是我的代码:'从hdfs导入配置
HDFS类客户端:_示例=无
def __init__(self, env):
self._client = Config().get_client(env)
def write_data(self, data, hdfs_filename):
"""
Write data to an HDFS file.
"""
pass
def write_data_stream(self, iterator, hdfs_filename):
"""
Write data to an HDFS file.
Data is retrieved iteratively from @iterator.
"""
self._client.write(hdfs_filename, iterator)
@staticmethod
def get_instance(env="dev"):
if HDFSClient._instance:
return HDFSClient._instance
HDFSClient._instance = HDFSClient(env)
return HDFSClient._instance`
我的配置文件也很简单:
[global]
default.alias = dev
[dev.alias]
url = http://localhost:9000
[prod.alias]
url = http://localhost:9000
以下是完整的错误:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 436, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
# Permission is hereby granted, free of charge, to any person obtaining a copy
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 289, in _read_status
raise BadStatusLine(line)
http.client.BadStatusLine:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\util\retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
# Permission is hereby granted, free of charge, to any person obtaining a copy
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\http\client.py", line 289, in _read_status
raise BadStatusLine(line)
urllib3.exceptions.ProtocolError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version 470\x0e:\x00@\x01Hõ*'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2450, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1867, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\src\with_cors.py", line 12, in decorated_function
return f(*args, **kwargs)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\utils\log.py", line 37, in decorated
return f(*args, **kwargs)
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\app.py", line 154, in heuristic
#=======================================================================================================================
File "C:\Users\Julien\Documents\Travail\PanoptesTechnologies\block-analysis-backend\api\utils\hadoop.py", line 21, in write_data_stream
self._client.write(hdfs_filename, iterator)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 459, in write
res = self._create(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 125, in api_handler
raise err
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 102, in api_handler
res = client._request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\hdfs\client.py", line 209, in _request
return self._session.request(
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\Julien\AppData\Local\Programs\Python\Python38\Lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine('\x00\x00\x00\x7f~\x08ÿÿÿÿ\x0f\x10\x02\x18\t")org.apache.hadoop.ipc.RPC$VersionMismatch*>Server IPC version 9 cannot communicate with client version 470\x0e:\x00@\x01Hõ*'))
Python版本:3.8 hdfscli版本:2.6.0 Hadoop版本:3.2.1
我已经检查了hdsfcli所需的软件包版本。
有什么我可以尝试解决这个问题吗?谢谢你的帮助。
最好的比赛,
1条答案
按热度按时间bbmckpt71#
嗯,我也有同样的挣扎。对于上下文:这个包使用webhdfs API,在其他版本上,它运行在50070端口上,但是对于Hadoop 3.2.1和3.3.1(我的版本),它运行在9870端口上。
只需更改:
收件人:
而你是金色的。