我用Angular6前端、Django1.11后端和Hadoop3.1构建web应用程序。我需要发送任何大小和格式的文件在尽可能快的方式从angular 6前端到hadoop通过django后端。我在django中的方法如下所示。对于不同格式的小文件,一切似乎都很正常。然而,当我试图上传更大的文件,我得到的错误显示在底部。有人知道我怎么解决这个问题吗?提前谢谢。
def post(self, request):
key = request.META.get('HTTP_AUTHORIZATION').split()[1]
user_id = Token.objects.get(key=key).user_id
user_name = User.objects.get(id=user_id).username
upload_file(request.FILES['file'], user_name)
url = 'http://192.168.0.12:9864/webhdfs/v1/user/' + str(user_name) + '/' + str(request.FILES['file']) + '?op=CREATE&user.name=myuser&createflag=&createparent=true&overwrite=false'
return HttpResponseRedirect(url)
错误
django_1 | Internal Server Error: /cloud/
django_1 | Traceback (most recent call last):
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
django_1 | chunked=chunked)
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
django_1 | conn.request(method, url,**httplib_request_kw)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1239, in request
django_1 | self._send_request(method, url, body, headers, encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1285, in _send_request
django_1 | self.endheaders(body, encode_chunked=encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1234, in endheaders
django_1 | self._send_output(message_body, encode_chunked=encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1065, in _send_output
django_1 | self.send(chunk)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 986, in send
django_1 | self.sock.sendall(data)
django_1 | BrokenPipeError: [Errno 32] Broken pipe
django_1 |
django_1 | During handling of the above exception, another exception occurred:
django_1 |
django_1 | Traceback (most recent call last):
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 445, in send
django_1 | timeout=timeout
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
django_1 | _stacktrace=sys.exc_info()[2])
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/util/retry.py", line 367, in increment
django_1 | raise six.reraise(type(error), error, _stacktrace)
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
django_1 | raise value.with_traceback(tb)
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
django_1 | chunked=chunked)
django_1 | File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
django_1 | conn.request(method, url,**httplib_request_kw)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1239, in request
django_1 | self._send_request(method, url, body, headers, encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1285, in _send_request
django_1 | self.endheaders(body, encode_chunked=encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1234, in endheaders
django_1 | self._send_output(message_body, encode_chunked=encode_chunked)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 1065, in _send_output
django_1 | self.send(chunk)
django_1 | File "/usr/local/lib/python3.6/http/client.py", line 986, in send
django_1 | self.sock.sendall(data)
django_1 | urllib3.exceptions.ProtocolError: ('Connection aborted.', BrokenPipeError(32, 'Broken pipe'))
django_1 |
django_1 | During handling of the above exception, another exception occurred:
django_1 |
django_1 | Traceback (most recent call last):
django_1 | File "/usr/local/lib/python3.6/site-packages/django/core/handlers/exception.py", line 41, in inner
django_1 | response = get_response(request)
django_1 | File "/usr/local/lib/python3.6/site-packages/django/core/handlers/base.py", line 187, in _get_response
django_1 | response = self.process_exception_by_middleware(e, request)
django_1 | File "/usr/local/lib/python3.6/site-packages/django/core/handlers/base.py", line 185, in _get_response
django_1 | response = wrapped_callback(request, *callback_args,**callback_kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
django_1 | return view_func(*args,**kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/django/views/generic/base.py", line 68, in view
django_1 | return self.dispatch(request, *args,**kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 483, in dispatch
django_1 | response = self.handle_exception(exc)
django_1 | File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 443, in handle_exception
django_1 | self.raise_uncaught_exception(exc)
django_1 | File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 480, in dispatch
django_1 | response = handler(request, *args,**kwargs)
django_1 | File "/code/backend/views.py", line 38, in post
django_1 | upload_file(request.FILES['file'], user_name)
django_1 | File "/code/backend/hdfs.py", line 8, in upload_file
django_1 | response = requests.put(url + ':9870/webhdfs/v1/user/' + str(user_name) + '/' + str(file) + '?op=CREATE&user.name=myuser&createflag=&createparent=true&overwrite=false' , data=file, headers={'content-type':'application/octet-stream'})
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/api.py", line 126, in put
django_1 | return request('put', url, data=data,**kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/api.py", line 58, in request
django_1 | return session.request(method=method, url=url,**kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 512, in request
django_1 | resp = self.send(prep,**send_kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 622, in send
django_1 | r = adapter.send(request,**kwargs)
django_1 | File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 495, in send
django_1 | raise ConnectionError(err, request=request)
django_1 | requests.exceptions.ConnectionError: ('Connection aborted.', BrokenPipeError(32, 'Broken pipe'))
django_1 | [29/Sep/2018 15:05:09] "POST /cloud/ HTTP/1.1" 500 17123
django_1 | [29/Sep/2018 15:05:09] "GET /cloud/ HTTP/1.1" 200 34
django_1 | /usr/local/lib/python3.6/site-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use "pip install psycopg2-binary" instead. For details see: <http://initd.org/psycopg/docs/install.html#binary-install-from-pypi>.
django_1 | """)
1条答案
按热度按时间pn9klfpd1#
我相信我了解你的问题是什么,这是由于文件的大尺寸。对于较小的文件,上传速度很可能很小,以至于不太明显,不管是毫秒还是秒。不过,对于大得多的文件,速度可能需要更长的时间,这取决于文件大小、速度和延迟以及其他一系列网络问题。
所有这些都是因为在上传完成之前,您不会返回对post请求的响应。如果上传过程花费的时间太长,连接将关闭(也就是您看到的连接中止错误),这意味着上传过程也将失败。
一种选择是增加连接保持打开的总时间,这对于一个有大量用户的生产系统来说可能不是最好的解决方案。根据整个系统配置,有多种方法可以做到这一点。
另一个选择是在后台进行上传,例如使用celery ,这将允许上传过程发生,然后允许您关闭连接并向用户返回响应。根据您的代码,您必须更改处理问题的方式,并且可以使用celery 和/或websockets在上载完成时返回。不过,这些只是我的想法。
祝你好运!