Fix read chunks (#291)
* Clean read chunked response * Remove walrus operator for python 3.7 * Simplify logic for put request * Fix tests * Deprecated field accept_put_without_content_length * Update CHANGELOG.md --------- Co-authored-by: Martin Wendt <github@wwwendt.de>
This commit is contained in:
Wermeille Bastien
GitHub
parent
96ed128c75
commit
dd3e53a5c5
6 changed files with 11 additions and 131 deletions
|
|
@ -7,7 +7,7 @@
|
|||
- Deprecate Python 3.7 (EOL: 2023-06-27)
|
||||
- Install pam_dc dependencies using extra syntax: `pip install wsgidav[pam]`
|
||||
- #281 Requesting range off end of file does not return 416 status code
|
||||
- #282 Hotfix PUT request without content-length (fix for Finder on MacOS Ventura)
|
||||
- #290 Unable to upload chunked big file using requests python library.
|
||||
- Add `logging.enable` option to activate the 'wsgidav' logger when this package
|
||||
is used as a library. This replaces an explicit call to `utils.init_logging()`.
|
||||
When running as CLI, this option is on by default.
|
||||
|
|
|
|||
|
|
@ -43,10 +43,6 @@ block_size: 8192
|
|||
add_header_MS_Author_Via: true
|
||||
|
||||
hotfixes:
|
||||
#: Some clients send PUT requests without a body and omit the
|
||||
#: `Content-Length` header ("Microsoft-WebDAV-MiniRedir", "gvfs/", "Darwin").
|
||||
#: See issues #10, #282. This option assumes "0" for missing headers:
|
||||
accept_put_without_content_length: true
|
||||
#: Handle Microsoft's Win32LastModifiedTime property.
|
||||
#: This is useful only in the case when you copy files from a Windows
|
||||
#: client into a WebDAV share. Windows sends the "last modified" time of
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ DEFAULT_CONFIG = {
|
|||
"emulate_win32_lastmod": False, # True: support Win32LastModifiedTime
|
||||
"re_encode_path_info": True, # (See issue #73)
|
||||
"unquote_path_info": False, # (See issue #8, #228)
|
||||
"accept_put_without_content_length": True, # (See issue #10, #282)
|
||||
# "accept_put_without_content_length": True, # (See issue #10, #282)
|
||||
# "treat_root_options_as_asterisk": False, # Hotfix for WinXP / Vista: accept 'OPTIONS /' for a 'OPTIONS *'
|
||||
# "win_accept_anonymous_options": False,
|
||||
# "winxp_accept_root_share_login": False,
|
||||
|
|
|
|||
|
|
@ -462,7 +462,7 @@ class FilesystemProvider(DAVProvider):
|
|||
"""
|
||||
self._count_get_resource_inst += 1
|
||||
fp = self._loc_to_file_path(path, environ)
|
||||
print(f"resolve {path} => {fp}")
|
||||
|
||||
if not os.path.exists(fp):
|
||||
return None
|
||||
if not self.fs_opts.get("follow_symlinks") and os.path.islink(fp):
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from wsgidav.dav_error import (
|
|||
HTTP_FAILED_DEPENDENCY,
|
||||
HTTP_FORBIDDEN,
|
||||
HTTP_INTERNAL_ERROR,
|
||||
HTTP_LENGTH_REQUIRED,
|
||||
HTTP_MEDIATYPE_NOT_SUPPORTED,
|
||||
HTTP_METHOD_NOT_ALLOWED,
|
||||
HTTP_NO_CONTENT,
|
||||
|
|
@ -662,93 +661,16 @@ class RequestServer:
|
|||
environ, start_response, res, HTTP_NO_CONTENT, error_list
|
||||
)
|
||||
|
||||
def _stream_data_chunked(self, environ, block_size):
|
||||
"""Get the data from a chunked transfer."""
|
||||
# Chunked Transfer Coding
|
||||
# http://www.servlets.com/rfcs/rfc2616-sec3.html#sec3.6.1
|
||||
|
||||
if "Darwin" in environ.get("HTTP_USER_AGENT", "") and environ.get(
|
||||
"HTTP_X_EXPECTED_ENTITY_LENGTH"
|
||||
):
|
||||
# Mac Finder, that does not prepend chunk-size + CRLF ,
|
||||
# like it should to comply with the spec. It sends chunk
|
||||
# size as integer in a HTTP header instead.
|
||||
WORKAROUND_CHUNK_LENGTH = True
|
||||
buf = environ.get("HTTP_X_EXPECTED_ENTITY_LENGTH", "0")
|
||||
length = int(buf)
|
||||
else:
|
||||
WORKAROUND_CHUNK_LENGTH = False
|
||||
buf = environ["wsgi.input"].readline()
|
||||
environ["wsgidav.some_input_read"] = 1
|
||||
if buf == b"":
|
||||
length = 0
|
||||
else:
|
||||
length = int(buf, 16)
|
||||
|
||||
while length > 0:
|
||||
def _stream_data(self, environ, block_size):
|
||||
"""Get the data."""
|
||||
while True:
|
||||
buf = environ["wsgi.input"].read(block_size)
|
||||
if buf == b"":
|
||||
break
|
||||
environ["wsgidav.some_input_read"] = 1
|
||||
yield buf
|
||||
if WORKAROUND_CHUNK_LENGTH:
|
||||
environ["wsgidav.some_input_read"] = 1
|
||||
# Keep receiving until we read expected size or reach
|
||||
# EOF
|
||||
if buf == b"":
|
||||
length = 0
|
||||
else:
|
||||
length -= len(buf)
|
||||
else:
|
||||
environ["wsgi.input"].readline()
|
||||
buf = environ["wsgi.input"].readline()
|
||||
if buf == b"":
|
||||
length = 0
|
||||
else:
|
||||
length = int(buf, 16)
|
||||
environ["wsgidav.all_input_read"] = 1
|
||||
|
||||
def _stream_data(self, environ, content_length, block_size):
|
||||
"""Get the data from a non-chunked transfer."""
|
||||
if content_length == 0:
|
||||
# TODO: review this
|
||||
# Windows MiniRedir submit PUT with Content-Length 0,
|
||||
# before LOCK and the real PUT. So we have to accept this.
|
||||
_logger.debug("PUT: Content-Length == 0. Creating empty file...")
|
||||
|
||||
# elif content_length < 0:
|
||||
# # TODO: review this
|
||||
# # If CONTENT_LENGTH is invalid, we may try to workaround this
|
||||
# # by reading until the end of the stream. This may block however!
|
||||
# # The iterator produced small chunks of varying size, but not
|
||||
# # sure, if we always get everything before it times out.
|
||||
# _logger.warning("PUT with invalid Content-Length (%s). "
|
||||
# "Trying to read all (this may timeout)..."
|
||||
# .format(environ.get("CONTENT_LENGTH")))
|
||||
# nb = 0
|
||||
# try:
|
||||
# for s in environ["wsgi.input"]:
|
||||
# environ["wsgidav.some_input_read"] = 1
|
||||
# _logger.debug("PUT: read from wsgi.input.__iter__, len=%s" % len(s))
|
||||
# yield s
|
||||
# nb += len (s)
|
||||
# except socket.timeout:
|
||||
# _logger.warning("PUT: input timed out after writing %s bytes" % nb)
|
||||
# hasErrors = True
|
||||
else:
|
||||
assert content_length > 0
|
||||
contentremain = content_length
|
||||
while contentremain > 0:
|
||||
n = min(contentremain, block_size)
|
||||
readbuffer = environ["wsgi.input"].read(n)
|
||||
# This happens with litmus expect-100 test:
|
||||
if not len(readbuffer) > 0:
|
||||
_logger.error("input.read({}) returned 0 bytes".format(n))
|
||||
break
|
||||
environ["wsgidav.some_input_read"] = 1
|
||||
yield readbuffer
|
||||
contentremain -= len(readbuffer)
|
||||
|
||||
if contentremain == 0:
|
||||
environ["wsgidav.all_input_read"] = 1
|
||||
|
||||
def do_PUT(self, environ, start_response):
|
||||
"""
|
||||
@see: http://www.webdav.org/specs/rfc4918.html#METHOD_PUT
|
||||
|
|
@ -788,48 +710,9 @@ class RequestServer:
|
|||
else:
|
||||
self._check_write_permission(res, "0", environ)
|
||||
|
||||
# Start Content Processing
|
||||
# Content-Length may be 0 or greater. (Set to -1 if missing or invalid.)
|
||||
try:
|
||||
content_length = max(-1, int(environ.get("CONTENT_LENGTH", -1)))
|
||||
except ValueError:
|
||||
content_length = -1
|
||||
|
||||
if (content_length < 0) and (
|
||||
environ.get("HTTP_TRANSFER_ENCODING", "").lower() != "chunked"
|
||||
):
|
||||
# HOTFIX: not fully understood, but MS sends PUT without content-length,
|
||||
# when creating new files
|
||||
|
||||
config = environ["wsgidav.config"]
|
||||
hotfixes = util.get_dict_value(config, "hotfixes", as_dict=True)
|
||||
accept_put_without_content_length = hotfixes.get(
|
||||
"accept_put_without_content_length", True
|
||||
)
|
||||
|
||||
# if ( "Microsoft-WebDAV-MiniRedir" in agent or "gvfs/" in agent):
|
||||
if accept_put_without_content_length: # issue #10, #282
|
||||
agent = environ.get("HTTP_USER_AGENT", "")
|
||||
_logger.warning(
|
||||
f"Set misssing Content-Length to 0 for PUT, agent={agent!r}"
|
||||
)
|
||||
content_length = 0
|
||||
else:
|
||||
util.fail(
|
||||
HTTP_LENGTH_REQUIRED,
|
||||
"PUT request with invalid Content-Length: ({})".format(
|
||||
environ.get("CONTENT_LENGTH")
|
||||
),
|
||||
)
|
||||
|
||||
hasErrors = False
|
||||
try:
|
||||
if environ.get("HTTP_TRANSFER_ENCODING", "").lower() == "chunked":
|
||||
data_stream = self._stream_data_chunked(environ, self.block_size)
|
||||
else:
|
||||
data_stream = self._stream_data(
|
||||
environ, content_length, self.block_size
|
||||
)
|
||||
data_stream = self._stream_data(environ, self.block_size)
|
||||
|
||||
fileobj = res.begin_write(content_type=environ.get("CONTENT_TYPE"))
|
||||
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def _check_config(config):
|
|||
deprecated_fields = {
|
||||
"acceptbasic": "http_authenticator.accept_basic",
|
||||
"acceptdigest": "http_authenticator.accept_digest",
|
||||
"accept_put_without_content_length": "(removed)",
|
||||
"catchall": "error_printer.catch_all",
|
||||
"debug_litmus": "logging.debug_litmus",
|
||||
"debug_methods": "logging.debug_methods",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue