diff --git a/docs/changes.md b/docs/changes.md index e479316d50..889fd13bf8 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,9 @@ # 最新动态 +2023/12/24 + * 完善data reader + * data\_reader\_http 支持 chunked data + 2023/12/23 * 增加函数path\_expand\_vars diff --git a/src/base/data_reader_asset.c b/src/base/data_reader_asset.c index cf7f630c19..0c05c67d34 100644 --- a/src/base/data_reader_asset.c +++ b/src/base/data_reader_asset.c @@ -81,8 +81,14 @@ data_reader_t* data_reader_asset_create(const char* assetname) { return_value_if_fail(assetname != NULL, NULL); asset = TKMEM_ZALLOC(data_reader_asset_t); return_value_if_fail(asset != NULL && am != NULL, NULL); - + + p = strstr(assetname, "://"); + if (p != NULL) { + p += 3; + assetname = p; + } p = strchr(assetname, '/'); + if (p != NULL) { tk_strncpy_s(type, sizeof(type), assetname, p - assetname); assetname = p + 1; diff --git a/src/tkc/data_reader_factory.c b/src/tkc/data_reader_factory.c index e0fd64fbb8..ce12535dc1 100644 --- a/src/tkc/data_reader_factory.c +++ b/src/tkc/data_reader_factory.c @@ -100,15 +100,7 @@ data_reader_t* data_reader_factory_create_reader(data_reader_factory_t* factory, iter = darray_find(&(factory->creators), (void*)protocol); if (iter != NULL) { - p = strstr(url, "://"); - - if (p == NULL) { - p = url; - } else { - p += 3; - } - - return iter->create(p); + return iter->create(url); } return NULL; diff --git a/src/tkc/data_reader_file.c b/src/tkc/data_reader_file.c index a071dd5056..cfc3c2e28b 100644 --- a/src/tkc/data_reader_file.c +++ b/src/tkc/data_reader_file.c @@ -61,11 +61,18 @@ static const data_reader_vtable_t s_data_reader_file_vtable = { data_reader_t* data_reader_file_create(const char* filename) { fs_stat_info_t st; + const char* p = NULL; data_reader_file_t* file = NULL; return_value_if_fail(filename != NULL, NULL); file = TKMEM_ZALLOC(data_reader_file_t); return_value_if_fail(file != NULL, NULL); + p = strstr(filename, "://"); + if (p != NULL) { + p += 3; + filename = p; + } + if (fs_stat(os_fs(), filename, &st) == RET_OK) { file->size = st.size; file->file = fs_open_file(os_fs(), filename, "rb"); diff --git a/src/tkc/data_reader_http.c b/src/tkc/data_reader_http.c index dfd3f9d791..9a056095d5 100644 --- a/src/tkc/data_reader_http.c +++ b/src/tkc/data_reader_http.c @@ -49,6 +49,57 @@ static const char* url_get_path(const char* url) { return p; } +static uint8_t* data_reader_http_get_chunked_data(uint8_t* data, uint32_t* size) { + uint8_t* p = data; + uint8_t* q = data; + uint32_t chunk_size = 0; + + return_value_if_fail(data != NULL && size != NULL, NULL); + + while (TRUE) { + if (p[0] == '\r' && p[1] == '\n') { + break; + } + p++; + } + + chunk_size = tk_strtol((const char*)q, NULL, 16); + p += 2; + + if (chunk_size == 0) { + *size = 0; + return NULL; + } + + *size = chunk_size; + return p; +} + +static ret_t data_reader_http_decode_chuncked_data(data_reader_http_t* http, wbuffer_t* wb) { + uint32_t size = 0; + uint8_t* src = NULL; + uint8_t* dst = NULL; + + return_value_if_fail(http != NULL && wb != NULL, RET_BAD_PARAMS); + http->data = TKMEM_ALLOC(wb->cursor + 1); + return_value_if_fail(http->data != NULL, RET_OOM); + + src = wb->data; + dst = http->data; + while ((src = data_reader_http_get_chunked_data(src, &size)) != NULL) { + memcpy(dst, src, size); + dst += size; + src += size + 2; + } + + http->size = dst - http->data; + http->data[http->size] = '\0'; + + log_debug("%s\n", http->data); + + return RET_OK; +} + static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) { char buff[1024]; int32_t nr = 0; @@ -63,8 +114,8 @@ static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) { io = tk_stream_factory_create_iostream(url); return_value_if_fail(io != NULL, RET_BAD_PARAMS); - tk_snprintf(buff, sizeof(buff) - 1, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", url_get_path(url), - aurl->host); + tk_snprintf(buff, sizeof(buff) - 1, "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n", + url_get_path(url), aurl->host); nr = tk_iostream_write(io, buff, strlen(buff)); url_destroy(aurl); goto_error_if_fail(nr > 0); @@ -73,30 +124,56 @@ static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) { if (nr > 0) { buff[nr] = '\0'; p = strstr(buff, "Content-Length:"); - goto_error_if_fail(p != NULL); - p += strlen("Content-Length:"); - while (*p == ' ') { - p++; - } - http->size = tk_atoi(p); - goto_error_if_fail(http->size > 0); - - http->data = TKMEM_ALLOC(http->size + 1); - goto_error_if_fail(http->data != NULL); - p = strstr(buff, "\r\n\r\n"); - goto_error_if_fail(p != NULL); - - p += 4; - nr = nr - (p - buff); - memcpy(http->data, p, nr); - - if (http->size > nr) { - nr = tk_iostream_read_len(io, http->data + nr, http->size - nr, TK_ISTREAM_DEFAULT_TIMEOUT); - goto_error_if_fail(nr > 0); + if (p != NULL) { + p += strlen("Content-Length:"); + while (*p == ' ') { + p++; + } + http->size = tk_atoi(p); + goto_error_if_fail(http->size > 0); + + http->data = TKMEM_ALLOC(http->size + 1); + goto_error_if_fail(http->data != NULL); + + p = strstr(buff, "\r\n\r\n"); + goto_error_if_fail(p != NULL); + + p += 4; + nr = nr - (p - buff); + memcpy(http->data, p, nr); + + if (http->size > nr) { + nr = tk_iostream_read_len(io, http->data + nr, http->size - nr, TK_ISTREAM_DEFAULT_TIMEOUT); + goto_error_if_fail(nr > 0); + } + + http->data[http->size] = '\0'; + } else if (strstr(buff, "Transfer-Encoding: chunked") != NULL) { + wbuffer_t wb; + + wbuffer_init_extendable(&wb); + wbuffer_extend_capacity(&wb, 10240); + + p = strstr(buff, "\r\n\r\n"); + goto_error_if_fail(p != NULL); + + p += 4; + nr = nr - (p - buff); + wbuffer_write_binary(&wb, p, nr); + + while ((nr = tk_iostream_read(io, buff, sizeof(buff) - 1)) > 0) { + if (wbuffer_write_binary(&wb, buff, nr) != RET_OK) { + wbuffer_deinit(&wb); + return RET_OOM; + } + } + + data_reader_http_decode_chuncked_data(http, &wb); + wbuffer_deinit(&wb); + } else { + goto error; } - - http->data[http->size] = '\0'; } TK_OBJECT_UNREF(io); diff --git a/src/tkc/data_reader_mem.c b/src/tkc/data_reader_mem.c index a908af62db..98b9f4d2a2 100644 --- a/src/tkc/data_reader_mem.c +++ b/src/tkc/data_reader_mem.c @@ -67,11 +67,18 @@ static const data_reader_vtable_t s_data_reader_mem_vtable = { data_reader_t* data_reader_mem_create(const char* memname) { int32_t nr = 0; uint32_t size = 0; + const char* p = NULL; const uint8_t* data = NULL; data_reader_mem_t* mem = NULL; return_value_if_fail(memname != NULL, NULL); mem = TKMEM_ZALLOC(data_reader_mem_t); return_value_if_fail(mem != NULL, NULL); + p = strstr(memname, "://"); + if (p != NULL) { + p += 3; + memname = p; + } + nr = tk_sscanf(memname, "%p:%u", &data, &size); if (nr == 2 && data != NULL) {