improve data readers

docs/changes.md

@@ -1,5 +1,9 @@
 # 最新动态
 
+2023/12/24
+  * 完善data reader
+  * data\_reader\_http 支持 chunked data 
+
 2023/12/23
   * 增加函数path\_expand\_vars
 

src/base/data_reader_asset.c

@@ -81,8 +81,14 @@ data_reader_t* data_reader_asset_create(const char* assetname) {
   return_value_if_fail(assetname != NULL, NULL);
   asset = TKMEM_ZALLOC(data_reader_asset_t);
   return_value_if_fail(asset != NULL && am != NULL, NULL);
-
+
+  p = strstr(assetname, "://");
+  if (p != NULL) {
+    p += 3;
+    assetname = p;
+  }
   p = strchr(assetname, '/');
+
   if (p != NULL) {
     tk_strncpy_s(type, sizeof(type), assetname, p - assetname);
     assetname = p + 1;

src/tkc/data_reader_factory.c

@@ -100,15 +100,7 @@ data_reader_t* data_reader_factory_create_reader(data_reader_factory_t* factory,
 
   iter = darray_find(&(factory->creators), (void*)protocol);
   if (iter != NULL) {
-    p = strstr(url, "://");
-
-    if (p == NULL) {
-      p = url;
-    } else {
-      p += 3;
-    }
-
-    return iter->create(p);
+    return iter->create(url);
   }
 
   return NULL;

src/tkc/data_reader_file.c

@@ -61,11 +61,18 @@ static const data_reader_vtable_t s_data_reader_file_vtable = {
 
 data_reader_t* data_reader_file_create(const char* filename) {
   fs_stat_info_t st;
+  const char* p = NULL;
   data_reader_file_t* file = NULL;
   return_value_if_fail(filename != NULL, NULL);
   file = TKMEM_ZALLOC(data_reader_file_t);
   return_value_if_fail(file != NULL, NULL);
 
+  p = strstr(filename, "://");
+  if (p != NULL) {
+    p += 3;
+    filename = p;
+  }
+
   if (fs_stat(os_fs(), filename, &st) == RET_OK) {
     file->size = st.size;
     file->file = fs_open_file(os_fs(), filename, "rb");

src/tkc/data_reader_http.c

@@ -49,6 +49,57 @@ static const char* url_get_path(const char* url) {
   return p;
 }
 
+static uint8_t* data_reader_http_get_chunked_data(uint8_t* data, uint32_t* size) {
+  uint8_t* p = data;
+  uint8_t* q = data;
+  uint32_t chunk_size = 0;
+
+  return_value_if_fail(data != NULL && size != NULL, NULL);
+
+  while (TRUE) {
+    if (p[0] == '\r' && p[1] == '\n') {
+      break;
+    }
+    p++;
+  }
+
+  chunk_size = tk_strtol((const char*)q, NULL, 16);
+  p += 2;
+
+  if (chunk_size == 0) {
+    *size = 0;
+    return NULL;
+  }
+
+  *size = chunk_size;
+  return p;
+}
+
+static ret_t data_reader_http_decode_chuncked_data(data_reader_http_t* http, wbuffer_t* wb) {
+  uint32_t size = 0;
+  uint8_t* src = NULL;
+  uint8_t* dst = NULL;
+
+  return_value_if_fail(http != NULL && wb != NULL, RET_BAD_PARAMS);
+  http->data = TKMEM_ALLOC(wb->cursor + 1);
+  return_value_if_fail(http->data != NULL, RET_OOM);
+
+  src = wb->data;
+  dst = http->data;
+  while ((src = data_reader_http_get_chunked_data(src, &size)) != NULL) {
+    memcpy(dst, src, size);
+    dst += size;
+    src += size + 2;
+  }
+
+  http->size = dst - http->data;
+  http->data[http->size] = '\0';
+
+  log_debug("%s\n", http->data);
+
+  return RET_OK;
+}
+
 static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) {
   char buff[1024];
   int32_t nr = 0;
@@ -63,8 +114,8 @@ static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) {
   io = tk_stream_factory_create_iostream(url);
   return_value_if_fail(io != NULL, RET_BAD_PARAMS);
 
-  tk_snprintf(buff, sizeof(buff) - 1, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", url_get_path(url),
-              aurl->host);
+  tk_snprintf(buff, sizeof(buff) - 1, "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n",
+              url_get_path(url), aurl->host);
   nr = tk_iostream_write(io, buff, strlen(buff));
   url_destroy(aurl);
   goto_error_if_fail(nr > 0);
@@ -73,30 +124,56 @@ static ret_t data_reader_http_get(data_reader_http_t* http, const char* url) {
   if (nr > 0) {
     buff[nr] = '\0';
     p = strstr(buff, "Content-Length:");
-    goto_error_if_fail(p != NULL);
-    p += strlen("Content-Length:");
-    while (*p == ' ') {
-      p++;
-    }
-    http->size = tk_atoi(p);
-    goto_error_if_fail(http->size > 0);
-
-    http->data = TKMEM_ALLOC(http->size + 1);
-    goto_error_if_fail(http->data != NULL);
 
-    p = strstr(buff, "\r\n\r\n");
-    goto_error_if_fail(p != NULL);
-
-    p += 4;
-    nr = nr - (p - buff);
-    memcpy(http->data, p, nr);
-
-    if (http->size > nr) {
-      nr = tk_iostream_read_len(io, http->data + nr, http->size - nr, TK_ISTREAM_DEFAULT_TIMEOUT);
-      goto_error_if_fail(nr > 0);
+    if (p != NULL) {
+      p += strlen("Content-Length:");
+      while (*p == ' ') {
+        p++;
+      }
+      http->size = tk_atoi(p);
+      goto_error_if_fail(http->size > 0);
+
+      http->data = TKMEM_ALLOC(http->size + 1);
+      goto_error_if_fail(http->data != NULL);
+
+      p = strstr(buff, "\r\n\r\n");
+      goto_error_if_fail(p != NULL);
+
+      p += 4;
+      nr = nr - (p - buff);
+      memcpy(http->data, p, nr);
+
+      if (http->size > nr) {
+        nr = tk_iostream_read_len(io, http->data + nr, http->size - nr, TK_ISTREAM_DEFAULT_TIMEOUT);
+        goto_error_if_fail(nr > 0);
+      }
+
+      http->data[http->size] = '\0';
+    } else if (strstr(buff, "Transfer-Encoding: chunked") != NULL) {
+      wbuffer_t wb;
+
+      wbuffer_init_extendable(&wb);
+      wbuffer_extend_capacity(&wb, 10240);
+
+      p = strstr(buff, "\r\n\r\n");
+      goto_error_if_fail(p != NULL);
+
+      p += 4;
+      nr = nr - (p - buff);
+      wbuffer_write_binary(&wb, p, nr);
+
+      while ((nr = tk_iostream_read(io, buff, sizeof(buff) - 1)) > 0) {
+        if (wbuffer_write_binary(&wb, buff, nr) != RET_OK) {
+          wbuffer_deinit(&wb);
+          return RET_OOM;
+        }
+      }
+
+      data_reader_http_decode_chuncked_data(http, &wb);
+      wbuffer_deinit(&wb);
+    } else {
+      goto error;
     }
-
-    http->data[http->size] = '\0';
   }
 
   TK_OBJECT_UNREF(io);

src/tkc/data_reader_mem.c

@@ -67,11 +67,18 @@ static const data_reader_vtable_t s_data_reader_mem_vtable = {
 data_reader_t* data_reader_mem_create(const char* memname) {
   int32_t nr = 0;
   uint32_t size = 0;
+  const char* p = NULL;
   const uint8_t* data = NULL;
   data_reader_mem_t* mem = NULL;
   return_value_if_fail(memname != NULL, NULL);
   mem = TKMEM_ZALLOC(data_reader_mem_t);
   return_value_if_fail(mem != NULL, NULL);
+  p = strstr(memname, "://");
+  if (p != NULL) {
+    p += 3;
+    memname = p;
+  }
+
   nr = tk_sscanf(memname, "%p:%u", &data, &size);
 
   if (nr == 2 && data != NULL) {