Skip to content

Commit

Permalink
Support vlen r/w with simple parent types (#124)
Browse files Browse the repository at this point in the history 
* Currently only supports variable-length of integer, floating-point or string data

* Remove unused RV_json_values_to_binary callback and RV_vlen_data_to_json
  • Loading branch information
@mattjala
mattjala authored May 28, 2024
1 parent 6e2981a commit 542eec6
Show file tree
Hide file tree
Showing 6 changed files with 1,181 additions and 413 deletions.
18 changes: 14 additions & 4 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,12 @@ jobs:
sudo apt update
sudo apt install valgrind
working-directory: ${{ github.workspace }}


# Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
- name: Fix requests version
run: |
pip install requests==2.31.0
- name: Start HSDS
if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand All @@ -133,7 +138,7 @@ jobs:
ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 &
sleep 10
working-directory: ${{github.workspace}}/hsds

- name: Test HSDS
if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand Down Expand Up @@ -240,7 +245,12 @@ jobs:
sudo apt update
sudo apt install valgrind
working-directory: ${{ github.workspace }}


# Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
- name: Fix requests version
run: |
pip install requests==2.31.0
- name: Start HSDS
if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand All @@ -254,7 +264,7 @@ jobs:
ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 &
sleep 10
working-directory: ${{github.workspace}}/hsds

- name: Test HSDS
if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}}
run: |
Expand Down
75 changes: 71 additions & 4 deletions src/rest_vol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3656,6 +3656,7 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
int maxfd = -1;
long timeout_ms = 0;
struct timeval timeout;
hid_t vlen_buf_space = H5I_INVALID_HID;

if ((failed_handles_to_retry = calloc(count, sizeof(CURL *))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
Expand Down Expand Up @@ -3757,6 +3758,8 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
fail_count++;
}
else if (response_code == 200) {
H5T_class_t dtype_class = H5T_NO_CLASS;

num_finished++;
succeed_count++;

Expand Down Expand Up @@ -3794,11 +3797,71 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
transfer_info[handle_index].curl_easy_handle = NULL;

if (transfer_info[handle_index].transfer_type == WRITE) {
RV_free(transfer_info[handle_index].u.write_info.write_body);
transfer_info[handle_index].u.write_info.write_body = NULL;
if (transfer_info[handle_index].tconv_buf) {
htri_t has_vlen = FALSE;

if ((has_vlen =
H5Tdetect_class(transfer_info[handle_index].mem_type_id, H5T_VLEN)) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL,
"can't check if dtype contains vlen");

/* Clean up memory allocated by type conversion of vlen types */
if (has_vlen > 0) {
/* Buffer was gathered before type conversion, so we can manually free vlen
* memory by iteration */
hssize_t num_elems = 0;
if ((num_elems = H5Sget_select_npoints(
transfer_info[handle_index].mem_space_id)) <= 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL,
"can't get number of elements in dataspace");

/* Vlen buffer is packed, so generate a 1D dataspace to describe its layout */
if ((vlen_buf_space = H5Screate_simple(1, &num_elems, NULL)) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL,
"can't create dataspace for vlen buffer");

if ((H5Treclaim(transfer_info[handle_index].mem_type_id, vlen_buf_space,
H5P_DEFAULT, transfer_info[handle_index].tconv_buf)) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL,
"can't free vlen data from buffer");

if (H5Sclose(vlen_buf_space) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL,
"can't close dataspace for vlen buffer");

vlen_buf_space = H5I_INVALID_HID;
}
}

if (transfer_info[handle_index].u.write_info.gather_buf) {
RV_free(transfer_info[handle_index].u.write_info.gather_buf);
transfer_info[handle_index].u.write_info.gather_buf = NULL;
}

if (transfer_info[handle_index].u.write_info.serialize_buf) {
RV_free(transfer_info[handle_index].u.write_info.serialize_buf);
transfer_info[handle_index].u.write_info.serialize_buf = NULL;
}

if (transfer_info[handle_index].u.write_info.base64_encoded_values) {
RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values);
transfer_info[handle_index].u.write_info.base64_encoded_values = NULL;
}

if (transfer_info[handle_index].u.write_info.point_sel_buf) {
RV_free(transfer_info[handle_index].u.write_info.point_sel_buf);
transfer_info[handle_index].u.write_info.point_sel_buf = NULL;
}
}

RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values);
transfer_info[handle_index].u.write_info.base64_encoded_values = NULL;
if (transfer_info[handle_index].tconv_buf) {
RV_free(transfer_info[handle_index].tconv_buf);
transfer_info[handle_index].tconv_buf = NULL;
}

if (transfer_info[handle_index].bkg_buf) {
RV_free(transfer_info[handle_index].bkg_buf);
transfer_info[handle_index].bkg_buf = NULL;
}

RV_free(transfer_info[handle_index].request_url);
Expand Down Expand Up @@ -3838,6 +3901,10 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i
done:
RV_free(failed_handles_to_retry);

if (vlen_buf_space != H5I_INVALID_HID)
if (H5Sclose(vlen_buf_space) < 0)
FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, "can't close dataspace for vlen buffer");

return ret_value;
}

Expand Down
10 changes: 7 additions & 3 deletions src/rest_vol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -586,10 +586,14 @@ struct RV_object_t {

/* Structures to hold information for cURL requests to read/write to datasets */
typedef struct dataset_write_info {
char *write_body;
char *base64_encoded_values;
curl_off_t write_len;
/* Dynamically allocated buffers for each step of the write pipeline */
void *gather_buf;
void *base64_encoded_values;
void *serialize_buf;
void *point_sel_buf;
void *vlen_buf;
upload_info uinfo;
/* Pointer to user-provided write buffer */
const void *buf;

/* If writing using compound subsetting, this is a packed version of the
Expand Down
Loading

0 comments on commit 542eec6

Please sign in to comment.